def main(root_dir, dst_image): transport = transport_pool.Http(httplib2.Http, size=32) cache_dir = os.path.join(root_dir, '.cache') if not os.path.exists(cache_dir): os.makedirs(cache_dir) base = docker_name.Tag('gcr.io/google-appengine/nodejs:latest') creds = docker_creds.DefaultKeychain.Resolve(base) with docker_image.FromRegistry(base, creds, transport) as base_img: npm = NPM(root_dir, base_img) for i in range(10): start = time.time() dst = docker_name.Tag(dst_image) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=32) as session: img = handle_app(npm, cache_dir) session.upload(img) print time.time() - start
def main(): args = parser.parse_args() transport = transport_pool.Http(httplib2.Http, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. name = docker_name.Tag(args.name) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name) with docker_session.Push(name, creds, transport, threads=_THREADS) as session: with v2_2_image.FromTarball(args.tarball) as v2_2_img: session.upload(v2_2_img)
def main(): logging_setup.DefineCommandLineArgs(parser) args = parser.parse_args() logging_setup.Init(args=args) if not args.name or not args.tarball: logging.fatal('--name and --tarball are required arguments.') retry_factory = retry.Factory() retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http) transport = transport_pool.Http(retry_factory.Build, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. name = Tag(args.name, args.stamp_info_file) logging.info('Reading v2.2 image from tarball %r', args.tarball) with v2_2_image.FromTarball(args.tarball) as v2_2_img: # Resolve the appropriate credential to use based on the standard Docker # client logic. try: creds = docker_creds.DefaultKeychain.Resolve(name) # pylint: disable=broad-except except Exception as e: logging.fatal('Error resolving credentials for %s: %s', name, e) try: with docker_session.Push(name, creds, transport, threads=_THREADS) as session: logging.info('Starting upload ...') if args.oci: with oci_compat.OCIFromV22(v2_2_img) as oci_img: session.upload(oci_img) digest = oci_img.digest() else: session.upload(v2_2_img) digest = v2_2_img.digest() print('{name} was published with digest: {digest}'.format( name=name, digest=digest)) # pylint: disable=broad-except except Exception as e: logging.fatal('Error publishing %s: %s', name, e)
def add_tags(self, digest, tag, dry_run): if dry_run: logging.debug('Would have tagged {0} with {1}'.format(digest, tag)) return src_name = docker_name.Digest(digest) dest_name = docker_name.Tag(tag) creds = docker_creds.DefaultKeychain.Resolve(src_name) transport = transport_pool.Http(httplib2.Http) with docker_image.FromRegistry(src_name, creds, transport) as src_img: if src_img.exists(): creds = docker_creds.DefaultKeychain.Resolve(dest_name) logging.debug('Tagging {0} with {1}'.format(digest, tag)) with docker_session.Push(dest_name, creds, transport) as push: push.upload(src_img) else: logging.debug("""Unable to tag {0} as the image can't be found""".format(digest))
def StoreImage(self, result_image): if self._args.output_path: with ftl_util.Timing("saving_tarball_image"): with tarfile.open(name=self._args.output_path, mode='w') as tar: save.tarball(self._target_image, result_image, tar) logging.info("{0} tarball located at {1}".format( str(self._target_image), self._args.output_path)) return if self._args.upload: with ftl_util.Timing("pushing_image_to_docker_registry"): with docker_session.Push(self._target_image, self._target_creds, self._transport, threads=_THREADS, mount=[self._base_name]) as session: logging.info('Pushing final image...') session.upload(result_image) return
def main(): args = parser.parse_args() transport = transport_pool.Http(httplib2.Http, size=_THREADS) # TODO(mattmoor): Support digest base images. base_name = docker_name.Tag(args.base) base_creds = docker_creds.DefaultKeychain.Resolve(base_name) target_image = docker_name.Tag(args.name) target_creds = docker_creds.DefaultKeychain.Resolve(target_image) with context.Workspace(args.directory) as ctx: with cache.Registry(target_image.as_repository(), target_creds, transport, threads=_THREADS, mount=[base_name]) as cash: with builder.From(ctx) as bldr: with docker_image.FromRegistry(base_name, base_creds, transport) as base_image: # Create (or pull from cache) the base image with the package # descriptor installation overlaid. with bldr.CreatePackageBase(base_image, cash) as base_with_deps: # Construct the application layer from the context. app_layer = bldr.BuildAppLayer() with append.Layer(base_with_deps, app_layer) as app_image: with docker_session.Push(target_image, target_creds, transport, threads=_THREADS, mount=[base_name ]) as session: session.upload(app_image) print("Hi, One Build.")
def _push_image(image_reference: str, image_file: str, credentials_lookup: typing.Callable[ [image_reference, oa.Privileges, bool], oa.OciConfig], threads=8): if not image_reference: raise ValueError(image_reference) if not os.path.isfile(image_file): raise ValueError(f'not an exiting file: {image_file=}') transport = _mk_transport_pool() image_reference = ou.normalise_image_reference(image_reference) image_reference = docker_name.from_string(image_reference) creds = _mk_credentials( image_reference=image_reference, credentials_lookup=credentials_lookup, privileges=oa.Privileges.READWRITE, ) # XXX fail if no creds were found with v2_2_image.FromTarball(image_file) as v2_2_img: try: with docker_session.Push( image_reference, creds, transport, threads=threads, ) as session: session.upload(v2_2_img) digest = v2_2_img.digest() logger.info( f'{image_reference} was uploaded - digest: {digest}') except Exception as e: import traceback traceback.print_exc() raise e
def main(args): args = parser.parse_args(args) logging.getLogger().setLevel(_LEVEL_MAP[args.verbosity]) transport = transport_pool.Http(httplib2.Http, size=_THREADS) # TODO(mattmoor): Support digest base images. base_name = docker_name.Tag(args.base) base_creds = docker_creds.DefaultKeychain.Resolve(base_name) target_image = docker_name.Tag(args.name) target_creds = docker_creds.DefaultKeychain.Resolve(target_image) ctx = context.Workspace(args.directory) cash = cache.Registry(target_image.as_repository(), target_creds, transport, threads=_THREADS, mount=[base_name]) bldr = builder.From(ctx) with docker_image.FromRegistry(base_name, base_creds, transport) as base_image: # Create (or pull from cache) the base image with the # package descriptor installation overlaid. logging.info('Generating dependency layer...') with bldr.CreatePackageBase(base_image, cash) as deps: # Construct the application layer from the context. logging.info('Generating app layer...') app_layer, diff_id = bldr.BuildAppLayer() with append.Layer(deps, app_layer, diff_id=diff_id) as app_image: with docker_session.Push(target_image, target_creds, transport, threads=_THREADS, mount=[base_name]) as session: logging.info('Pushing final image...') session.upload(app_image)
def _push_image(image_reference: str, image_file: str, threads=8): import util util.not_none(image_reference) util.existing_file(image_file) transport = _mk_transport() image_reference = normalise_image_reference(image_reference) name = _parse_image_reference(image_reference) try: # first try container_registry cfgs from available cfg creds = _credentials(image_reference=image_reference, privileges=Privileges.READ_WRITE) if not creds: print('could not find rw-creds') # fall-back to default docker lookup creds = docker_creds.DefaultKeychain.Resolve(name) except Exception as e: util.fail('Error resolving credentials for {name}: {e}'.format( name=name, e=e)) with v2_2_image.FromTarball(image_file) as v2_2_img: try: with docker_session.Push( name, creds, transport, threads=threads, ) as session: session.upload(v2_2_img) digest = v2_2_img.digest() print(f'{name} was uploaded - digest: {digest}') except Exception as e: import traceback traceback.print_exc() raise e
def main(): logging_setup.DefineCommandLineArgs(parser) args = parser.parse_args() logging_setup.Init(args=args) if not args.src_image or not args.tarball or not args.dst_image: raise Exception('--src-image, --dst-image and --tarball are required ' 'arguments.') transport = transport_pool.Http(httplib2.Http, size=_THREADS) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. src = docker_name.Tag(args.src_image) dst = docker_name.Tag(args.dst_image) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(src) logging.info('Pulling v2.2 image from %r ...', src) with v2_2_image.FromRegistry(src, creds, transport) as src_image: with open(args.tarball, 'rb') as f: new_img = append.Layer(src_image, f.read()) creds = docker_creds.DefaultKeychain.Resolve(dst) with docker_session.Push(dst, creds, transport, threads=_THREADS, mount=[src.as_repository()]) as session: logging.info('Starting upload ...') session.upload(new_img) digest = new_img.digest() print('{name} was published with digest: {digest}'.format( name=dst, digest=digest))
def main(): logging_setup.DefineCommandLineArgs(parser) args = parser.parse_args() logging_setup.Init(args=args) if not args.name: logging.fatal('--name is a required arguments.') sys.exit(1) # This library can support push-by-digest, but the likelihood of a user # correctly providing us with the digest without using this library # directly is essentially nil. name = Tag(args.name, args.stamp_info_file) if not args.config and (args.layer or args.digest): logging.fatal( 'Using --layer or --digest requires --config to be specified.') sys.exit(1) if not args.config and not args.tarball: logging.fatal('Either --config or --tarball must be specified.') sys.exit(1) # If config is specified, use that. Otherwise, fallback on reading # the config from the tarball. config = args.config if args.config: logging.info('Reading config from %r', args.config) with open(args.config, 'r') as reader: config = reader.read() elif args.tarball: logging.info('Reading config from tarball %r', args.tarball) with v2_2_image.FromTarball(args.tarball) as base: config = base.config_file() if len(args.digest or []) != len(args.layer or []): logging.fatal('--digest and --layer must have matching lengths.') sys.exit(1) retry_factory = retry.Factory() retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http) transport = transport_pool.Http(retry_factory.Build, size=_THREADS) logging.info('Loading v2.2 image from disk ...') with v2_2_image.FromDisk(config, zip(args.digest or [], args.layer or []), legacy_base=args.tarball) as v2_2_img: # Resolve the appropriate credential to use based on the standard Docker # client logic. try: creds = docker_creds.DefaultKeychain.Resolve(name) # pylint: disable=broad-except except Exception as e: logging.fatal('Error resolving credentials for %s: %s', name, e) sys.exit(1) try: with docker_session.Push(name, creds, transport, threads=_THREADS) as session: logging.info('Starting upload ...') if args.oci: with oci_compat.OCIFromV22(v2_2_img) as oci_img: session.upload(oci_img) digest = oci_img.digest() else: session.upload(v2_2_img) digest = v2_2_img.digest() print('{name} was published with digest: {digest}'.format( name=name, digest=digest)) # pylint: disable=broad-except except Exception as e: logging.fatal('Error publishing %s: %s', name, e) sys.exit(1)
while (chunk := fileobj.read(4096)): sha256_hash.update(chunk) sha256_digest = sha256_hash.hexdigest() fileobj.seek(0) logger.debug(f'{sha256_digest=}') image_ref = image_name image_name = _ou.docker_name.from_string(image_name) contents = fileobj.read() from containerregistry.client.v2_2 import docker_session push_sess = docker_session.Push( name=image_name, creds=_ou._mk_credentials( image_reference=image_ref, privileges=oa.Privileges.READWRITE, credentials_lookup=credentials_lookup, ), transport=_ou._mk_transport_pool(), ) logger.debug(f'{len(contents)=}') # XXX superdirty hack - force usage of our blob :( push_sess._get_blob = lambda a, b: contents push_sess._patch_upload( image_name, f'sha256:{sha256_digest}', ) logger.debug(f'successfully pushed {image_name=} {sha256_digest=}') return sha256_digest
def Publish(transport, image_chroot, stamp_info={}, name=None, tarball=None, config=None, digest=None, layer=None): if not name: raise Exception('Expected "name" kwarg') if not config and (layer or digest): raise Exception( name + ': Using "layer" or "digest" requires "config" to be specified.') if config: with open(config, 'r') as reader: config = reader.read() elif tarball: with v2_2_image.FromTarball(tarball) as base: config = base.config_file() else: raise Exception(name + ': Either "config" or "tarball" must be specified.') if digest or layer: digest = digest.split(',') layer = layer.split(',') if len(digest) != len(layer): raise Exception( name + ': "digest" and "layer" must have matching lengths.') else: digest = [] layer = [] resolved_name = name.format(**stamp_info) name_to_replace = name if image_chroot: name_to_publish = docker_name.Tag(os.path.join(image_chroot, resolved_name), strict=False) else: # Without a chroot, the left-hand-side must be a valid tag. name_to_publish = docker_name.Tag(resolved_name, strict=False) # Resolve the appropriate credential to use based on the standard Docker # client logic. creds = docker_creds.DefaultKeychain.Resolve(name_to_publish) with v2_2_session.Push(name_to_publish, creds, transport, threads=_THREADS) as session: with v2_2_image.FromDisk(config, zip(digest or [], layer or []), legacy_base=tarball) as v2_2_img: session.upload(v2_2_img) return (name_to_replace, str(name_to_publish), docker_name.Digest('{repository}@{digest}'.format( repository=name_to_publish.as_repository(), digest=v2_2_img.digest())))