def main(root_dir, dst_image):
transport = transport_pool.Http(httplib2.Http, size=32)
cache_dir = os.path.join(root_dir, '.cache')
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
base = docker_name.Tag('gcr.io/google-appengine/nodejs:latest')
creds = docker_creds.DefaultKeychain.Resolve(base)
with docker_image.FromRegistry(base, creds, transport) as base_img:
npm = NPM(root_dir, base_img)
for i in range(10):
start = time.time()
dst = docker_name.Tag(dst_image)
creds = docker_creds.DefaultKeychain.Resolve(dst)
with docker_session.Push(dst, creds, transport,
threads=32) as session:
img = handle_app(npm, cache_dir)
session.upload(img)
print time.time() - start
def main():
args = parser.parse_args()
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# This library can support push-by-digest, but the likelihood of a user
# correctly providing us with the digest without using this library
# directly is essentially nil.
name = docker_name.Tag(args.name)
# Resolve the appropriate credential to use based on the standard Docker
# client logic.
creds = docker_creds.DefaultKeychain.Resolve(name)
with docker_session.Push(name, creds, transport,
threads=_THREADS) as session:
with v2_2_image.FromTarball(args.tarball) as v2_2_img:
session.upload(v2_2_img)
def main():
logging_setup.DefineCommandLineArgs(parser)
args = parser.parse_args()
logging_setup.Init(args=args)
if not args.name or not args.tarball:
logging.fatal('--name and --tarball are required arguments.')
retry_factory = retry.Factory()
retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http)
transport = transport_pool.Http(retry_factory.Build, size=_THREADS)
# This library can support push-by-digest, but the likelihood of a user
# correctly providing us with the digest without using this library
# directly is essentially nil.
name = Tag(args.name, args.stamp_info_file)
logging.info('Reading v2.2 image from tarball %r', args.tarball)
with v2_2_image.FromTarball(args.tarball) as v2_2_img:
# Resolve the appropriate credential to use based on the standard Docker
# client logic.
try:
creds = docker_creds.DefaultKeychain.Resolve(name)
# pylint: disable=broad-except
except Exception as e:
logging.fatal('Error resolving credentials for %s: %s', name, e)
try:
with docker_session.Push(name, creds, transport,
threads=_THREADS) as session:
logging.info('Starting upload ...')
if args.oci:
with oci_compat.OCIFromV22(v2_2_img) as oci_img:
session.upload(oci_img)
digest = oci_img.digest()
else:
session.upload(v2_2_img)
digest = v2_2_img.digest()
print('{name} was published with digest: {digest}'.format(
name=name, digest=digest))
# pylint: disable=broad-except
except Exception as e:
logging.fatal('Error publishing %s: %s', name, e)
def add_tags(self, digest, tag, dry_run):
if dry_run:
logging.debug('Would have tagged {0} with {1}'.format(digest, tag))
return
src_name = docker_name.Digest(digest)
dest_name = docker_name.Tag(tag)
creds = docker_creds.DefaultKeychain.Resolve(src_name)
transport = transport_pool.Http(httplib2.Http)
with docker_image.FromRegistry(src_name, creds, transport) as src_img:
if src_img.exists():
creds = docker_creds.DefaultKeychain.Resolve(dest_name)
logging.debug('Tagging {0} with {1}'.format(digest, tag))
with docker_session.Push(dest_name, creds, transport) as push:
push.upload(src_img)
else:
logging.debug("""Unable to tag {0}
as the image can't be found""".format(digest))
def StoreImage(self, result_image):
if self._args.output_path:
with ftl_util.Timing("saving_tarball_image"):
with tarfile.open(name=self._args.output_path,
mode='w') as tar:
save.tarball(self._target_image, result_image, tar)
logging.info("{0} tarball located at {1}".format(
str(self._target_image), self._args.output_path))
return
if self._args.upload:
with ftl_util.Timing("pushing_image_to_docker_registry"):
with docker_session.Push(self._target_image,
self._target_creds,
self._transport,
threads=_THREADS,
mount=[self._base_name]) as session:
logging.info('Pushing final image...')
session.upload(result_image)
return
def main():
args = parser.parse_args()
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# TODO(mattmoor): Support digest base images.
base_name = docker_name.Tag(args.base)
base_creds = docker_creds.DefaultKeychain.Resolve(base_name)
target_image = docker_name.Tag(args.name)
target_creds = docker_creds.DefaultKeychain.Resolve(target_image)
with context.Workspace(args.directory) as ctx:
with cache.Registry(target_image.as_repository(),
target_creds,
transport,
threads=_THREADS,
mount=[base_name]) as cash:
with builder.From(ctx) as bldr:
with docker_image.FromRegistry(base_name, base_creds,
transport) as base_image:
# Create (or pull from cache) the base image with the package
# descriptor installation overlaid.
with bldr.CreatePackageBase(base_image,
cash) as base_with_deps:
# Construct the application layer from the context.
app_layer = bldr.BuildAppLayer()
with append.Layer(base_with_deps,
app_layer) as app_image:
with docker_session.Push(target_image,
target_creds,
transport,
threads=_THREADS,
mount=[base_name
]) as session:
session.upload(app_image)
print("Hi, One Build.")
def _push_image(image_reference: str,
image_file: str,
credentials_lookup: typing.Callable[
[image_reference, oa.Privileges, bool], oa.OciConfig],
threads=8):
if not image_reference:
raise ValueError(image_reference)
if not os.path.isfile(image_file):
raise ValueError(f'not an exiting file: {image_file=}')
transport = _mk_transport_pool()
image_reference = ou.normalise_image_reference(image_reference)
image_reference = docker_name.from_string(image_reference)
creds = _mk_credentials(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
privileges=oa.Privileges.READWRITE,
)
# XXX fail if no creds were found
with v2_2_image.FromTarball(image_file) as v2_2_img:
try:
with docker_session.Push(
image_reference,
creds,
transport,
threads=threads,
) as session:
session.upload(v2_2_img)
digest = v2_2_img.digest()
logger.info(
f'{image_reference} was uploaded - digest: {digest}')
except Exception as e:
import traceback
traceback.print_exc()
raise e
def main(args):
args = parser.parse_args(args)
logging.getLogger().setLevel(_LEVEL_MAP[args.verbosity])
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# TODO(mattmoor): Support digest base images.
base_name = docker_name.Tag(args.base)
base_creds = docker_creds.DefaultKeychain.Resolve(base_name)
target_image = docker_name.Tag(args.name)
target_creds = docker_creds.DefaultKeychain.Resolve(target_image)
ctx = context.Workspace(args.directory)
cash = cache.Registry(target_image.as_repository(),
target_creds,
transport,
threads=_THREADS,
mount=[base_name])
bldr = builder.From(ctx)
with docker_image.FromRegistry(base_name, base_creds,
transport) as base_image:
# Create (or pull from cache) the base image with the
# package descriptor installation overlaid.
logging.info('Generating dependency layer...')
with bldr.CreatePackageBase(base_image, cash) as deps:
# Construct the application layer from the context.
logging.info('Generating app layer...')
app_layer, diff_id = bldr.BuildAppLayer()
with append.Layer(deps, app_layer, diff_id=diff_id) as app_image:
with docker_session.Push(target_image,
target_creds,
transport,
threads=_THREADS,
mount=[base_name]) as session:
logging.info('Pushing final image...')
session.upload(app_image)
def _push_image(image_reference: str, image_file: str, threads=8):
import util
util.not_none(image_reference)
util.existing_file(image_file)
transport = _mk_transport()
image_reference = normalise_image_reference(image_reference)
name = _parse_image_reference(image_reference)
try:
# first try container_registry cfgs from available cfg
creds = _credentials(image_reference=image_reference,
privileges=Privileges.READ_WRITE)
if not creds:
print('could not find rw-creds')
# fall-back to default docker lookup
creds = docker_creds.DefaultKeychain.Resolve(name)
except Exception as e:
util.fail('Error resolving credentials for {name}: {e}'.format(
name=name, e=e))
with v2_2_image.FromTarball(image_file) as v2_2_img:
try:
with docker_session.Push(
name,
creds,
transport,
threads=threads,
) as session:
session.upload(v2_2_img)
digest = v2_2_img.digest()
print(f'{name} was uploaded - digest: {digest}')
except Exception as e:
import traceback
traceback.print_exc()
raise e
def main():
logging_setup.DefineCommandLineArgs(parser)
args = parser.parse_args()
logging_setup.Init(args=args)
if not args.src_image or not args.tarball or not args.dst_image:
raise Exception('--src-image, --dst-image and --tarball are required '
'arguments.')
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# This library can support push-by-digest, but the likelihood of a user
# correctly providing us with the digest without using this library
# directly is essentially nil.
src = docker_name.Tag(args.src_image)
dst = docker_name.Tag(args.dst_image)
# Resolve the appropriate credential to use based on the standard Docker
# client logic.
creds = docker_creds.DefaultKeychain.Resolve(src)
logging.info('Pulling v2.2 image from %r ...', src)
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
with open(args.tarball, 'rb') as f:
new_img = append.Layer(src_image, f.read())
creds = docker_creds.DefaultKeychain.Resolve(dst)
with docker_session.Push(dst,
creds,
transport,
threads=_THREADS,
mount=[src.as_repository()]) as session:
logging.info('Starting upload ...')
session.upload(new_img)
digest = new_img.digest()
print('{name} was published with digest: {digest}'.format(
name=dst, digest=digest))
def main():
logging_setup.DefineCommandLineArgs(parser)
args = parser.parse_args()
logging_setup.Init(args=args)
if not args.name:
logging.fatal('--name is a required arguments.')
sys.exit(1)
# This library can support push-by-digest, but the likelihood of a user
# correctly providing us with the digest without using this library
# directly is essentially nil.
name = Tag(args.name, args.stamp_info_file)
if not args.config and (args.layer or args.digest):
logging.fatal(
'Using --layer or --digest requires --config to be specified.')
sys.exit(1)
if not args.config and not args.tarball:
logging.fatal('Either --config or --tarball must be specified.')
sys.exit(1)
# If config is specified, use that. Otherwise, fallback on reading
# the config from the tarball.
config = args.config
if args.config:
logging.info('Reading config from %r', args.config)
with open(args.config, 'r') as reader:
config = reader.read()
elif args.tarball:
logging.info('Reading config from tarball %r', args.tarball)
with v2_2_image.FromTarball(args.tarball) as base:
config = base.config_file()
if len(args.digest or []) != len(args.layer or []):
logging.fatal('--digest and --layer must have matching lengths.')
sys.exit(1)
retry_factory = retry.Factory()
retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http)
transport = transport_pool.Http(retry_factory.Build, size=_THREADS)
logging.info('Loading v2.2 image from disk ...')
with v2_2_image.FromDisk(config,
zip(args.digest or [], args.layer or []),
legacy_base=args.tarball) as v2_2_img:
# Resolve the appropriate credential to use based on the standard Docker
# client logic.
try:
creds = docker_creds.DefaultKeychain.Resolve(name)
# pylint: disable=broad-except
except Exception as e:
logging.fatal('Error resolving credentials for %s: %s', name, e)
sys.exit(1)
try:
with docker_session.Push(name, creds, transport,
threads=_THREADS) as session:
logging.info('Starting upload ...')
if args.oci:
with oci_compat.OCIFromV22(v2_2_img) as oci_img:
session.upload(oci_img)
digest = oci_img.digest()
else:
session.upload(v2_2_img)
digest = v2_2_img.digest()
print('{name} was published with digest: {digest}'.format(
name=name, digest=digest))
# pylint: disable=broad-except
except Exception as e:
logging.fatal('Error publishing %s: %s', name, e)
sys.exit(1)
while (chunk := fileobj.read(4096)):
sha256_hash.update(chunk)
sha256_digest = sha256_hash.hexdigest()
fileobj.seek(0)
logger.debug(f'{sha256_digest=}')
image_ref = image_name
image_name = _ou.docker_name.from_string(image_name)
contents = fileobj.read()
from containerregistry.client.v2_2 import docker_session
push_sess = docker_session.Push(
name=image_name,
creds=_ou._mk_credentials(
image_reference=image_ref,
privileges=oa.Privileges.READWRITE,
credentials_lookup=credentials_lookup,
),
transport=_ou._mk_transport_pool(),
)
logger.debug(f'{len(contents)=}')
# XXX superdirty hack - force usage of our blob :(
push_sess._get_blob = lambda a, b: contents
push_sess._patch_upload(
image_name,
f'sha256:{sha256_digest}',
)
logger.debug(f'successfully pushed {image_name=} {sha256_digest=}')
return sha256_digest
def Publish(transport,
image_chroot,
stamp_info={},
name=None,
tarball=None,
config=None,
digest=None,
layer=None):
if not name:
raise Exception('Expected "name" kwarg')
if not config and (layer or digest):
raise Exception(
name +
': Using "layer" or "digest" requires "config" to be specified.')
if config:
with open(config, 'r') as reader:
config = reader.read()
elif tarball:
with v2_2_image.FromTarball(tarball) as base:
config = base.config_file()
else:
raise Exception(name +
': Either "config" or "tarball" must be specified.')
if digest or layer:
digest = digest.split(',')
layer = layer.split(',')
if len(digest) != len(layer):
raise Exception(
name + ': "digest" and "layer" must have matching lengths.')
else:
digest = []
layer = []
resolved_name = name.format(**stamp_info)
name_to_replace = name
if image_chroot:
name_to_publish = docker_name.Tag(os.path.join(image_chroot,
resolved_name),
strict=False)
else:
# Without a chroot, the left-hand-side must be a valid tag.
name_to_publish = docker_name.Tag(resolved_name, strict=False)
# Resolve the appropriate credential to use based on the standard Docker
# client logic.
creds = docker_creds.DefaultKeychain.Resolve(name_to_publish)
with v2_2_session.Push(name_to_publish, creds, transport,
threads=_THREADS) as session:
with v2_2_image.FromDisk(config,
zip(digest or [], layer or []),
legacy_base=tarball) as v2_2_img:
session.upload(v2_2_img)
return (name_to_replace, str(name_to_publish),
docker_name.Digest('{repository}@{digest}'.format(
repository=name_to_publish.as_repository(),
digest=v2_2_img.digest())))
