def append(self):
if notebook_helper.is_in_notebook():
notebook_helper.export_notebook_to_tar_gz(
self.notebook_file,
TEMP_TAR_GZ_FILENAME,
converted_filename=self.get_python_entrypoint())
else:
utils.generate_context_tarball(".", TEMP_TAR_GZ_FILENAME)
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
src = docker_name.Tag(self.base_image)
creds = docker_creds.DefaultKeychain.Resolve(src)
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
with open(TEMP_TAR_GZ_FILENAME, 'rb') as f:
new_img = append.Layer(src_image, f.read())
if self.image_tag is None:
self.image_tag = new_img.digest().split(":")[1]
dst = docker_name.Tag(self.full_image_name())
creds = docker_creds.DefaultKeychain.Resolve(dst)
with docker_session.Push(dst,
creds,
transport,
threads=_THREADS,
mount=[src.as_repository()]) as session:
logger.warn("Uploading {}".format(self.full_image_name()))
session.upload(new_img)
os.remove(TEMP_TAR_GZ_FILENAME)
logger.warn("Pushed image {}".format(self.full_image_name()))
def main():
args = parser.parse_args()
if not args.src_image or not args.tarball or not args.dst_image:
raise Exception('--src-image, --dst-image and --tarball are required '
'arguments.')
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# This library can support push-by-digest, but the likelihood of a user
# correctly providing us with the digest without using this library
# directly is essentially nil.
src = docker_name.Tag(args.src_image)
dst = docker_name.Tag(args.dst_image)
# Resolve the appropriate credential to use based on the standard Docker
# client logic.
creds = docker_creds.DefaultKeychain.Resolve(src)
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
with open(args.tarball, 'rb') as f:
new_img = append.Layer(src_image, f.read())
creds = docker_creds.DefaultKeychain.Resolve(dst)
with docker_session.Push(dst, creds, transport,
threads=_THREADS) as session:
session.upload(new_img)
def main():
logging_setup.DefineCommandLineArgs(parser)
args = parser.parse_args()
logging_setup.Init(args=args)
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# This library can support push-by-digest, but the likelihood of a user
# correctly providing us with the digest without using this library
# directly is essentially nil.
src = docker_name.Tag(args.src_image)
dst = docker_name.Tag(args.dst_image)
# Resolve the appropriate credential to use based on the standard Docker
# client logic.
creds = docker_creds.DefaultKeychain.Resolve(src)
logging.info('Pulling v2.2 image from %r ...', src)
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
with open(args.tarball, 'rb') as f:
new_img = append.Layer(src_image, f.read())
creds = docker_creds.DefaultKeychain.Resolve(dst)
with docker_session.Push(dst,
creds,
transport,
threads=_THREADS,
mount=[src.as_repository()]) as session:
logging.info('Starting upload ...')
session.upload(new_img)
digest = new_img.digest()
print(('{name} was published with digest: {digest}'.format(
name=dst, digest=digest)))
def CreatePackageBase(self, base_image, cache, use_cache=True):
"""Override."""
# Figure out if we need to override entrypoint.
# Save the overrides for later to avoid adding an extra layer.
pj_contents = {}
if self._ctx.Contains(_PACKAGE_JSON):
pj_contents = json.loads(self._ctx.GetFile(_PACKAGE_JSON))
entrypoint = parse_entrypoint(pj_contents)
overrides = metadata.Overrides(entrypoint=entrypoint)
descriptor = None
for f in [_PACKAGE_LOCK, _PACKAGE_JSON]:
if self._ctx.Contains(f):
descriptor = f
descriptor_contents = self._ctx.GetFile(f)
break
if not descriptor:
logging.info('No package descriptor found. No packages installed.')
# Add the overrides now.
return append.Layer(base_image, tar_gz=None, overrides=overrides)
checksum = hashlib.sha256(descriptor_contents).hexdigest()
if use_cache:
hit = cache.Get(base_image, _NODE_NAMESPACE, checksum)
if hit:
logging.info('Found cached dependency layer for %s' % checksum)
return hit
else:
logging.info('No cached dependency layer for %s' % checksum)
else:
logging.info('Skipping checking cache for dependency layer %s'
% checksum)
layer, sha = self._gen_package_tar(descriptor, descriptor_contents)
with append.Layer(
base_image, layer, diff_id=sha, overrides=overrides) as dep_image:
if use_cache:
logging.info('Storing layer %s in cache.', sha)
cache.Store(base_image, _NODE_NAMESPACE, checksum, dep_image)
else:
logging.info('Skipping storing layer %s in cache.', sha)
return dep_image
def _build(self, transport, src):
file, hash = self.preprocessor.context_tar_gz()
self.context_file, self.context_hash = file, hash
self.image_tag = self.full_image_name(self.context_hash)
creds = docker_creds.DefaultKeychain.Resolve(src)
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
with open(self.context_file, 'rb') as f:
new_img = append.Layer(src_image, f.read())
return new_img
def CreatePackageBase(self, base_image, cache):
"""Override."""
descriptor = self._ctx.GetFile('requirements.txt')
checksum = hashlib.sha256(descriptor).hexdigest()
hit = cache.Get(base_image, _PYTHON_NAMESPACE, checksum)
if hit:
return hit
buf = cStringIO.StringIO()
with tarfile.open(fileobj=buf, mode='w:gz') as out:
for whl in self._ResolveWHLs(descriptor):
self._AddWHLFiles(whl, out)
self._AddWHLScripts(whl, out)
layer = buf.getvalue()
with append.Layer(base_image, layer) as dep_image:
cache.Store(base_image, _PYTHON_NAMESPACE, checksum, dep_image)
return append.Layer(base_image, layer)
def _build(self, transport, src):
file, hash = self.preprocessor.context_tar_gz() # pylint:disable=redefined-builtin
self.context_file, self.context_hash = file, hash
self.image_tag = self.full_image_name(self.context_hash)
creds = docker_creds.DefaultKeychain.Resolve(src)
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
with open(self.context_file, 'rb') as f:
new_img = append.Layer(src_image, f.read(), overrides=metadata.Overrides(
cmd=self.preprocessor.get_command(),
user='******', env={"FAIRING_RUNTIME": "1"}))
return new_img
def handle_app(pm, cache_dir):
packages = pm.get_package_list()
img = pm.base_image()
# This loop could be parallelized.
for pkg in packages:
contents = get_layer_cache(pkg, cache_dir)
if not contents:
contents = pm.build_layer_for_one_package(pkg)
set_layer_cache(pkg, contents, cache_dir)
img = append.Layer(img, contents)
return img
def AppendLayersIntoImage(imgs):
with Timing('Stitching layers into final image'):
for i, img in enumerate(imgs):
if i == 0:
result_image = img
continue
diff_ids = img.diff_ids()
for diff_id in diff_ids:
lyr = img.blob(img._diff_id_to_digest(diff_id))
overrides = CfgDctToOverrides(json.loads(img.config_file()))
result_image = append.Layer(result_image,
lyr,
diff_id=diff_id,
overrides=overrides)
return result_image
def AppendLayersIntoImage(self, lyr_imgs):
for i, lyr_img in enumerate(lyr_imgs):
if i == 0:
result_image = lyr_img
continue
img = lyr_img.GetImage()
diff_ids = img.diff_ids()
for diff_id in diff_ids:
lyr = img.blob(img._diff_id_to_digest(diff_id))
overrides = ftl_util.CfgDctToOverrides(
json.loads(img.config_file()))
result_image = append.Layer(result_image,
lyr,
diff_id=diff_id,
overrides=overrides)
return result_image
def main(args):
args = parser.parse_args(args)
logging.getLogger().setLevel(_LEVEL_MAP[args.verbosity])
logging.basicConfig(
format='%(asctime)s.%(msecs)03d %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d,%H:%M:%S')
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# TODO(mattmoor): Support digest base images.
base_name = docker_name.Tag(args.base)
base_creds = docker_creds.DefaultKeychain.Resolve(base_name)
target_image = docker_name.Tag(args.name)
target_creds = docker_creds.DefaultKeychain.Resolve(target_image)
ctx = context.Workspace(args.directory)
cash = cache.Registry(target_image.as_repository(),
target_creds,
transport,
threads=_THREADS,
mount=[base_name])
bldr = builder.From(ctx)
with docker_image.FromRegistry(base_name, base_creds,
transport) as base_image:
# Create (or pull from cache) the base image with the
# package descriptor installation overlaid.
logging.info('Generating dependency layer...')
with bldr.CreatePackageBase(base_image, cash, args.cache) as deps:
# Construct the application layer from the context.
logging.info('Generating app layer...')
app_layer, diff_id = bldr.BuildAppLayer()
with append.Layer(deps, app_layer, diff_id=diff_id) as app_image:
if args.output_path:
with tarfile.open(name=args.output_path, mode='w') as tar:
save.tarball(target_image, app_image, tar)
logging.info("{0} tarball located at {1}".format(
str(target_image), args.output_path))
return
with docker_session.Push(target_image,
target_creds,
transport,
threads=_THREADS,
mount=[base_name]) as session:
logging.info('Pushing final image...')
session.upload(app_image)
def main():
args = parser.parse_args()
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# TODO(mattmoor): Support digest base images.
base_name = docker_name.Tag(args.base)
base_creds = docker_creds.DefaultKeychain.Resolve(base_name)
target_image = docker_name.Tag(args.name)
target_creds = docker_creds.DefaultKeychain.Resolve(target_image)
with context.Workspace(args.directory) as ctx:
with cache.Registry(target_image.as_repository(),
target_creds,
transport,
threads=_THREADS,
mount=[base_name]) as cash:
with builder.From(ctx) as bldr:
with docker_image.FromRegistry(base_name, base_creds,
transport) as base_image:
# Create (or pull from cache) the base image with the package
# descriptor installation overlaid.
with bldr.CreatePackageBase(base_image,
cash) as base_with_deps:
# Construct the application layer from the context.
app_layer = bldr.BuildAppLayer()
with append.Layer(base_with_deps,
app_layer) as app_image:
with docker_session.Push(target_image,
target_creds,
transport,
threads=_THREADS,
mount=[base_name
]) as session:
session.upload(app_image)
print("Hi, One Build.")
def main(args):
args = parser.parse_args(args)
logging.getLogger().setLevel(_LEVEL_MAP[args.verbosity])
transport = transport_pool.Http(httplib2.Http, size=_THREADS)
# TODO(mattmoor): Support digest base images.
base_name = docker_name.Tag(args.base)
base_creds = docker_creds.DefaultKeychain.Resolve(base_name)
target_image = docker_name.Tag(args.name)
target_creds = docker_creds.DefaultKeychain.Resolve(target_image)
ctx = context.Workspace(args.directory)
cash = cache.Registry(target_image.as_repository(),
target_creds,
transport,
threads=_THREADS,
mount=[base_name])
bldr = builder.From(ctx)
with docker_image.FromRegistry(base_name, base_creds,
transport) as base_image:
# Create (or pull from cache) the base image with the
# package descriptor installation overlaid.
logging.info('Generating dependency layer...')
with bldr.CreatePackageBase(base_image, cash) as deps:
# Construct the application layer from the context.
logging.info('Generating app layer...')
app_layer, diff_id = bldr.BuildAppLayer()
with append.Layer(deps, app_layer, diff_id=diff_id) as app_image:
with docker_session.Push(target_image,
target_creds,
transport,
threads=_THREADS,
mount=[base_name]) as session:
logging.info('Pushing final image...')
session.upload(app_image)
def CreatePackageBase(self, base_image, cache):
"""Override."""
# Figure out if we need to override entrypoint.
# Save the overrides for later to avoid adding an extra layer.
pj_contents = {}
if self._ctx.Contains(_PACKAGE_JSON):
pj_contents = json.loads(self._ctx.GetFile(_PACKAGE_JSON))
entrypoint = parse_entrypoint(pj_contents)
overrides = metadata.Overrides(entrypoint=entrypoint)
descriptor = None
for f in [_PACKAGE_LOCK, _PACKAGE_JSON]:
if self._ctx.Contains(f):
descriptor = f
descriptor_contents = self._ctx.GetFile(f)
break
if not descriptor:
logging.info('No package descriptor found. No packages installed.')
# Add the overrides now.
return append.Layer(base_image, tar_gz=None, overrides=overrides)
checksum = hashlib.sha256(descriptor_contents).hexdigest()
hit = cache.Get(base_image, _NODE_NAMESPACE, checksum)
if hit:
logging.info('Found cached dependency layer for %s' % checksum)
return hit
else:
logging.info('No cached dependency layer for %s' % checksum)
# We want the node_modules directory rooted at /app/node_modules in
# the final image.
# So we build a hierarchy like:
# /$tmp/app/node_modules
# And use the -C flag to tar to root the tarball at /$tmp.
tmp = tempfile.mkdtemp()
app_dir = os.path.join(tmp, 'app')
os.mkdir(app_dir)
# Copy out the relevant package descriptors to a tempdir.
with open(os.path.join(app_dir, descriptor), 'w') as f:
f.write(descriptor_contents)
tar_path = tempfile.mktemp()
check_gcp_build(json.loads(self._ctx.GetFile(_PACKAGE_JSON)), app_dir)
subprocess.check_call(
['rm', '-rf', os.path.join(app_dir, 'node_modules')])
subprocess.check_call(['npm', 'install', '--production', '--no-cache'],
cwd=app_dir)
subprocess.check_call(['tar', '-C', tmp, '-cf', tar_path, '.'])
# We need the sha of the unzipped and zipped tarball.
# So for performance, tar, sha, zip, sha.
# We use gzip for performance instead of python's zip.
sha = 'sha256:' + hashlib.sha256(open(tar_path).read()).hexdigest()
subprocess.check_call(['gzip', tar_path])
layer = open(os.path.join(tmp, tar_path + '.gz'), 'rb').read()
with append.Layer(base_image, layer, diff_id=sha,
overrides=overrides) as dep_image:
logging.info('Storing layer %s in cache.', sha)
cache.Store(base_image, _NODE_NAMESPACE, checksum, dep_image)
return dep_image
