def _parse_composer_pkgs(self):
descriptor_contents = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
composer_json = json.loads(descriptor_contents)
pkgs = []
for k, v in composer_json['require'].iteritems():
pkgs.append((k, v))
return pkgs
def _parse_composer_lock_pkgs(self):
descriptor_contents = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
composer_lock_json = json.loads(descriptor_contents)
pkgs = []
for pkg in composer_lock_json['packages']:
pkgs.append((pkg['name'], pkg['version']))
return pkgs
def GetCacheKeyRaw(self):
if self._pkg_descriptor is not None:
# phase 2 cache key
return "%s %s %s" % (self._pkg_descriptor[0],
self._pkg_descriptor[1],
self._destination_path)
# phase 1 cache key
return "%s %s" % (ftl_util.descriptor_parser(
self._descriptor_files, self._ctx), self._destination_path)
def _parse_pipfile_pkgs(self):
pkg_descriptor = ftl_util.descriptor_parser(self._descriptor_files,
self._ctx)
pipfile_json = json.loads(pkg_descriptor)
pkgs = []
for pkg, info in pipfile_json['default'].iteritems():
version = info['version']
pkgs.append((pkg, version))
return pkgs
def BuildLayer(self):
cached_img = None
if self._cache:
with ftl_util.Timing('checking_cached_requirements.txt_layer'):
key = self.GetCacheKey()
cached_img = self._cache.Get(key)
self._log_cache_result(False if cached_img is None else True)
if cached_img:
self.SetImage(cached_img)
else:
python_util.setup_venv(self._venv_dir, self._venv_cmd,
self._python_cmd)
pkg_descriptor = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
self._pip_download_wheels(pkg_descriptor)
whls = self._resolve_whls()
pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls]
req_txt_imgs = []
with ftl_util.Timing('uploading_all_package_layers'):
with concurrent.futures.ThreadPoolExecutor(
max_workers=constants.THREADS) as executor:
future_to_params = {
executor.submit(self._build_pkg, whl_pkg_dir,
req_txt_imgs): whl_pkg_dir
for whl_pkg_dir in pkg_dirs
}
for future in concurrent.futures.as_completed(
future_to_params):
future.result()
req_txt_image = ftl_util.AppendLayersIntoImage(req_txt_imgs)
if req_txt_image:
self.SetImage(req_txt_image)
if self._cache:
with ftl_util.Timing('uploading_requirements.txt_pkg_lyr'):
self._cache.Set(self.GetCacheKey(), self.GetImage())
def BuildLayer(self):
cached_img = None
if self._cache:
with ftl_util.Timing('checking_cached_requirements.txt_layer'):
key = self.GetCacheKey()
cached_img = self._cache.Get(key)
self._log_cache_result(False if cached_img is None else True)
if cached_img:
self.SetImage(cached_img)
else:
self._setup_venv()
pkg_descriptor = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
self._pip_download_wheels(pkg_descriptor)
whls = self._resolve_whls()
pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls]
req_txt_imgs = []
for whl_pkg_dir in pkg_dirs:
layer_builder = PackageLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
pkg_dir=whl_pkg_dir,
dep_img_lyr=self._dep_img_lyr,
cache=self._cache)
layer_builder.BuildLayer()
req_txt_imgs.append(layer_builder.GetImage())
req_txt_image = ftl_util.AppendLayersIntoImage(req_txt_imgs)
self.SetImage(req_txt_image)
if self._cache:
with ftl_util.Timing('uploading_requirements.txt_pkg_lyr'):
self._cache.Set(self.GetCacheKey(), self.GetImage())
def GetCacheKeyRaw(self):
cache_key = "%s %s" % (ftl_util.descriptor_parser(
self._descriptor_files, self._ctx), self._destination_path)
return "%s %s" % (cache_key, self._cache_key_version)
def GetCacheKeyRaw(self):
return "%s %s" % (ftl_util.descriptor_parser(
self._descriptor_files, self._ctx), self._destination_path)
def GetCacheKeyRaw(self):
descriptor_contents = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
return "%s %s" % (descriptor_contents, self._destination_path)
def GetCacheKeyRaw(self):
descriptor_contents = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
return "%s %s" % (descriptor_contents,
self._dep_img_lyr.GetCacheKeyRaw())
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
interpreter_builder = package_builder.InterpreterLayerBuilder(
self._venv_dir, self._args.python_version)
cached_int_img = None
if self._args.cache:
with ftl_util.Timing("checking cached int layer"):
key = interpreter_builder.GetCacheKey()
cached_int_img = self._cache.Get(key)
if cached_int_img is not None:
interpreter_builder.SetImage(cached_int_img)
else:
with ftl_util.Timing("building int layer"):
interpreter_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading int layer"):
self._cache.Set(interpreter_builder.GetCacheKey(),
interpreter_builder.GetImage())
lyr_imgs.append(interpreter_builder)
pkg_descriptor = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
with ftl_util.Timing("installing pip packages"):
self._pip_install(pkg_descriptor)
with ftl_util.Timing("resolving whl paths"):
whls = self._resolve_whls()
pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls]
for whl_pkg_dir in pkg_dirs:
layer_builder = package_builder.PackageLayerBuilder(
self._ctx, self._descriptor_files, whl_pkg_dir,
interpreter_builder)
cached_pkg_img = None
if self._args.cache:
with ftl_util.Timing("checking cached pkg layer"):
key = layer_builder.GetCacheKey()
cached_pkg_img = self._cache.Get(key)
if cached_pkg_img is not None:
layer_builder.SetImage(cached_pkg_img)
else:
with ftl_util.Timing("building pkg layer"):
layer_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading pkg layer"):
self._cache.Set(layer_builder.GetCacheKey(),
layer_builder.GetImage())
lyr_imgs.append(layer_builder)
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
with ftl_util.Timing("builder app layer"):
app.BuildLayer()
lyr_imgs.append(app)
with ftl_util.Timing("stitching lyrs into final image"):
ftl_image = self.AppendLayersIntoImage(lyr_imgs)
with ftl_util.Timing("uploading final image"):
self.StoreImage(ftl_image)
def GetCacheKeyRaw(self):
descriptor_contents = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
cache_key = '%s %s' % (descriptor_contents,
self._dep_img_lyr.GetCacheKeyRaw())
return "%s %s" % (cache_key, self._cache_key_version)
