def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
layer_builder = node_builder.LayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
destination_path=self._args.destination_path)
cached_pkg_img = None
if self._args.cache:
with ftl_util.Timing("checking cached pkg layer"):
key = layer_builder.GetCacheKey()
cached_pkg_img = self._cache.Get(key)
if cached_pkg_img is not None:
layer_builder.SetImage(cached_pkg_img)
else:
with ftl_util.Timing("building pkg layer"):
layer_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading pkg layer"):
self._cache.Set(layer_builder.GetCacheKey(),
layer_builder.GetImage())
lyr_imgs.append(layer_builder)
app = base_builder.AppLayerBuilder(self._ctx,
self._args.destination_path,
self._args.entrypoint,
self._args.exposed_ports)
with ftl_util.Timing("builder app layer"):
app.BuildLayer()
lyr_imgs.append(app)
with ftl_util.Timing("stitching lyrs into final image"):
ftl_image = self.AppendLayersIntoImage(lyr_imgs)
with ftl_util.Timing("uploading final image"):
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
interpreter_builder = package_builder.InterpreterLayerBuilder(
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
venv_cmd=self._venv_cmd,
cache=self._cache)
interpreter_builder.BuildLayer()
lyr_imgs.append(interpreter_builder.GetImage())
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
# build interpreter layer
if self._is_phase2:
# do a phase 2 build of the package layers w/ Pipfile.lock
# iterate over package/version Pipfile.lock
python_util.setup_venv(self._venv_dir,
self._venv_cmd,
self._python_cmd)
pkgs = self._parse_pipfile_pkgs()
with ftl_util.Timing('uploading_all_package_layers'):
with concurrent.futures.ThreadPoolExecutor(
max_workers=constants.THREADS) as executor:
future_to_params = {executor.submit(
self._build_pkg, pkg,
interpreter_builder, lyr_imgs): pkg
for pkg in pkgs
}
for future in concurrent.futures.as_completed(
future_to_params):
future.result()
else:
# do a phase 1 build of the package layers w/ requirements.txt
req_txt_builder = package_builder.RequirementsLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
directory=self._args.directory,
pkg_dir=None,
wheel_dir=self._wheel_dir,
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
pip_cmd=self._pip_cmd,
venv_cmd=self._venv_cmd,
dep_img_lyr=interpreter_builder,
cache=self._cache)
req_txt_builder.BuildLayer()
if req_txt_builder.GetImage():
lyr_imgs.append(req_txt_builder.GetImage())
app = base_builder.AppLayerBuilder(
directory=self._args.directory,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
# build interpreter layer
interpreter_builder = package_builder.InterpreterLayerBuilder(
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
venv_cmd=self._venv_cmd,
cache=self._cache)
interpreter_builder.BuildLayer()
lyr_imgs.append(interpreter_builder.GetImage())
if self._is_phase2:
# do a phase 2 build of the package layers w/ Pipfile.lock
# iterate over package/version Pipfile.lock
pkgs = self._parse_pipfile_pkgs()
for pkg in pkgs:
pipfile_builder = package_builder.PipfileLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
pkg_descriptor=pkg,
pkg_dir=None,
wheel_dir=ftl_util.gen_tmp_dir(constants.WHEEL_DIR),
venv_dir=self._venv_dir,
pip_cmd=self._pip_cmd,
venv_cmd=self._venv_cmd,
dep_img_lyr=interpreter_builder,
cache=self._cache)
pipfile_builder.BuildLayer()
lyr_imgs.append(pipfile_builder.GetImage())
else:
# do a phase 1 build of the package layers w/ requirements.txt
req_txt_builder = package_builder.RequirementsLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
pkg_dir=None,
wheel_dir=self._wheel_dir,
venv_dir=self._venv_dir,
python_cmd=self._python_cmd,
pip_cmd=self._pip_cmd,
venv_cmd=self._venv_cmd,
dep_img_lyr=interpreter_builder,
cache=self._cache)
req_txt_builder.BuildLayer()
lyr_imgs.append(req_txt_builder.GetImage())
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def _gen_package_lock_if_required(self, ctx):
if not ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
return
if ctx.Contains(constants.PACKAGE_JSON) and \
not ctx.Contains(constants.YARN_LOCK) and \
not ctx.Contains(constants.PACKAGE_LOCK):
logging.info('Found neither yarn.lock or package-lock.json,'
'generating package-lock.json from package.json')
gen_package_lock_cmd = ['npm', 'install', '--package-lock-only']
ftl_util.run_command('gen_package_lock',
gen_package_lock_cmd,
cmd_cwd=self._args.directory,
err_type=ftl_error.FTLErrors.USER())
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
pkgs = self._parse_composer_pkgs()
# if there are 42 or more packages, revert to using phase 1
if len(pkgs) > 41:
pkgs = [None]
for pkg_txt in pkgs:
logging.info('building package layer')
logging.info(pkg_txt)
layer_builder = php_builder.LayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
pkg_descriptor=pkg_txt,
destination_path=self._args.destination_path)
cached_pkg_img = None
if self._args.cache:
with ftl_util.Timing("checking cached pkg layer"):
key = layer_builder.GetCacheKey()
cached_pkg_img = self._cache.Get(key)
if cached_pkg_img is not None:
layer_builder.SetImage(cached_pkg_img)
else:
with ftl_util.Timing("building pkg layer"):
layer_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading pkg layer"):
self._cache.Set(layer_builder.GetCacheKey(),
layer_builder.GetImage())
lyr_imgs.append(layer_builder)
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
with ftl_util.Timing("builder app layer"):
app.BuildLayer()
lyr_imgs.append(app)
with ftl_util.Timing("stitching lyrs into final image"):
ftl_image = self.AppendLayersIntoImage(lyr_imgs)
with ftl_util.Timing("uploading final image"):
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
if self._ctx.Contains(constants.COMPOSER_LOCK):
pkgs = self._parse_composer_lock_pkgs()
else:
pkgs = self._parse_composer_pkgs()
# due to image layers limits, we revert to using phase 1 if over
# the threshold
if len(pkgs) > 41:
# phase 1
logging.info('Building package layer')
layer_builder = php_builder.PhaseOneLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
destination_path=self._args.destination_path,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
else:
# phase 2
for pkg_txt in pkgs:
logging.info('Building package layer: {0} {1}'.format(
pkg_txt[0], pkg_txt[1]))
layer_builder = php_builder.PhaseTwoLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
pkg_descriptor=pkg_txt,
destination_path=self._args.destination_path,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
if self._ctx.Contains(constants.COMPOSER_LOCK):
pkgs = self._parse_composer_lock_pkgs()
else:
pkgs = self._parse_composer_pkgs()
# due to image layers limits, we revert to using phase 1 if over
# the threshold
if len(pkgs) > 41:
# phase 1
logging.info('Building package layer')
layer_builder = php_builder.PhaseOneLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
destination_path=self._args.destination_path,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
else:
# phase 2
with ftl_util.Timing('uploading_all_package_layers'):
with concurrent.futures.ThreadPoolExecutor(
max_workers=constants.THREADS) as executor:
future_to_params = {executor.submit(
self._build_pkg, pkg_txt, lyr_imgs): pkg_txt
for pkg_txt in pkgs
}
for future in concurrent.futures.as_completed(
future_to_params):
future.result()
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
layer_builder = node_builder.LayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
destination_path=self._args.destination_path,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
app = base_builder.AppLayerBuilder(self._ctx,
self._args.destination_path,
self._args.entrypoint,
self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
# delete any existing files in node_modules folder
if self._args.directory:
modules_dir = os.path.join(self._args.directory, "node_modules")
rm_cmd = ['rm', '-rf', modules_dir]
ftl_util.run_command('rm_node_modules', rm_cmd)
os.makedirs(os.path.join(modules_dir))
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
layer_builder = node_builder.LayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
directory=self._args.directory,
destination_path=self._args.destination_path,
should_use_yarn=self._should_use_yarn,
cache_key_version=self._args.cache_key_version,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
app = base_builder.AppLayerBuilder(
directory=self._args.directory,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
if self._args.additional_directory:
additional_directory = base_builder.AppLayerBuilder(
directory=self._args.additional_directory,
destination_path=self._args.additional_directory,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
additional_directory.BuildLayer()
lyr_imgs.append(additional_directory.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
# delete any existing files in vendor folder
if self._args.directory:
vendor_dir = os.path.join(self._args.directory, 'vendor')
rm_cmd = ['rm', '-rf', vendor_dir]
ftl_util.run_command('rm_vendor_dir', rm_cmd)
os.makedirs(os.path.join(vendor_dir))
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
self._gen_composer_lock()
layer_builder = php_builder.PhaseOneLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
directory=self._args.directory,
destination_path=self._args.destination_path,
cache_key_version=self._args.cache_key_version,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
app = base_builder.AppLayerBuilder(
directory=self._args.directory,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
if self._args.additional_directory:
additional_directory = base_builder.AppLayerBuilder(
directory=self._args.additional_directory,
destination_path=self._args.additional_directory,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
additional_directory.BuildLayer()
lyr_imgs.append(additional_directory.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
logging.info('Building package layer')
layer_builder = php_builder.PhaseOneLayerBuilder(
ctx=self._ctx,
descriptor_files=self._descriptor_files,
destination_path=self._args.destination_path,
directory=self._args.directory,
cache=self._cache)
layer_builder.BuildLayer()
lyr_imgs.append(layer_builder.GetImage())
app = base_builder.AppLayerBuilder(
directory=self._args.directory,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
app.BuildLayer()
lyr_imgs.append(app.GetImage())
ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs)
self.StoreImage(ftl_image)
def Build(self):
lyr_imgs = []
lyr_imgs.append(self._base_image)
if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx):
interpreter_builder = package_builder.InterpreterLayerBuilder(
self._venv_dir, self._args.python_version)
cached_int_img = None
if self._args.cache:
with ftl_util.Timing("checking cached int layer"):
key = interpreter_builder.GetCacheKey()
cached_int_img = self._cache.Get(key)
if cached_int_img is not None:
interpreter_builder.SetImage(cached_int_img)
else:
with ftl_util.Timing("building int layer"):
interpreter_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading int layer"):
self._cache.Set(interpreter_builder.GetCacheKey(),
interpreter_builder.GetImage())
lyr_imgs.append(interpreter_builder)
pkg_descriptor = ftl_util.descriptor_parser(
self._descriptor_files, self._ctx)
with ftl_util.Timing("installing pip packages"):
self._pip_install(pkg_descriptor)
with ftl_util.Timing("resolving whl paths"):
whls = self._resolve_whls()
pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls]
for whl_pkg_dir in pkg_dirs:
layer_builder = package_builder.PackageLayerBuilder(
self._ctx, self._descriptor_files, whl_pkg_dir,
interpreter_builder)
cached_pkg_img = None
if self._args.cache:
with ftl_util.Timing("checking cached pkg layer"):
key = layer_builder.GetCacheKey()
cached_pkg_img = self._cache.Get(key)
if cached_pkg_img is not None:
layer_builder.SetImage(cached_pkg_img)
else:
with ftl_util.Timing("building pkg layer"):
layer_builder.BuildLayer()
if self._args.cache:
with ftl_util.Timing("uploading pkg layer"):
self._cache.Set(layer_builder.GetCacheKey(),
layer_builder.GetImage())
lyr_imgs.append(layer_builder)
app = base_builder.AppLayerBuilder(
ctx=self._ctx,
destination_path=self._args.destination_path,
entrypoint=self._args.entrypoint,
exposed_ports=self._args.exposed_ports)
with ftl_util.Timing("builder app layer"):
app.BuildLayer()
lyr_imgs.append(app)
with ftl_util.Timing("stitching lyrs into final image"):
ftl_image = self.AppendLayersIntoImage(lyr_imgs)
with ftl_util.Timing("uploading final image"):
self.StoreImage(ftl_image)
