def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): layer_builder = node_builder.LayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path) cached_pkg_img = None if self._args.cache: with ftl_util.Timing("checking cached pkg layer"): key = layer_builder.GetCacheKey() cached_pkg_img = self._cache.Get(key) if cached_pkg_img is not None: layer_builder.SetImage(cached_pkg_img) else: with ftl_util.Timing("building pkg layer"): layer_builder.BuildLayer() if self._args.cache: with ftl_util.Timing("uploading pkg layer"): self._cache.Set(layer_builder.GetCacheKey(), layer_builder.GetImage()) lyr_imgs.append(layer_builder) app = base_builder.AppLayerBuilder(self._ctx, self._args.destination_path, self._args.entrypoint, self._args.exposed_ports) with ftl_util.Timing("builder app layer"): app.BuildLayer() lyr_imgs.append(app) with ftl_util.Timing("stitching lyrs into final image"): ftl_image = self.AppendLayersIntoImage(lyr_imgs) with ftl_util.Timing("uploading final image"): self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) interpreter_builder = package_builder.InterpreterLayerBuilder( venv_dir=self._venv_dir, python_cmd=self._python_cmd, venv_cmd=self._venv_cmd, cache=self._cache) interpreter_builder.BuildLayer() lyr_imgs.append(interpreter_builder.GetImage()) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): # build interpreter layer if self._is_phase2: # do a phase 2 build of the package layers w/ Pipfile.lock # iterate over package/version Pipfile.lock python_util.setup_venv(self._venv_dir, self._venv_cmd, self._python_cmd) pkgs = self._parse_pipfile_pkgs() with ftl_util.Timing('uploading_all_package_layers'): with concurrent.futures.ThreadPoolExecutor( max_workers=constants.THREADS) as executor: future_to_params = {executor.submit( self._build_pkg, pkg, interpreter_builder, lyr_imgs): pkg for pkg in pkgs } for future in concurrent.futures.as_completed( future_to_params): future.result() else: # do a phase 1 build of the package layers w/ requirements.txt req_txt_builder = package_builder.RequirementsLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, directory=self._args.directory, pkg_dir=None, wheel_dir=self._wheel_dir, venv_dir=self._venv_dir, python_cmd=self._python_cmd, pip_cmd=self._pip_cmd, venv_cmd=self._venv_cmd, dep_img_lyr=interpreter_builder, cache=self._cache) req_txt_builder.BuildLayer() if req_txt_builder.GetImage(): lyr_imgs.append(req_txt_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): # build interpreter layer interpreter_builder = package_builder.InterpreterLayerBuilder( venv_dir=self._venv_dir, python_cmd=self._python_cmd, venv_cmd=self._venv_cmd, cache=self._cache) interpreter_builder.BuildLayer() lyr_imgs.append(interpreter_builder.GetImage()) if self._is_phase2: # do a phase 2 build of the package layers w/ Pipfile.lock # iterate over package/version Pipfile.lock pkgs = self._parse_pipfile_pkgs() for pkg in pkgs: pipfile_builder = package_builder.PipfileLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_descriptor=pkg, pkg_dir=None, wheel_dir=ftl_util.gen_tmp_dir(constants.WHEEL_DIR), venv_dir=self._venv_dir, pip_cmd=self._pip_cmd, venv_cmd=self._venv_cmd, dep_img_lyr=interpreter_builder, cache=self._cache) pipfile_builder.BuildLayer() lyr_imgs.append(pipfile_builder.GetImage()) else: # do a phase 1 build of the package layers w/ requirements.txt req_txt_builder = package_builder.RequirementsLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_dir=None, wheel_dir=self._wheel_dir, venv_dir=self._venv_dir, python_cmd=self._python_cmd, pip_cmd=self._pip_cmd, venv_cmd=self._venv_cmd, dep_img_lyr=interpreter_builder, cache=self._cache) req_txt_builder.BuildLayer() lyr_imgs.append(req_txt_builder.GetImage()) app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def _gen_package_lock_if_required(self, ctx): if not ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): return if ctx.Contains(constants.PACKAGE_JSON) and \ not ctx.Contains(constants.YARN_LOCK) and \ not ctx.Contains(constants.PACKAGE_LOCK): logging.info('Found neither yarn.lock or package-lock.json,' 'generating package-lock.json from package.json') gen_package_lock_cmd = ['npm', 'install', '--package-lock-only'] ftl_util.run_command('gen_package_lock', gen_package_lock_cmd, cmd_cwd=self._args.directory, err_type=ftl_error.FTLErrors.USER())
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): pkgs = self._parse_composer_pkgs() # if there are 42 or more packages, revert to using phase 1 if len(pkgs) > 41: pkgs = [None] for pkg_txt in pkgs: logging.info('building package layer') logging.info(pkg_txt) layer_builder = php_builder.LayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_descriptor=pkg_txt, destination_path=self._args.destination_path) cached_pkg_img = None if self._args.cache: with ftl_util.Timing("checking cached pkg layer"): key = layer_builder.GetCacheKey() cached_pkg_img = self._cache.Get(key) if cached_pkg_img is not None: layer_builder.SetImage(cached_pkg_img) else: with ftl_util.Timing("building pkg layer"): layer_builder.BuildLayer() if self._args.cache: with ftl_util.Timing("uploading pkg layer"): self._cache.Set(layer_builder.GetCacheKey(), layer_builder.GetImage()) lyr_imgs.append(layer_builder) app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) with ftl_util.Timing("builder app layer"): app.BuildLayer() lyr_imgs.append(app) with ftl_util.Timing("stitching lyrs into final image"): ftl_image = self.AppendLayersIntoImage(lyr_imgs) with ftl_util.Timing("uploading final image"): self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): if self._ctx.Contains(constants.COMPOSER_LOCK): pkgs = self._parse_composer_lock_pkgs() else: pkgs = self._parse_composer_pkgs() # due to image layers limits, we revert to using phase 1 if over # the threshold if len(pkgs) > 41: # phase 1 logging.info('Building package layer') layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) else: # phase 2 for pkg_txt in pkgs: logging.info('Building package layer: {0} {1}'.format( pkg_txt[0], pkg_txt[1])) layer_builder = php_builder.PhaseTwoLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_descriptor=pkg_txt, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): if self._ctx.Contains(constants.COMPOSER_LOCK): pkgs = self._parse_composer_lock_pkgs() else: pkgs = self._parse_composer_pkgs() # due to image layers limits, we revert to using phase 1 if over # the threshold if len(pkgs) > 41: # phase 1 logging.info('Building package layer') layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) else: # phase 2 with ftl_util.Timing('uploading_all_package_layers'): with concurrent.futures.ThreadPoolExecutor( max_workers=constants.THREADS) as executor: future_to_params = {executor.submit( self._build_pkg, pkg_txt, lyr_imgs): pkg_txt for pkg_txt in pkgs } for future in concurrent.futures.as_completed( future_to_params): future.result() app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): layer_builder = node_builder.LayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder(self._ctx, self._args.destination_path, self._args.entrypoint, self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) # delete any existing files in node_modules folder if self._args.directory: modules_dir = os.path.join(self._args.directory, "node_modules") rm_cmd = ['rm', '-rf', modules_dir] ftl_util.run_command('rm_node_modules', rm_cmd) os.makedirs(os.path.join(modules_dir)) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): layer_builder = node_builder.LayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, directory=self._args.directory, destination_path=self._args.destination_path, should_use_yarn=self._should_use_yarn, cache_key_version=self._args.cache_key_version, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) if self._args.additional_directory: additional_directory = base_builder.AppLayerBuilder( directory=self._args.additional_directory, destination_path=self._args.additional_directory, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) additional_directory.BuildLayer() lyr_imgs.append(additional_directory.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) # delete any existing files in vendor folder if self._args.directory: vendor_dir = os.path.join(self._args.directory, 'vendor') rm_cmd = ['rm', '-rf', vendor_dir] ftl_util.run_command('rm_vendor_dir', rm_cmd) os.makedirs(os.path.join(vendor_dir)) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): self._gen_composer_lock() layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, directory=self._args.directory, destination_path=self._args.destination_path, cache_key_version=self._args.cache_key_version, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) if self._args.additional_directory: additional_directory = base_builder.AppLayerBuilder( directory=self._args.additional_directory, destination_path=self._args.additional_directory, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) additional_directory.BuildLayer() lyr_imgs.append(additional_directory.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): logging.info('Building package layer') layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, directory=self._args.directory, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): interpreter_builder = package_builder.InterpreterLayerBuilder( self._venv_dir, self._args.python_version) cached_int_img = None if self._args.cache: with ftl_util.Timing("checking cached int layer"): key = interpreter_builder.GetCacheKey() cached_int_img = self._cache.Get(key) if cached_int_img is not None: interpreter_builder.SetImage(cached_int_img) else: with ftl_util.Timing("building int layer"): interpreter_builder.BuildLayer() if self._args.cache: with ftl_util.Timing("uploading int layer"): self._cache.Set(interpreter_builder.GetCacheKey(), interpreter_builder.GetImage()) lyr_imgs.append(interpreter_builder) pkg_descriptor = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) with ftl_util.Timing("installing pip packages"): self._pip_install(pkg_descriptor) with ftl_util.Timing("resolving whl paths"): whls = self._resolve_whls() pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls] for whl_pkg_dir in pkg_dirs: layer_builder = package_builder.PackageLayerBuilder( self._ctx, self._descriptor_files, whl_pkg_dir, interpreter_builder) cached_pkg_img = None if self._args.cache: with ftl_util.Timing("checking cached pkg layer"): key = layer_builder.GetCacheKey() cached_pkg_img = self._cache.Get(key) if cached_pkg_img is not None: layer_builder.SetImage(cached_pkg_img) else: with ftl_util.Timing("building pkg layer"): layer_builder.BuildLayer() if self._args.cache: with ftl_util.Timing("uploading pkg layer"): self._cache.Set(layer_builder.GetCacheKey(), layer_builder.GetImage()) lyr_imgs.append(layer_builder) app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) with ftl_util.Timing("builder app layer"): app.BuildLayer() lyr_imgs.append(app) with ftl_util.Timing("stitching lyrs into final image"): ftl_image = self.AppendLayersIntoImage(lyr_imgs) with ftl_util.Timing("uploading final image"): self.StoreImage(ftl_image)