def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) interpreter_builder = package_builder.InterpreterLayerBuilder( venv_dir=self._venv_dir, python_cmd=self._python_cmd, venv_cmd=self._venv_cmd, cache=self._cache) interpreter_builder.BuildLayer() lyr_imgs.append(interpreter_builder.GetImage()) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): # build interpreter layer if self._is_phase2: # do a phase 2 build of the package layers w/ Pipfile.lock # iterate over package/version Pipfile.lock python_util.setup_venv(self._venv_dir, self._venv_cmd, self._python_cmd) pkgs = self._parse_pipfile_pkgs() with ftl_util.Timing('uploading_all_package_layers'): with concurrent.futures.ThreadPoolExecutor( max_workers=constants.THREADS) as executor: future_to_params = {executor.submit( self._build_pkg, pkg, interpreter_builder, lyr_imgs): pkg for pkg in pkgs } for future in concurrent.futures.as_completed( future_to_params): future.result() else: # do a phase 1 build of the package layers w/ requirements.txt req_txt_builder = package_builder.RequirementsLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, directory=self._args.directory, pkg_dir=None, wheel_dir=self._wheel_dir, venv_dir=self._venv_dir, python_cmd=self._python_cmd, pip_cmd=self._pip_cmd, venv_cmd=self._venv_cmd, dep_img_lyr=interpreter_builder, cache=self._cache) req_txt_builder.BuildLayer() if req_txt_builder.GetImage(): lyr_imgs.append(req_txt_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): # build interpreter layer interpreter_builder = package_builder.InterpreterLayerBuilder( venv_dir=self._venv_dir, python_cmd=self._python_cmd, venv_cmd=self._venv_cmd, cache=self._cache) interpreter_builder.BuildLayer() lyr_imgs.append(interpreter_builder.GetImage()) if self._is_phase2: # do a phase 2 build of the package layers w/ Pipfile.lock # iterate over package/version Pipfile.lock pkgs = self._parse_pipfile_pkgs() for pkg in pkgs: pipfile_builder = package_builder.PipfileLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_descriptor=pkg, pkg_dir=None, wheel_dir=ftl_util.gen_tmp_dir(constants.WHEEL_DIR), venv_dir=self._venv_dir, pip_cmd=self._pip_cmd, venv_cmd=self._venv_cmd, dep_img_lyr=interpreter_builder, cache=self._cache) pipfile_builder.BuildLayer() lyr_imgs.append(pipfile_builder.GetImage()) else: # do a phase 1 build of the package layers w/ requirements.txt req_txt_builder = package_builder.RequirementsLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_dir=None, wheel_dir=self._wheel_dir, venv_dir=self._venv_dir, python_cmd=self._python_cmd, pip_cmd=self._pip_cmd, venv_cmd=self._venv_cmd, dep_img_lyr=interpreter_builder, cache=self._cache) req_txt_builder.BuildLayer() lyr_imgs.append(req_txt_builder.GetImage()) app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): if self._ctx.Contains(constants.COMPOSER_LOCK): pkgs = self._parse_composer_lock_pkgs() else: pkgs = self._parse_composer_pkgs() # due to image layers limits, we revert to using phase 1 if over # the threshold if len(pkgs) > 41: # phase 1 logging.info('Building package layer') layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) else: # phase 2 for pkg_txt in pkgs: logging.info('Building package layer: {0} {1}'.format( pkg_txt[0], pkg_txt[1])) layer_builder = php_builder.PhaseTwoLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_descriptor=pkg_txt, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): if self._ctx.Contains(constants.COMPOSER_LOCK): pkgs = self._parse_composer_lock_pkgs() else: pkgs = self._parse_composer_pkgs() # due to image layers limits, we revert to using phase 1 if over # the threshold if len(pkgs) > 41: # phase 1 logging.info('Building package layer') layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) else: # phase 2 with ftl_util.Timing('uploading_all_package_layers'): with concurrent.futures.ThreadPoolExecutor( max_workers=constants.THREADS) as executor: future_to_params = {executor.submit( self._build_pkg, pkg_txt, lyr_imgs): pkg_txt for pkg_txt in pkgs } for future in concurrent.futures.as_completed( future_to_params): future.result() app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): layer_builder = node_builder.LayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder(self._ctx, self._args.destination_path, self._args.entrypoint, self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def BuildLayer(self): cached_img = None if self._cache: with ftl_util.Timing('checking_cached_requirements.txt_layer'): key = self.GetCacheKey() cached_img = self._cache.Get(key) self._log_cache_result(False if cached_img is None else True) if cached_img: self.SetImage(cached_img) else: python_util.setup_venv(self._venv_dir, self._venv_cmd, self._python_cmd) pkg_descriptor = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) self._pip_download_wheels(pkg_descriptor) whls = self._resolve_whls() pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls] req_txt_imgs = [] with ftl_util.Timing('uploading_all_package_layers'): with concurrent.futures.ThreadPoolExecutor( max_workers=constants.THREADS) as executor: future_to_params = { executor.submit(self._build_pkg, whl_pkg_dir, req_txt_imgs): whl_pkg_dir for whl_pkg_dir in pkg_dirs } for future in concurrent.futures.as_completed( future_to_params): future.result() req_txt_image = ftl_util.AppendLayersIntoImage(req_txt_imgs) if req_txt_image: self.SetImage(req_txt_image) if self._cache: with ftl_util.Timing('uploading_requirements.txt_pkg_lyr'): self._cache.Set(self.GetCacheKey(), self.GetImage())
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) # delete any existing files in node_modules folder if self._args.directory: modules_dir = os.path.join(self._args.directory, "node_modules") rm_cmd = ['rm', '-rf', modules_dir] ftl_util.run_command('rm_node_modules', rm_cmd) os.makedirs(os.path.join(modules_dir)) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): layer_builder = node_builder.LayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, directory=self._args.directory, destination_path=self._args.destination_path, should_use_yarn=self._should_use_yarn, cache_key_version=self._args.cache_key_version, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) if self._args.additional_directory: additional_directory = base_builder.AppLayerBuilder( directory=self._args.additional_directory, destination_path=self._args.additional_directory, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) additional_directory.BuildLayer() lyr_imgs.append(additional_directory.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) # delete any existing files in vendor folder if self._args.directory: vendor_dir = os.path.join(self._args.directory, 'vendor') rm_cmd = ['rm', '-rf', vendor_dir] ftl_util.run_command('rm_vendor_dir', rm_cmd) os.makedirs(os.path.join(vendor_dir)) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): self._gen_composer_lock() layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, directory=self._args.directory, destination_path=self._args.destination_path, cache_key_version=self._args.cache_key_version, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) if self._args.additional_directory: additional_directory = base_builder.AppLayerBuilder( directory=self._args.additional_directory, destination_path=self._args.additional_directory, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) additional_directory.BuildLayer() lyr_imgs.append(additional_directory.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): logging.info('Building package layer') layer_builder = php_builder.PhaseOneLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, destination_path=self._args.destination_path, directory=self._args.directory, cache=self._cache) layer_builder.BuildLayer() lyr_imgs.append(layer_builder.GetImage()) app = base_builder.AppLayerBuilder( directory=self._args.directory, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) app.BuildLayer() lyr_imgs.append(app.GetImage()) ftl_image = ftl_util.AppendLayersIntoImage(lyr_imgs) self.StoreImage(ftl_image)
def BuildLayer(self): cached_img = None if self._cache: with ftl_util.Timing('checking_cached_requirements.txt_layer'): key = self.GetCacheKey() cached_img = self._cache.Get(key) self._log_cache_result(False if cached_img is None else True) if cached_img: self.SetImage(cached_img) else: self._setup_venv() pkg_descriptor = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) self._pip_download_wheels(pkg_descriptor) whls = self._resolve_whls() pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls] req_txt_imgs = [] for whl_pkg_dir in pkg_dirs: layer_builder = PackageLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_dir=whl_pkg_dir, dep_img_lyr=self._dep_img_lyr, cache=self._cache) layer_builder.BuildLayer() req_txt_imgs.append(layer_builder.GetImage()) req_txt_image = ftl_util.AppendLayersIntoImage(req_txt_imgs) self.SetImage(req_txt_image) if self._cache: with ftl_util.Timing('uploading_requirements.txt_pkg_lyr'): self._cache.Set(self.GetCacheKey(), self.GetImage())