def _parse_composer_pkgs(self): descriptor_contents = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) composer_json = json.loads(descriptor_contents) pkgs = [] for k, v in composer_json['require'].iteritems(): pkgs.append((k, v)) return pkgs
def _parse_composer_lock_pkgs(self): descriptor_contents = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) composer_lock_json = json.loads(descriptor_contents) pkgs = [] for pkg in composer_lock_json['packages']: pkgs.append((pkg['name'], pkg['version'])) return pkgs
def GetCacheKeyRaw(self): if self._pkg_descriptor is not None: # phase 2 cache key return "%s %s %s" % (self._pkg_descriptor[0], self._pkg_descriptor[1], self._destination_path) # phase 1 cache key return "%s %s" % (ftl_util.descriptor_parser( self._descriptor_files, self._ctx), self._destination_path)
def _parse_pipfile_pkgs(self): pkg_descriptor = ftl_util.descriptor_parser(self._descriptor_files, self._ctx) pipfile_json = json.loads(pkg_descriptor) pkgs = [] for pkg, info in pipfile_json['default'].iteritems(): version = info['version'] pkgs.append((pkg, version)) return pkgs
def BuildLayer(self): cached_img = None if self._cache: with ftl_util.Timing('checking_cached_requirements.txt_layer'): key = self.GetCacheKey() cached_img = self._cache.Get(key) self._log_cache_result(False if cached_img is None else True) if cached_img: self.SetImage(cached_img) else: python_util.setup_venv(self._venv_dir, self._venv_cmd, self._python_cmd) pkg_descriptor = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) self._pip_download_wheels(pkg_descriptor) whls = self._resolve_whls() pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls] req_txt_imgs = [] with ftl_util.Timing('uploading_all_package_layers'): with concurrent.futures.ThreadPoolExecutor( max_workers=constants.THREADS) as executor: future_to_params = { executor.submit(self._build_pkg, whl_pkg_dir, req_txt_imgs): whl_pkg_dir for whl_pkg_dir in pkg_dirs } for future in concurrent.futures.as_completed( future_to_params): future.result() req_txt_image = ftl_util.AppendLayersIntoImage(req_txt_imgs) if req_txt_image: self.SetImage(req_txt_image) if self._cache: with ftl_util.Timing('uploading_requirements.txt_pkg_lyr'): self._cache.Set(self.GetCacheKey(), self.GetImage())
def BuildLayer(self): cached_img = None if self._cache: with ftl_util.Timing('checking_cached_requirements.txt_layer'): key = self.GetCacheKey() cached_img = self._cache.Get(key) self._log_cache_result(False if cached_img is None else True) if cached_img: self.SetImage(cached_img) else: self._setup_venv() pkg_descriptor = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) self._pip_download_wheels(pkg_descriptor) whls = self._resolve_whls() pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls] req_txt_imgs = [] for whl_pkg_dir in pkg_dirs: layer_builder = PackageLayerBuilder( ctx=self._ctx, descriptor_files=self._descriptor_files, pkg_dir=whl_pkg_dir, dep_img_lyr=self._dep_img_lyr, cache=self._cache) layer_builder.BuildLayer() req_txt_imgs.append(layer_builder.GetImage()) req_txt_image = ftl_util.AppendLayersIntoImage(req_txt_imgs) self.SetImage(req_txt_image) if self._cache: with ftl_util.Timing('uploading_requirements.txt_pkg_lyr'): self._cache.Set(self.GetCacheKey(), self.GetImage())
def GetCacheKeyRaw(self): cache_key = "%s %s" % (ftl_util.descriptor_parser( self._descriptor_files, self._ctx), self._destination_path) return "%s %s" % (cache_key, self._cache_key_version)
def GetCacheKeyRaw(self): return "%s %s" % (ftl_util.descriptor_parser( self._descriptor_files, self._ctx), self._destination_path)
def GetCacheKeyRaw(self): descriptor_contents = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) return "%s %s" % (descriptor_contents, self._destination_path)
def GetCacheKeyRaw(self): descriptor_contents = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) return "%s %s" % (descriptor_contents, self._dep_img_lyr.GetCacheKeyRaw())
def Build(self): lyr_imgs = [] lyr_imgs.append(self._base_image) if ftl_util.has_pkg_descriptor(self._descriptor_files, self._ctx): interpreter_builder = package_builder.InterpreterLayerBuilder( self._venv_dir, self._args.python_version) cached_int_img = None if self._args.cache: with ftl_util.Timing("checking cached int layer"): key = interpreter_builder.GetCacheKey() cached_int_img = self._cache.Get(key) if cached_int_img is not None: interpreter_builder.SetImage(cached_int_img) else: with ftl_util.Timing("building int layer"): interpreter_builder.BuildLayer() if self._args.cache: with ftl_util.Timing("uploading int layer"): self._cache.Set(interpreter_builder.GetCacheKey(), interpreter_builder.GetImage()) lyr_imgs.append(interpreter_builder) pkg_descriptor = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) with ftl_util.Timing("installing pip packages"): self._pip_install(pkg_descriptor) with ftl_util.Timing("resolving whl paths"): whls = self._resolve_whls() pkg_dirs = [self._whl_to_fslayer(whl) for whl in whls] for whl_pkg_dir in pkg_dirs: layer_builder = package_builder.PackageLayerBuilder( self._ctx, self._descriptor_files, whl_pkg_dir, interpreter_builder) cached_pkg_img = None if self._args.cache: with ftl_util.Timing("checking cached pkg layer"): key = layer_builder.GetCacheKey() cached_pkg_img = self._cache.Get(key) if cached_pkg_img is not None: layer_builder.SetImage(cached_pkg_img) else: with ftl_util.Timing("building pkg layer"): layer_builder.BuildLayer() if self._args.cache: with ftl_util.Timing("uploading pkg layer"): self._cache.Set(layer_builder.GetCacheKey(), layer_builder.GetImage()) lyr_imgs.append(layer_builder) app = base_builder.AppLayerBuilder( ctx=self._ctx, destination_path=self._args.destination_path, entrypoint=self._args.entrypoint, exposed_ports=self._args.exposed_ports) with ftl_util.Timing("builder app layer"): app.BuildLayer() lyr_imgs.append(app) with ftl_util.Timing("stitching lyrs into final image"): ftl_image = self.AppendLayersIntoImage(lyr_imgs) with ftl_util.Timing("uploading final image"): self.StoreImage(ftl_image)
def GetCacheKeyRaw(self): descriptor_contents = ftl_util.descriptor_parser( self._descriptor_files, self._ctx) cache_key = '%s %s' % (descriptor_contents, self._dep_img_lyr.GetCacheKeyRaw()) return "%s %s" % (cache_key, self._cache_key_version)