def _download_and_copy(download_fn, output_dir): """ Runs the download function to download files into a temporary directory and then copy the files over to the ``output_dir`` Parameters ---------- download_fn : function Method to be called to download. It needs to accept a parameter called `clone_to_dir`. This will be set to the temporary directory output_dir : str Path to the directory where files will be copied to Returns ------- output_dir """ with osutils.mkdir_temp(ignore_errors=True) as tempdir: downloaded_dir = download_fn(clone_to_dir=tempdir) osutils.copytree(downloaded_dir, output_dir, ignore=shutil.ignore_patterns("*.git")) return output_dir
def build_single_layer_definition(self, layer_definition): """ Builds single layer definition with caching """ code_dir = str(pathlib.Path(self._base_dir, layer_definition.codeuri).resolve()) source_md5 = dir_checksum(code_dir) cache_function_dir = pathlib.Path(self._cache_dir, layer_definition.uuid) layer_build_result = {} if not cache_function_dir.exists() or layer_definition.source_md5 != source_md5: LOG.info("Cache is invalid, running build and copying resources to layer build definition of %s", layer_definition.uuid) build_result = self._delegate_build_strategy.build_single_layer_definition(layer_definition) layer_build_result.update(build_result) if cache_function_dir.exists(): shutil.rmtree(str(cache_function_dir)) layer_definition.source_md5 = source_md5 # Since all the build contents are same for a build definition, just copy any one of them into the cache for _, value in build_result.items(): osutils.copytree(value, cache_function_dir) break else: LOG.info("Valid cache found, copying previously built resources from layer build definition of %s", layer_definition.uuid) # artifacts directory will be created by the builder artifacts_dir = str(pathlib.Path(self._build_dir, layer_definition.layer.name)) LOG.debug("Copying artifacts from %s to %s", cache_function_dir, artifacts_dir) osutils.copytree(cache_function_dir, artifacts_dir) layer_build_result[layer_definition.layer.name] = artifacts_dir return layer_build_result
def build_single_function_definition(self, build_definition): """ Build the unique definition and then copy the artifact to the corresponding function folder """ function_build_results = {} LOG.info("Building codeuri: %s runtime: %s metadata: %s functions: %s", build_definition.codeuri, build_definition.runtime, build_definition.metadata, [function.name for function in build_definition.functions]) # build into one of the functions from this build definition single_function_name = build_definition.get_function_name() single_build_dir = str( pathlib.Path(self._build_dir, single_function_name)) LOG.debug("Building to following folder %s", single_build_dir) self._build_function(build_definition.get_function_name(), build_definition.codeuri, build_definition.runtime, build_definition.get_handler_name(), single_build_dir, build_definition.metadata) function_build_results[single_function_name] = single_build_dir # copy results to other functions for function in build_definition.functions: if function.name is not single_function_name: # artifacts directory will be created by the builder artifacts_dir = str( pathlib.Path(self._build_dir, function.name)) LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir) osutils.copytree(single_build_dir, artifacts_dir) function_build_results[function.name] = artifacts_dir return function_build_results
def _build_functions(self): """ Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder """ build_graph = self._get_build_graph() function_build_results = {} for build_definition in build_graph.get_build_definitions(): LOG.info( "Building codeuri: %s runtime: %s metadata: %s functions: %s", build_definition.codeuri, build_definition.runtime, build_definition.metadata, [function.name for function in build_definition.functions]) with osutils.mkdir_temp() as temporary_build_dir: LOG.debug("Building to following folder %s", temporary_build_dir) self._build_function(build_definition.get_function_name(), build_definition.codeuri, build_definition.runtime, build_definition.get_handler_name(), temporary_build_dir, build_definition.metadata) for function in build_definition.functions: # artifacts directory will be created by the builder artifacts_dir = str( pathlib.Path(self._build_dir, function.name)) LOG.debug("Copying artifacts from %s to %s", temporary_build_dir, artifacts_dir) osutils.copytree(temporary_build_dir, artifacts_dir) function_build_results[function.name] = artifacts_dir return function_build_results
def build_single_function_definition( self, build_definition: FunctionBuildDefinition) -> Dict[str, str]: """ Build the unique definition and then copy the artifact to the corresponding function folder """ function_build_results = {} LOG.info( "Building codeuri: %s runtime: %s metadata: %s functions: %s", build_definition.codeuri, build_definition.runtime, build_definition.metadata, [function.full_path for function in build_definition.functions], ) # build into one of the functions from this build definition single_full_path = build_definition.get_full_path() single_build_dir = build_definition.get_build_dir(self._build_dir) LOG.debug("Building to following folder %s", single_build_dir) # when a function is passed here, it is ZIP function, codeuri and runtime are not None result = self._build_function( build_definition.get_function_name(), build_definition.codeuri, # type: ignore build_definition.packagetype, build_definition.runtime, # type: ignore build_definition.get_handler_name(), single_build_dir, build_definition.metadata, build_definition.env_vars, ) function_build_results[single_full_path] = result # copy results to other functions if build_definition.packagetype == ZIP: for function in build_definition.functions: if function.full_path != single_full_path: # for zip function we need to copy over the artifacts # artifacts directory will be created by the builder artifacts_dir = function.get_build_dir(self._build_dir) LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir) osutils.copytree(single_build_dir, artifacts_dir) function_build_results[function.full_path] = artifacts_dir elif build_definition.packagetype == IMAGE: for function in build_definition.functions: if function.full_path != single_full_path: # for image function, we just need to copy the image tag function_build_results[function.full_path] = result return function_build_results
def build_single_function_definition( self, build_definition: FunctionBuildDefinition) -> Dict[str, str]: """ Builds single function definition with caching """ if build_definition.packagetype == IMAGE: return self._delegate_build_strategy.build_single_function_definition( build_definition) code_dir = str( pathlib.Path(self._base_dir, build_definition.codeuri).resolve()) source_md5 = dir_checksum(code_dir) cache_function_dir = pathlib.Path(self._cache_dir, build_definition.uuid) function_build_results = {} if not cache_function_dir.exists( ) or build_definition.source_md5 != source_md5: LOG.info( "Cache is invalid, running build and copying resources to function build definition of %s", build_definition.uuid, ) build_result = self._delegate_build_strategy.build_single_function_definition( build_definition) function_build_results.update(build_result) if cache_function_dir.exists(): shutil.rmtree(str(cache_function_dir)) build_definition.source_md5 = source_md5 # Since all the build contents are same for a build definition, just copy any one of them into the cache for _, value in build_result.items(): osutils.copytree(value, cache_function_dir) break else: LOG.info( "Valid cache found, copying previously built resources from function build definition of %s", build_definition.uuid, ) for function in build_definition.functions: # artifacts directory will be created by the builder artifacts_dir = str( pathlib.Path(self._build_dir, function.name)) LOG.debug("Copying artifacts from %s to %s", cache_function_dir, artifacts_dir) osutils.copytree(cache_function_dir, artifacts_dir) function_build_results[function.name] = artifacts_dir return function_build_results
def _prepare_application_environment(self): """ Create an application environment where Gemfile will be in the root folder of the app; ├── RubyWithRootGemfile │ └── app.rb ├── Gemfile └── template.yaml """ # copy gemfile to the root of the project shutil.copyfile(Path(self.template_path).parent.joinpath("Gemfile"), Path(self.working_dir).joinpath("Gemfile")) # copy function source code in its folder osutils.copytree( Path(self.template_path).parent.joinpath("RubyWithRootGemfile"), Path(self.working_dir).joinpath("RubyWithRootGemfile"), ) # copy template to the root folder shutil.copyfile(Path(self.template_path), Path(self.working_dir).joinpath("template.yaml")) # update template path with new location self.template_path = str(Path(self.working_dir).joinpath("template.yaml"))
def _build_functions(self): """ Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder """ build_graph = self._get_build_graph() function_build_results = {} for build_definition in build_graph.get_build_definitions(): LOG.info("Building codeuri: %s runtime: %s metadata: %s functions: %s", build_definition.codeuri, build_definition.runtime, build_definition.metadata, [function.name for function in build_definition.functions]) # build into one of the functions from this build definition single_function_name = build_definition.get_function_name() single_build_dir = str(pathlib.Path(self._build_dir, single_function_name)) LOG.debug("Building to following folder %s", single_build_dir) self._build_function(build_definition.get_function_name(), build_definition.codeuri, build_definition.runtime, build_definition.get_handler_name(), single_build_dir, build_definition.metadata) function_build_results[single_function_name] = single_build_dir # copy results to other functions for function in build_definition.functions: if function.name is not single_function_name: # artifacts directory will be created by the builder artifacts_dir = str(pathlib.Path(self._build_dir, function.name)) LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir) osutils.copytree(single_build_dir, artifacts_dir) function_build_results[function.name] = artifacts_dir return function_build_results