def _download_and_copy(download_fn, output_dir):
"""
Runs the download function to download files into a temporary directory and then copy the files over to
the ``output_dir``
Parameters
----------
download_fn : function
Method to be called to download. It needs to accept a parameter called `clone_to_dir`. This will be
set to the temporary directory
output_dir : str
Path to the directory where files will be copied to
Returns
-------
output_dir
"""
with osutils.mkdir_temp(ignore_errors=True) as tempdir:
downloaded_dir = download_fn(clone_to_dir=tempdir)
osutils.copytree(downloaded_dir,
output_dir,
ignore=shutil.ignore_patterns("*.git"))
return output_dir
def build_single_layer_definition(self, layer_definition):
"""
Builds single layer definition with caching
"""
code_dir = str(pathlib.Path(self._base_dir, layer_definition.codeuri).resolve())
source_md5 = dir_checksum(code_dir)
cache_function_dir = pathlib.Path(self._cache_dir, layer_definition.uuid)
layer_build_result = {}
if not cache_function_dir.exists() or layer_definition.source_md5 != source_md5:
LOG.info("Cache is invalid, running build and copying resources to layer build definition of %s",
layer_definition.uuid)
build_result = self._delegate_build_strategy.build_single_layer_definition(layer_definition)
layer_build_result.update(build_result)
if cache_function_dir.exists():
shutil.rmtree(str(cache_function_dir))
layer_definition.source_md5 = source_md5
# Since all the build contents are same for a build definition, just copy any one of them into the cache
for _, value in build_result.items():
osutils.copytree(value, cache_function_dir)
break
else:
LOG.info("Valid cache found, copying previously built resources from layer build definition of %s",
layer_definition.uuid)
# artifacts directory will be created by the builder
artifacts_dir = str(pathlib.Path(self._build_dir, layer_definition.layer.name))
LOG.debug("Copying artifacts from %s to %s", cache_function_dir, artifacts_dir)
osutils.copytree(cache_function_dir, artifacts_dir)
layer_build_result[layer_definition.layer.name] = artifacts_dir
return layer_build_result
def build_single_function_definition(self, build_definition):
"""
Build the unique definition and then copy the artifact to the corresponding function folder
"""
function_build_results = {}
LOG.info("Building codeuri: %s runtime: %s metadata: %s functions: %s",
build_definition.codeuri, build_definition.runtime,
build_definition.metadata,
[function.name for function in build_definition.functions])
# build into one of the functions from this build definition
single_function_name = build_definition.get_function_name()
single_build_dir = str(
pathlib.Path(self._build_dir, single_function_name))
LOG.debug("Building to following folder %s", single_build_dir)
self._build_function(build_definition.get_function_name(),
build_definition.codeuri,
build_definition.runtime,
build_definition.get_handler_name(),
single_build_dir, build_definition.metadata)
function_build_results[single_function_name] = single_build_dir
# copy results to other functions
for function in build_definition.functions:
if function.name is not single_function_name:
# artifacts directory will be created by the builder
artifacts_dir = str(
pathlib.Path(self._build_dir, function.name))
LOG.debug("Copying artifacts from %s to %s", single_build_dir,
artifacts_dir)
osutils.copytree(single_build_dir, artifacts_dir)
function_build_results[function.name] = artifacts_dir
return function_build_results
def _build_functions(self):
"""
Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder
"""
build_graph = self._get_build_graph()
function_build_results = {}
for build_definition in build_graph.get_build_definitions():
LOG.info(
"Building codeuri: %s runtime: %s metadata: %s functions: %s",
build_definition.codeuri, build_definition.runtime,
build_definition.metadata,
[function.name for function in build_definition.functions])
with osutils.mkdir_temp() as temporary_build_dir:
LOG.debug("Building to following folder %s",
temporary_build_dir)
self._build_function(build_definition.get_function_name(),
build_definition.codeuri,
build_definition.runtime,
build_definition.get_handler_name(),
temporary_build_dir,
build_definition.metadata)
for function in build_definition.functions:
# artifacts directory will be created by the builder
artifacts_dir = str(
pathlib.Path(self._build_dir, function.name))
LOG.debug("Copying artifacts from %s to %s",
temporary_build_dir, artifacts_dir)
osutils.copytree(temporary_build_dir, artifacts_dir)
function_build_results[function.name] = artifacts_dir
return function_build_results
def build_single_function_definition(
self, build_definition: FunctionBuildDefinition) -> Dict[str, str]:
"""
Build the unique definition and then copy the artifact to the corresponding function folder
"""
function_build_results = {}
LOG.info(
"Building codeuri: %s runtime: %s metadata: %s functions: %s",
build_definition.codeuri,
build_definition.runtime,
build_definition.metadata,
[function.full_path for function in build_definition.functions],
)
# build into one of the functions from this build definition
single_full_path = build_definition.get_full_path()
single_build_dir = build_definition.get_build_dir(self._build_dir)
LOG.debug("Building to following folder %s", single_build_dir)
# when a function is passed here, it is ZIP function, codeuri and runtime are not None
result = self._build_function(
build_definition.get_function_name(),
build_definition.codeuri, # type: ignore
build_definition.packagetype,
build_definition.runtime, # type: ignore
build_definition.get_handler_name(),
single_build_dir,
build_definition.metadata,
build_definition.env_vars,
)
function_build_results[single_full_path] = result
# copy results to other functions
if build_definition.packagetype == ZIP:
for function in build_definition.functions:
if function.full_path != single_full_path:
# for zip function we need to copy over the artifacts
# artifacts directory will be created by the builder
artifacts_dir = function.get_build_dir(self._build_dir)
LOG.debug("Copying artifacts from %s to %s",
single_build_dir, artifacts_dir)
osutils.copytree(single_build_dir, artifacts_dir)
function_build_results[function.full_path] = artifacts_dir
elif build_definition.packagetype == IMAGE:
for function in build_definition.functions:
if function.full_path != single_full_path:
# for image function, we just need to copy the image tag
function_build_results[function.full_path] = result
return function_build_results
def build_single_function_definition(
self, build_definition: FunctionBuildDefinition) -> Dict[str, str]:
"""
Builds single function definition with caching
"""
if build_definition.packagetype == IMAGE:
return self._delegate_build_strategy.build_single_function_definition(
build_definition)
code_dir = str(
pathlib.Path(self._base_dir, build_definition.codeuri).resolve())
source_md5 = dir_checksum(code_dir)
cache_function_dir = pathlib.Path(self._cache_dir,
build_definition.uuid)
function_build_results = {}
if not cache_function_dir.exists(
) or build_definition.source_md5 != source_md5:
LOG.info(
"Cache is invalid, running build and copying resources to function build definition of %s",
build_definition.uuid,
)
build_result = self._delegate_build_strategy.build_single_function_definition(
build_definition)
function_build_results.update(build_result)
if cache_function_dir.exists():
shutil.rmtree(str(cache_function_dir))
build_definition.source_md5 = source_md5
# Since all the build contents are same for a build definition, just copy any one of them into the cache
for _, value in build_result.items():
osutils.copytree(value, cache_function_dir)
break
else:
LOG.info(
"Valid cache found, copying previously built resources from function build definition of %s",
build_definition.uuid,
)
for function in build_definition.functions:
# artifacts directory will be created by the builder
artifacts_dir = str(
pathlib.Path(self._build_dir, function.name))
LOG.debug("Copying artifacts from %s to %s",
cache_function_dir, artifacts_dir)
osutils.copytree(cache_function_dir, artifacts_dir)
function_build_results[function.name] = artifacts_dir
return function_build_results
def _prepare_application_environment(self):
"""
Create an application environment where Gemfile will be in the root folder of the app;
├── RubyWithRootGemfile
│ └── app.rb
├── Gemfile
└── template.yaml
"""
# copy gemfile to the root of the project
shutil.copyfile(Path(self.template_path).parent.joinpath("Gemfile"), Path(self.working_dir).joinpath("Gemfile"))
# copy function source code in its folder
osutils.copytree(
Path(self.template_path).parent.joinpath("RubyWithRootGemfile"),
Path(self.working_dir).joinpath("RubyWithRootGemfile"),
)
# copy template to the root folder
shutil.copyfile(Path(self.template_path), Path(self.working_dir).joinpath("template.yaml"))
# update template path with new location
self.template_path = str(Path(self.working_dir).joinpath("template.yaml"))
def _build_functions(self):
"""
Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder
"""
build_graph = self._get_build_graph()
function_build_results = {}
for build_definition in build_graph.get_build_definitions():
LOG.info("Building codeuri: %s runtime: %s metadata: %s functions: %s",
build_definition.codeuri,
build_definition.runtime,
build_definition.metadata,
[function.name for function in build_definition.functions])
# build into one of the functions from this build definition
single_function_name = build_definition.get_function_name()
single_build_dir = str(pathlib.Path(self._build_dir, single_function_name))
LOG.debug("Building to following folder %s", single_build_dir)
self._build_function(build_definition.get_function_name(),
build_definition.codeuri,
build_definition.runtime,
build_definition.get_handler_name(),
single_build_dir,
build_definition.metadata)
function_build_results[single_function_name] = single_build_dir
# copy results to other functions
for function in build_definition.functions:
if function.name is not single_function_name:
# artifacts directory will be created by the builder
artifacts_dir = str(pathlib.Path(self._build_dir, function.name))
LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir)
osutils.copytree(single_build_dir, artifacts_dir)
function_build_results[function.name] = artifacts_dir
return function_build_results
