def _build_layer(self, layer_name, codeuri, runtime):
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())
config = get_workflow_config(runtime, code_dir, self._base_dir)
subfolder = get_layer_subfolder(runtime)
# artifacts directory will be created by the builder
artifacts_dir = str(
pathlib.Path(self._build_dir, layer_name, subfolder))
with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(
code_dir, config.manifest_name)
# By default prefer to build in-process for speed
build_method = self._build_function_in_process
if self._container_manager:
build_method = self._build_function_on_container
build_method(config, code_dir, artifacts_dir, scratch_dir,
manifest_path, runtime, None)
# Not including subfolder in return so that we copy subfolder, instead of copying artifacts inside it.
return str(pathlib.Path(self._build_dir, layer_name))
def _build_functions(self):
"""
Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder
"""
build_graph = self._get_build_graph()
function_build_results = {}
for build_definition in build_graph.get_build_definitions():
LOG.info(
"Building codeuri: %s runtime: %s metadata: %s functions: %s",
build_definition.codeuri, build_definition.runtime,
build_definition.metadata,
[function.name for function in build_definition.functions])
with osutils.mkdir_temp() as temporary_build_dir:
LOG.debug("Building to following folder %s",
temporary_build_dir)
self._build_function(build_definition.get_function_name(),
build_definition.codeuri,
build_definition.runtime,
build_definition.get_handler_name(),
temporary_build_dir,
build_definition.metadata)
for function in build_definition.functions:
# artifacts directory will be created by the builder
artifacts_dir = str(
pathlib.Path(self._build_dir, function.name))
LOG.debug("Copying artifacts from %s to %s",
temporary_build_dir, artifacts_dir)
osutils.copytree(temporary_build_dir, artifacts_dir)
function_build_results[function.name] = artifacts_dir
return function_build_results
def test_if_cached_invalid_with_no_cached_folder(self, build_layer_mock,
build_function_mock,
copytree_mock):
with osutils.mkdir_temp() as temp_base_dir:
build_dir = Path(temp_base_dir, ".aws-sam", "build")
build_dir.mkdir(parents=True)
cache_dir = Path(temp_base_dir, ".aws-sam", "cache")
cache_dir.mkdir(parents=True)
build_function_mock.return_value = {
"HelloWorldPython": "artifact1",
"HelloWorldPython2": "artifact2"
}
build_layer_mock.return_value = {"SumLayer": "artifact3"}
build_graph_path = Path(build_dir.parent, "build.toml")
build_graph_path.write_text(
CachedBuildStrategyTest.BUILD_GRAPH_CONTENTS)
build_graph = BuildGraph(str(build_dir))
cached_build_strategy = CachedBuildStrategy(
build_graph, DefaultBuildStrategy, temp_base_dir, build_dir,
cache_dir, True)
cached_build_strategy.build_single_function_definition(
build_graph.get_function_build_definitions()[0])
cached_build_strategy.build_single_layer_definition(
build_graph.get_layer_build_definitions()[0])
build_function_mock.assert_called_once()
build_layer_mock.assert_called_once()
self.assertEqual(copytree_mock.call_count, 2)
def _overwrite_existing_templates(self, expected_path: str):
self.repo_path = expected_path
# workflow to clone a copy to a new directory and overwrite
with osutils.mkdir_temp(ignore_errors=True) as tempdir:
try:
expected_temp_path = os.path.normpath(
os.path.join(tempdir, self._repo_name))
LOG.info("\nCloning app templates from %s", self._repo_url)
subprocess.check_output(
[
self._git_executable(), "clone", self._repo_url,
self._repo_name
],
cwd=tempdir,
stderr=subprocess.STDOUT,
)
# Now we need to delete the old repo and move this one.
self._replace_app_templates(expected_temp_path, expected_path)
self.repo_path = expected_path
except OSError as ex:
LOG.warning("WARN: Could not clone app template repo.",
exc_info=ex)
except subprocess.CalledProcessError as clone_error:
output = clone_error.output.decode("utf-8")
if "not found" in output.lower():
click.echo("WARN: Could not clone app template repo.")
def _build_layer(
self, layer_name: str, codeuri: str, specified_workflow: str, compatible_runtimes: List[str]
) -> str:
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())
config = get_workflow_config(None, code_dir, self._base_dir, specified_workflow)
subfolder = get_layer_subfolder(specified_workflow)
# artifacts directory will be created by the builder
artifacts_dir = str(pathlib.Path(self._build_dir, layer_name, subfolder))
with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(code_dir, config.manifest_name)
# By default prefer to build in-process for speed
build_runtime = specified_workflow
build_method = self._build_function_in_process
if self._container_manager:
build_method = self._build_function_on_container
if config.language == "provided":
LOG.warning(
"For container layer build, first compatible runtime is chosen as build target for container."
)
# Only set to this value if specified workflow is makefile
# which will result in config language as provided
build_runtime = compatible_runtimes[0]
options = ApplicationBuilder._get_build_options(layer_name, config.language, None)
build_method(config, code_dir, artifacts_dir, scratch_dir, manifest_path, build_runtime, options)
# Not including subfolder in return so that we copy subfolder, instead of copying artifacts inside it.
return str(pathlib.Path(self._build_dir, layer_name))
def _download_and_copy(download_fn, output_dir):
"""
Runs the download function to download files into a temporary directory and then copy the files over to
the ``output_dir``
Parameters
----------
download_fn : function
Method to be called to download. It needs to accept a parameter called `clone_to_dir`. This will be
set to the temporary directory
output_dir : str
Path to the directory where files will be copied to
Returns
-------
output_dir
"""
with osutils.mkdir_temp(ignore_errors=True) as tempdir:
downloaded_dir = download_fn(clone_to_dir=tempdir)
osutils.copytree(downloaded_dir,
output_dir,
ignore=shutil.ignore_patterns("*.git"))
return output_dir
def test_functions_should_be_added_existing_build_graph(self):
with osutils.mkdir_temp() as temp_base_dir:
build_dir = Path(temp_base_dir, ".aws-sam", "build")
build_dir.mkdir(parents=True)
build_graph_path = Path(build_dir.parent, "build.toml")
build_graph_path.write_text(TestBuildGraph.BUILD_GRAPH_CONTENTS)
build_graph = BuildGraph(str(build_dir))
build_definition1 = BuildDefinition(TestBuildGraph.RUNTIME,
TestBuildGraph.CODEURI,
TestBuildGraph.METADATA)
function1 = generate_function(runtime=TestBuildGraph.RUNTIME,
codeuri=TestBuildGraph.CODEURI,
metadata=TestBuildGraph.METADATA)
build_graph.put_build_definition(build_definition1, function1)
self.assertTrue(len(build_graph.get_build_definitions()), 1)
for build_definition in build_graph.get_build_definitions():
self.assertTrue(len(build_definition.functions), 1)
self.assertTrue(build_definition.functions[0], function1)
self.assertEqual(build_definition.uuid, TestBuildGraph.UUID)
build_definition2 = BuildDefinition("another_runtime",
"another_codeuri", None)
function2 = generate_function(name="another_function")
build_graph.put_build_definition(build_definition2, function2)
self.assertTrue(len(build_graph.get_build_definitions()), 2)
def _build_function(self, function_name, codeuri, runtime, handler, metadata=None):
"""
Given the function information, this method will build the Lambda function. Depending on the configuration
it will either build the function in process or by spinning up a Docker container.
Parameters
----------
function_name : str
Name or LogicalId of the function
codeuri : str
Path to where the code lives
runtime : str
AWS Lambda function runtime
metadata : dict
AWS Lambda function metadata
Returns
-------
str
Path to the location where built artifacts are available
"""
if runtime in self._deprecated_runtimes:
message = f"WARNING: {runtime} is no longer supported by AWS Lambda, please update to a newer supported runtime. SAM CLI " \
f"will drop support for all deprecated runtimes {self._deprecated_runtimes} on May 1st. " \
f"See issue: https://github.com/awslabs/aws-sam-cli/issues/1934 for more details."
LOG.warning(self._colored.yellow(message))
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())
# Determine if there was a build workflow that was specified directly in the template.
specified_build_workflow = metadata.get("BuildMethod", None) if metadata else None
config = get_workflow_config(runtime, code_dir, self._base_dir, specified_workflow=specified_build_workflow)
# artifacts directory will be created by the builder
artifacts_dir = str(pathlib.Path(self._build_dir, function_name))
with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(code_dir, config.manifest_name)
# By default prefer to build in-process for speed
build_method = self._build_function_in_process
if self._container_manager:
build_method = self._build_function_on_container
options = ApplicationBuilder._get_build_options(function_name, config.language, handler)
return build_method(config,
code_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime,
options)
def test_if_cached_valid_when_build_single_function_definition(
self, dir_checksum_mock, exists_mock, copytree_mock):
pass
with osutils.mkdir_temp() as temp_base_dir:
build_dir = Path(temp_base_dir, ".aws-sam", "build")
build_dir.mkdir(parents=True)
cache_dir = Path(temp_base_dir, ".aws-sam", "cache")
cache_dir.mkdir(parents=True)
exists_mock.return_value = True
dir_checksum_mock.return_value = CachedBuildStrategyTest.SOURCE_MD5
build_graph_path = Path(build_dir.parent, "build.toml")
build_graph_path.write_text(
CachedBuildStrategyTest.BUILD_GRAPH_CONTENTS)
build_graph = BuildGraph(str(build_dir))
cached_build_strategy = CachedBuildStrategy(
build_graph, DefaultBuildStrategy, temp_base_dir, build_dir,
cache_dir, True)
func1 = Mock()
func1.name = "func1_name"
func2 = Mock()
func2.name = "func2_name"
build_definition = build_graph.get_function_build_definitions()[0]
layer_definition = build_graph.get_layer_build_definitions()[0]
build_graph.put_function_build_definition(build_definition, func1)
build_graph.put_function_build_definition(build_definition, func2)
layer = Mock()
layer.name = "layer_name"
build_graph.put_layer_build_definition(layer_definition, layer)
cached_build_strategy.build_single_function_definition(
build_definition)
cached_build_strategy.build_single_layer_definition(
layer_definition)
self.assertEqual(copytree_mock.call_count, 3)
def test_should_instantiate_first_time_and_update(self):
with osutils.mkdir_temp() as temp_base_dir:
build_dir = Path(temp_base_dir, ".aws-sam", "build")
build_dir.mkdir(parents=True)
# create a build graph and persist it
build_graph1 = BuildGraph(str(build_dir))
build_definition1 = FunctionBuildDefinition(
TestBuildGraph.RUNTIME,
TestBuildGraph.CODEURI,
TestBuildGraph.ZIP,
TestBuildGraph.METADATA,
TestBuildGraph.SOURCE_MD5,
)
function1 = generate_function(runtime=TestBuildGraph.RUNTIME,
codeuri=TestBuildGraph.CODEURI,
metadata=TestBuildGraph.METADATA)
build_graph1.put_function_build_definition(build_definition1,
function1)
build_graph1.clean_redundant_definitions_and_update(True)
# read previously persisted graph and compare
build_graph2 = BuildGraph(str(build_dir))
self.assertEqual(
len(build_graph1.get_function_build_definitions()),
len(build_graph2.get_function_build_definitions()))
self.assertEqual(
list(build_graph1.get_function_build_definitions())[0],
list(build_graph2.get_function_build_definitions())[0],
)
def test_must_delete_temp_dir_after_use(self):
dir_name = None
with osutils.mkdir_temp() as tempdir:
dir_name = tempdir
self.assertTrue(os.path.exists(tempdir))
self.assertFalse(os.path.exists(dir_name))
def test_must_delete_temp_dir_after_use(self):
dir_name = None
with osutils.mkdir_temp() as tempdir:
dir_name = tempdir
self.assertTrue(os.path.exists(tempdir))
self.assertFalse(os.path.exists(dir_name))
def test_should_instantiate_first_time(self):
with osutils.mkdir_temp() as temp_base_dir:
build_dir = Path(temp_base_dir, ".aws-sam", "build")
build_dir.mkdir(parents=True)
build_graph1 = BuildGraph(str(build_dir.resolve()))
build_graph1.clean_redundant_functions_and_update(True)
build_graph2 = BuildGraph(str(build_dir.resolve()))
self.assertEqual(build_graph1.get_build_definitions(), build_graph2.get_build_definitions())
def test_redundant_cached_should_be_clean(self):
with osutils.mkdir_temp() as temp_base_dir:
build_dir = Path(temp_base_dir, ".aws-sam", "build")
build_dir.mkdir(parents=True)
build_graph = BuildGraph(str(build_dir.resolve()))
cache_dir = Path(temp_base_dir, ".aws-sam", "cache")
cache_dir.mkdir(parents=True)
redundant_cache_folder = Path(cache_dir, "redundant")
redundant_cache_folder.mkdir(parents=True)
cached_build_strategy = CachedBuildStrategy(build_graph, Mock(), temp_base_dir, build_dir, cache_dir, True)
cached_build_strategy._clean_redundant_cached()
self.assertTrue(not redundant_cache_folder.exists())
def test_should_read_existing_build_graph(self):
with osutils.mkdir_temp() as temp_base_dir:
build_dir = Path(temp_base_dir, ".aws-sam", "build")
build_dir.mkdir(parents=True)
build_graph_path = Path(build_dir.parent, "build.toml")
build_graph_path.write_text(TestBuildGraph.BUILD_GRAPH_CONTENTS)
build_graph = BuildGraph(str(build_dir))
for build_definition in build_graph.get_build_definitions():
self.assertEqual(build_definition.codeuri, TestBuildGraph.CODEURI)
self.assertEqual(build_definition.runtime, TestBuildGraph.RUNTIME)
self.assertEqual(build_definition.metadata, TestBuildGraph.METADATA)
def _clone_new_app_templates(self, shared_dir, expected_path):
with osutils.mkdir_temp(ignore_errors=True) as tempdir:
expected_temp_path = os.path.normpath(os.path.join(tempdir, self._repo_name))
try:
LOG.info("\nCloning app templates from %s", self._repo_url)
subprocess.check_output(
[self._git_executable(), "clone", self._repo_url], cwd=tempdir, stderr=subprocess.STDOUT
)
shutil.copytree(expected_temp_path, expected_path, ignore=shutil.ignore_patterns("*.git"))
self.repo_path = expected_path
except OSError as ex:
LOG.warning("WARN: Can't clone app repo, git executable not found", exc_info=ex)
except subprocess.CalledProcessError as clone_error:
output = clone_error.output.decode("utf-8")
if "not found" in output.lower():
click.echo("WARN: Could not clone app template repo.")
def _build_function(self, function_name, codeuri, runtime, handler):
"""
Given the function information, this method will build the Lambda function. Depending on the configuration
it will either build the function in process or by spinning up a Docker container.
Parameters
----------
function_name : str
Name or LogicalId of the function
codeuri : str
Path to where the code lives
runtime : str
AWS Lambda function runtime
Returns
-------
str
Path to the location where built artifacts are available
"""
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())
config = get_workflow_config(runtime, code_dir, self._base_dir)
# artifacts directory will be created by the builder
artifacts_dir = str(pathlib.Path(self._build_dir, function_name))
with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(
code_dir, config.manifest_name)
# By default prefer to build in-process for speed
build_method = self._build_function_in_process
if self._container_manager:
build_method = self._build_function_on_container
options = ApplicationBuilder._get_build_options(
config.language, handler)
return build_method(config, code_dir, artifacts_dir, scratch_dir,
manifest_path, runtime, options)
def _build_function(self, function_name, codeuri, runtime):
config = _get_workflow_config(runtime)
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())
# artifacts directory will be created by the builder
artifacts_dir = str(pathlib.Path(self._build_dir, function_name))
with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(
code_dir, config.manifest_name)
# By default prefer to build in-process for speed
build_method = self._build_function_in_process
if self._container_manager:
build_method = self._build_function_on_container
return build_method(config, code_dir, artifacts_dir, scratch_dir,
manifest_path, runtime)
def test_raises_on_cleanup_failure(self, rmdir_mock):
rmdir_mock.side_effect = OSError("fail")
with self.assertRaises(OSError):
with osutils.mkdir_temp() as tempdir:
self.assertTrue(os.path.exists(tempdir))
def test_must_return_temp_dir(self):
with osutils.mkdir_temp() as tempdir:
self.assertTrue(os.path.exists(tempdir))
def test_must_return_temp_dir(self):
with osutils.mkdir_temp() as tempdir:
self.assertTrue(os.path.exists(tempdir))
def test_handles_ignore_error_case(self, rmdir_mock):
rmdir_mock.side_effect = OSError("fail")
dir_name = None
with osutils.mkdir_temp(ignore_errors=True) as tempdir:
dir_name = tempdir
self.assertTrue(os.path.exists(tempdir))
def _build_function( # pylint: disable=R1710
self,
function_name: str,
codeuri: str,
packagetype: str,
runtime: str,
handler: Optional[str],
artifact_dir: str,
metadata: Optional[Dict] = None,
container_env_vars: Optional[Dict] = None,
) -> str:
"""
Given the function information, this method will build the Lambda function. Depending on the configuration
it will either build the function in process or by spinning up a Docker container.
Parameters
----------
function_name : str
Name or LogicalId of the function
codeuri : str
Path to where the code lives
packagetype : str
The package type, 'Zip' or 'Image', see samcli/lib/utils/packagetype.py
runtime : str
AWS Lambda function runtime
handler : Optional[str]
An optional string to specify which function the handler should be
artifact_dir: str
Path to where function will be build into
metadata : dict
AWS Lambda function metadata
container_env_vars : Optional[Dict]
An optional dictionary of environment variables to pass to the container.
Returns
-------
str
Path to the location where built artifacts are available
"""
if packagetype == IMAGE:
# pylint: disable=fixme
# FIXME: _build_lambda_image assumes metadata is not None, we need to throw an exception here
return self._build_lambda_image(function_name=function_name,
metadata=metadata) # type: ignore
if packagetype == ZIP:
if runtime in self._deprecated_runtimes:
message = (
f"WARNING: {runtime} is no longer supported by AWS Lambda, "
"please update to a newer supported runtime. SAM CLI "
f"will drop support for all deprecated runtimes {self._deprecated_runtimes} on May 1st. "
"See issue: https://github.com/awslabs/aws-sam-cli/issues/1934 for more details."
)
LOG.warning(self._colored.yellow(message))
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())
# Determine if there was a build workflow that was specified directly in the template.
specified_build_workflow = metadata.get("BuildMethod",
None) if metadata else None
config = get_workflow_config(
runtime,
code_dir,
self._base_dir,
specified_workflow=specified_build_workflow)
with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(
code_dir, config.manifest_name)
options = ApplicationBuilder._get_build_options(
function_name, config.language, handler)
# By default prefer to build in-process for speed
build_method = self._build_function_in_process
if self._container_manager:
build_method = self._build_function_on_container
return build_method(
config,
code_dir,
artifact_dir,
scratch_dir,
manifest_path,
runtime,
options,
container_env_vars,
)
return build_method(config, code_dir, artifact_dir,
scratch_dir, manifest_path, runtime,
options)
# pylint: disable=fixme
# FIXME: we need to throw an exception here, packagetype could be something else
return # type: ignore
def _build_layer(
self,
layer_name: str,
codeuri: str,
specified_workflow: str,
compatible_runtimes: List[str],
artifact_dir: str,
container_env_vars: Optional[Dict] = None,
) -> str:
"""
Given the layer information, this method will build the Lambda layer. Depending on the configuration
it will either build the function in process or by spinning up a Docker container.
Parameters
----------
layer_name : str
Name or LogicalId of the function
codeuri : str
Path to where the code lives
specified_workflow : str
The specified workflow
compatible_runtimes : List[str]
List of runtimes the layer build is compatible with
artifact_dir : str
Path to where layer will be build into.
A subfolder will be created in this directory depending on the specified workflow.
container_env_vars : Optional[Dict]
An optional dictionary of environment variables to pass to the container.
Returns
-------
str
Path to the location where built artifacts are available
"""
# Create the arguments to pass to the builder
# Code is always relative to the given base directory.
code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve())
config = get_workflow_config(None, code_dir, self._base_dir,
specified_workflow)
subfolder = get_layer_subfolder(specified_workflow)
# artifacts directory will be created by the builder
artifact_subdir = str(pathlib.Path(artifact_dir, subfolder))
with osutils.mkdir_temp() as scratch_dir:
manifest_path = self._manifest_path_override or os.path.join(
code_dir, config.manifest_name)
# By default prefer to build in-process for speed
build_runtime = specified_workflow
build_method = self._build_function_in_process
if self._container_manager:
build_method = self._build_function_on_container
if config.language == "provided":
LOG.warning(
"For container layer build, first compatible runtime is chosen as build target for container."
)
# Only set to this value if specified workflow is makefile
# which will result in config language as provided
build_runtime = compatible_runtimes[0]
options = ApplicationBuilder._get_build_options(
layer_name, config.language, None)
build_method(
config,
code_dir,
artifact_subdir,
scratch_dir,
manifest_path,
build_runtime,
options,
container_env_vars,
)
# Not including subfolder in return so that we copy subfolder, instead of copying artifacts inside it.
return artifact_dir
