def _build_layer(self, layer_name, codeuri, runtime): # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) config = get_workflow_config(runtime, code_dir, self._base_dir) subfolder = get_layer_subfolder(runtime) # artifacts directory will be created by the builder artifacts_dir = str( pathlib.Path(self._build_dir, layer_name, subfolder)) with osutils.mkdir_temp() as scratch_dir: manifest_path = self._manifest_path_override or os.path.join( code_dir, config.manifest_name) # By default prefer to build in-process for speed build_method = self._build_function_in_process if self._container_manager: build_method = self._build_function_on_container build_method(config, code_dir, artifacts_dir, scratch_dir, manifest_path, runtime, None) # Not including subfolder in return so that we copy subfolder, instead of copying artifacts inside it. return str(pathlib.Path(self._build_dir, layer_name))
def _build_functions(self): """ Iterates through build graph and runs each unique build and copies outcome to the corresponding function folder """ build_graph = self._get_build_graph() function_build_results = {} for build_definition in build_graph.get_build_definitions(): LOG.info( "Building codeuri: %s runtime: %s metadata: %s functions: %s", build_definition.codeuri, build_definition.runtime, build_definition.metadata, [function.name for function in build_definition.functions]) with osutils.mkdir_temp() as temporary_build_dir: LOG.debug("Building to following folder %s", temporary_build_dir) self._build_function(build_definition.get_function_name(), build_definition.codeuri, build_definition.runtime, build_definition.get_handler_name(), temporary_build_dir, build_definition.metadata) for function in build_definition.functions: # artifacts directory will be created by the builder artifacts_dir = str( pathlib.Path(self._build_dir, function.name)) LOG.debug("Copying artifacts from %s to %s", temporary_build_dir, artifacts_dir) osutils.copytree(temporary_build_dir, artifacts_dir) function_build_results[function.name] = artifacts_dir return function_build_results
def test_if_cached_invalid_with_no_cached_folder(self, build_layer_mock, build_function_mock, copytree_mock): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) cache_dir = Path(temp_base_dir, ".aws-sam", "cache") cache_dir.mkdir(parents=True) build_function_mock.return_value = { "HelloWorldPython": "artifact1", "HelloWorldPython2": "artifact2" } build_layer_mock.return_value = {"SumLayer": "artifact3"} build_graph_path = Path(build_dir.parent, "build.toml") build_graph_path.write_text( CachedBuildStrategyTest.BUILD_GRAPH_CONTENTS) build_graph = BuildGraph(str(build_dir)) cached_build_strategy = CachedBuildStrategy( build_graph, DefaultBuildStrategy, temp_base_dir, build_dir, cache_dir, True) cached_build_strategy.build_single_function_definition( build_graph.get_function_build_definitions()[0]) cached_build_strategy.build_single_layer_definition( build_graph.get_layer_build_definitions()[0]) build_function_mock.assert_called_once() build_layer_mock.assert_called_once() self.assertEqual(copytree_mock.call_count, 2)
def _overwrite_existing_templates(self, expected_path: str): self.repo_path = expected_path # workflow to clone a copy to a new directory and overwrite with osutils.mkdir_temp(ignore_errors=True) as tempdir: try: expected_temp_path = os.path.normpath( os.path.join(tempdir, self._repo_name)) LOG.info("\nCloning app templates from %s", self._repo_url) subprocess.check_output( [ self._git_executable(), "clone", self._repo_url, self._repo_name ], cwd=tempdir, stderr=subprocess.STDOUT, ) # Now we need to delete the old repo and move this one. self._replace_app_templates(expected_temp_path, expected_path) self.repo_path = expected_path except OSError as ex: LOG.warning("WARN: Could not clone app template repo.", exc_info=ex) except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): click.echo("WARN: Could not clone app template repo.")
def _build_layer( self, layer_name: str, codeuri: str, specified_workflow: str, compatible_runtimes: List[str] ) -> str: # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) config = get_workflow_config(None, code_dir, self._base_dir, specified_workflow) subfolder = get_layer_subfolder(specified_workflow) # artifacts directory will be created by the builder artifacts_dir = str(pathlib.Path(self._build_dir, layer_name, subfolder)) with osutils.mkdir_temp() as scratch_dir: manifest_path = self._manifest_path_override or os.path.join(code_dir, config.manifest_name) # By default prefer to build in-process for speed build_runtime = specified_workflow build_method = self._build_function_in_process if self._container_manager: build_method = self._build_function_on_container if config.language == "provided": LOG.warning( "For container layer build, first compatible runtime is chosen as build target for container." ) # Only set to this value if specified workflow is makefile # which will result in config language as provided build_runtime = compatible_runtimes[0] options = ApplicationBuilder._get_build_options(layer_name, config.language, None) build_method(config, code_dir, artifacts_dir, scratch_dir, manifest_path, build_runtime, options) # Not including subfolder in return so that we copy subfolder, instead of copying artifacts inside it. return str(pathlib.Path(self._build_dir, layer_name))
def _download_and_copy(download_fn, output_dir): """ Runs the download function to download files into a temporary directory and then copy the files over to the ``output_dir`` Parameters ---------- download_fn : function Method to be called to download. It needs to accept a parameter called `clone_to_dir`. This will be set to the temporary directory output_dir : str Path to the directory where files will be copied to Returns ------- output_dir """ with osutils.mkdir_temp(ignore_errors=True) as tempdir: downloaded_dir = download_fn(clone_to_dir=tempdir) osutils.copytree(downloaded_dir, output_dir, ignore=shutil.ignore_patterns("*.git")) return output_dir
def test_functions_should_be_added_existing_build_graph(self): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) build_graph_path = Path(build_dir.parent, "build.toml") build_graph_path.write_text(TestBuildGraph.BUILD_GRAPH_CONTENTS) build_graph = BuildGraph(str(build_dir)) build_definition1 = BuildDefinition(TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, TestBuildGraph.METADATA) function1 = generate_function(runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=TestBuildGraph.METADATA) build_graph.put_build_definition(build_definition1, function1) self.assertTrue(len(build_graph.get_build_definitions()), 1) for build_definition in build_graph.get_build_definitions(): self.assertTrue(len(build_definition.functions), 1) self.assertTrue(build_definition.functions[0], function1) self.assertEqual(build_definition.uuid, TestBuildGraph.UUID) build_definition2 = BuildDefinition("another_runtime", "another_codeuri", None) function2 = generate_function(name="another_function") build_graph.put_build_definition(build_definition2, function2) self.assertTrue(len(build_graph.get_build_definitions()), 2)
def _build_function(self, function_name, codeuri, runtime, handler, metadata=None): """ Given the function information, this method will build the Lambda function. Depending on the configuration it will either build the function in process or by spinning up a Docker container. Parameters ---------- function_name : str Name or LogicalId of the function codeuri : str Path to where the code lives runtime : str AWS Lambda function runtime metadata : dict AWS Lambda function metadata Returns ------- str Path to the location where built artifacts are available """ if runtime in self._deprecated_runtimes: message = f"WARNING: {runtime} is no longer supported by AWS Lambda, please update to a newer supported runtime. SAM CLI " \ f"will drop support for all deprecated runtimes {self._deprecated_runtimes} on May 1st. " \ f"See issue: https://github.com/awslabs/aws-sam-cli/issues/1934 for more details." LOG.warning(self._colored.yellow(message)) # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) # Determine if there was a build workflow that was specified directly in the template. specified_build_workflow = metadata.get("BuildMethod", None) if metadata else None config = get_workflow_config(runtime, code_dir, self._base_dir, specified_workflow=specified_build_workflow) # artifacts directory will be created by the builder artifacts_dir = str(pathlib.Path(self._build_dir, function_name)) with osutils.mkdir_temp() as scratch_dir: manifest_path = self._manifest_path_override or os.path.join(code_dir, config.manifest_name) # By default prefer to build in-process for speed build_method = self._build_function_in_process if self._container_manager: build_method = self._build_function_on_container options = ApplicationBuilder._get_build_options(function_name, config.language, handler) return build_method(config, code_dir, artifacts_dir, scratch_dir, manifest_path, runtime, options)
def test_if_cached_valid_when_build_single_function_definition( self, dir_checksum_mock, exists_mock, copytree_mock): pass with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) cache_dir = Path(temp_base_dir, ".aws-sam", "cache") cache_dir.mkdir(parents=True) exists_mock.return_value = True dir_checksum_mock.return_value = CachedBuildStrategyTest.SOURCE_MD5 build_graph_path = Path(build_dir.parent, "build.toml") build_graph_path.write_text( CachedBuildStrategyTest.BUILD_GRAPH_CONTENTS) build_graph = BuildGraph(str(build_dir)) cached_build_strategy = CachedBuildStrategy( build_graph, DefaultBuildStrategy, temp_base_dir, build_dir, cache_dir, True) func1 = Mock() func1.name = "func1_name" func2 = Mock() func2.name = "func2_name" build_definition = build_graph.get_function_build_definitions()[0] layer_definition = build_graph.get_layer_build_definitions()[0] build_graph.put_function_build_definition(build_definition, func1) build_graph.put_function_build_definition(build_definition, func2) layer = Mock() layer.name = "layer_name" build_graph.put_layer_build_definition(layer_definition, layer) cached_build_strategy.build_single_function_definition( build_definition) cached_build_strategy.build_single_layer_definition( layer_definition) self.assertEqual(copytree_mock.call_count, 3)
def test_should_instantiate_first_time_and_update(self): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) # create a build graph and persist it build_graph1 = BuildGraph(str(build_dir)) build_definition1 = FunctionBuildDefinition( TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, TestBuildGraph.ZIP, TestBuildGraph.METADATA, TestBuildGraph.SOURCE_MD5, ) function1 = generate_function(runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=TestBuildGraph.METADATA) build_graph1.put_function_build_definition(build_definition1, function1) build_graph1.clean_redundant_definitions_and_update(True) # read previously persisted graph and compare build_graph2 = BuildGraph(str(build_dir)) self.assertEqual( len(build_graph1.get_function_build_definitions()), len(build_graph2.get_function_build_definitions())) self.assertEqual( list(build_graph1.get_function_build_definitions())[0], list(build_graph2.get_function_build_definitions())[0], )
def test_must_delete_temp_dir_after_use(self): dir_name = None with osutils.mkdir_temp() as tempdir: dir_name = tempdir self.assertTrue(os.path.exists(tempdir)) self.assertFalse(os.path.exists(dir_name))
def test_must_delete_temp_dir_after_use(self): dir_name = None with osutils.mkdir_temp() as tempdir: dir_name = tempdir self.assertTrue(os.path.exists(tempdir)) self.assertFalse(os.path.exists(dir_name))
def test_should_instantiate_first_time(self): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) build_graph1 = BuildGraph(str(build_dir.resolve())) build_graph1.clean_redundant_functions_and_update(True) build_graph2 = BuildGraph(str(build_dir.resolve())) self.assertEqual(build_graph1.get_build_definitions(), build_graph2.get_build_definitions())
def test_redundant_cached_should_be_clean(self): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) build_graph = BuildGraph(str(build_dir.resolve())) cache_dir = Path(temp_base_dir, ".aws-sam", "cache") cache_dir.mkdir(parents=True) redundant_cache_folder = Path(cache_dir, "redundant") redundant_cache_folder.mkdir(parents=True) cached_build_strategy = CachedBuildStrategy(build_graph, Mock(), temp_base_dir, build_dir, cache_dir, True) cached_build_strategy._clean_redundant_cached() self.assertTrue(not redundant_cache_folder.exists())
def test_should_read_existing_build_graph(self): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) build_graph_path = Path(build_dir.parent, "build.toml") build_graph_path.write_text(TestBuildGraph.BUILD_GRAPH_CONTENTS) build_graph = BuildGraph(str(build_dir)) for build_definition in build_graph.get_build_definitions(): self.assertEqual(build_definition.codeuri, TestBuildGraph.CODEURI) self.assertEqual(build_definition.runtime, TestBuildGraph.RUNTIME) self.assertEqual(build_definition.metadata, TestBuildGraph.METADATA)
def _clone_new_app_templates(self, shared_dir, expected_path): with osutils.mkdir_temp(ignore_errors=True) as tempdir: expected_temp_path = os.path.normpath(os.path.join(tempdir, self._repo_name)) try: LOG.info("\nCloning app templates from %s", self._repo_url) subprocess.check_output( [self._git_executable(), "clone", self._repo_url], cwd=tempdir, stderr=subprocess.STDOUT ) shutil.copytree(expected_temp_path, expected_path, ignore=shutil.ignore_patterns("*.git")) self.repo_path = expected_path except OSError as ex: LOG.warning("WARN: Can't clone app repo, git executable not found", exc_info=ex) except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): click.echo("WARN: Could not clone app template repo.")
def _build_function(self, function_name, codeuri, runtime, handler): """ Given the function information, this method will build the Lambda function. Depending on the configuration it will either build the function in process or by spinning up a Docker container. Parameters ---------- function_name : str Name or LogicalId of the function codeuri : str Path to where the code lives runtime : str AWS Lambda function runtime Returns ------- str Path to the location where built artifacts are available """ # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) config = get_workflow_config(runtime, code_dir, self._base_dir) # artifacts directory will be created by the builder artifacts_dir = str(pathlib.Path(self._build_dir, function_name)) with osutils.mkdir_temp() as scratch_dir: manifest_path = self._manifest_path_override or os.path.join( code_dir, config.manifest_name) # By default prefer to build in-process for speed build_method = self._build_function_in_process if self._container_manager: build_method = self._build_function_on_container options = ApplicationBuilder._get_build_options( config.language, handler) return build_method(config, code_dir, artifacts_dir, scratch_dir, manifest_path, runtime, options)
def _build_function(self, function_name, codeuri, runtime): config = _get_workflow_config(runtime) # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) # artifacts directory will be created by the builder artifacts_dir = str(pathlib.Path(self._build_dir, function_name)) with osutils.mkdir_temp() as scratch_dir: manifest_path = self._manifest_path_override or os.path.join( code_dir, config.manifest_name) # By default prefer to build in-process for speed build_method = self._build_function_in_process if self._container_manager: build_method = self._build_function_on_container return build_method(config, code_dir, artifacts_dir, scratch_dir, manifest_path, runtime)
def test_raises_on_cleanup_failure(self, rmdir_mock): rmdir_mock.side_effect = OSError("fail") with self.assertRaises(OSError): with osutils.mkdir_temp() as tempdir: self.assertTrue(os.path.exists(tempdir))
def test_must_return_temp_dir(self): with osutils.mkdir_temp() as tempdir: self.assertTrue(os.path.exists(tempdir))
def test_must_return_temp_dir(self): with osutils.mkdir_temp() as tempdir: self.assertTrue(os.path.exists(tempdir))
def test_handles_ignore_error_case(self, rmdir_mock): rmdir_mock.side_effect = OSError("fail") dir_name = None with osutils.mkdir_temp(ignore_errors=True) as tempdir: dir_name = tempdir self.assertTrue(os.path.exists(tempdir))
def _build_function( # pylint: disable=R1710 self, function_name: str, codeuri: str, packagetype: str, runtime: str, handler: Optional[str], artifact_dir: str, metadata: Optional[Dict] = None, container_env_vars: Optional[Dict] = None, ) -> str: """ Given the function information, this method will build the Lambda function. Depending on the configuration it will either build the function in process or by spinning up a Docker container. Parameters ---------- function_name : str Name or LogicalId of the function codeuri : str Path to where the code lives packagetype : str The package type, 'Zip' or 'Image', see samcli/lib/utils/packagetype.py runtime : str AWS Lambda function runtime handler : Optional[str] An optional string to specify which function the handler should be artifact_dir: str Path to where function will be build into metadata : dict AWS Lambda function metadata container_env_vars : Optional[Dict] An optional dictionary of environment variables to pass to the container. Returns ------- str Path to the location where built artifacts are available """ if packagetype == IMAGE: # pylint: disable=fixme # FIXME: _build_lambda_image assumes metadata is not None, we need to throw an exception here return self._build_lambda_image(function_name=function_name, metadata=metadata) # type: ignore if packagetype == ZIP: if runtime in self._deprecated_runtimes: message = ( f"WARNING: {runtime} is no longer supported by AWS Lambda, " "please update to a newer supported runtime. SAM CLI " f"will drop support for all deprecated runtimes {self._deprecated_runtimes} on May 1st. " "See issue: https://github.com/awslabs/aws-sam-cli/issues/1934 for more details." ) LOG.warning(self._colored.yellow(message)) # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) # Determine if there was a build workflow that was specified directly in the template. specified_build_workflow = metadata.get("BuildMethod", None) if metadata else None config = get_workflow_config( runtime, code_dir, self._base_dir, specified_workflow=specified_build_workflow) with osutils.mkdir_temp() as scratch_dir: manifest_path = self._manifest_path_override or os.path.join( code_dir, config.manifest_name) options = ApplicationBuilder._get_build_options( function_name, config.language, handler) # By default prefer to build in-process for speed build_method = self._build_function_in_process if self._container_manager: build_method = self._build_function_on_container return build_method( config, code_dir, artifact_dir, scratch_dir, manifest_path, runtime, options, container_env_vars, ) return build_method(config, code_dir, artifact_dir, scratch_dir, manifest_path, runtime, options) # pylint: disable=fixme # FIXME: we need to throw an exception here, packagetype could be something else return # type: ignore
def _build_layer( self, layer_name: str, codeuri: str, specified_workflow: str, compatible_runtimes: List[str], artifact_dir: str, container_env_vars: Optional[Dict] = None, ) -> str: """ Given the layer information, this method will build the Lambda layer. Depending on the configuration it will either build the function in process or by spinning up a Docker container. Parameters ---------- layer_name : str Name or LogicalId of the function codeuri : str Path to where the code lives specified_workflow : str The specified workflow compatible_runtimes : List[str] List of runtimes the layer build is compatible with artifact_dir : str Path to where layer will be build into. A subfolder will be created in this directory depending on the specified workflow. container_env_vars : Optional[Dict] An optional dictionary of environment variables to pass to the container. Returns ------- str Path to the location where built artifacts are available """ # Create the arguments to pass to the builder # Code is always relative to the given base directory. code_dir = str(pathlib.Path(self._base_dir, codeuri).resolve()) config = get_workflow_config(None, code_dir, self._base_dir, specified_workflow) subfolder = get_layer_subfolder(specified_workflow) # artifacts directory will be created by the builder artifact_subdir = str(pathlib.Path(artifact_dir, subfolder)) with osutils.mkdir_temp() as scratch_dir: manifest_path = self._manifest_path_override or os.path.join( code_dir, config.manifest_name) # By default prefer to build in-process for speed build_runtime = specified_workflow build_method = self._build_function_in_process if self._container_manager: build_method = self._build_function_on_container if config.language == "provided": LOG.warning( "For container layer build, first compatible runtime is chosen as build target for container." ) # Only set to this value if specified workflow is makefile # which will result in config language as provided build_runtime = compatible_runtimes[0] options = ApplicationBuilder._get_build_options( layer_name, config.language, None) build_method( config, code_dir, artifact_subdir, scratch_dir, manifest_path, build_runtime, options, container_env_vars, ) # Not including subfolder in return so that we copy subfolder, instead of copying artifacts inside it. return artifact_dir