def setUp(self): # create a build graph with 2 function definitions and 2 layer definitions self.build_graph = BuildGraph("build_dir") self.function1_1 = Mock() self.function1_1.inlinecode = None self.function1_2 = Mock() self.function1_2.inlinecode = None self.function2 = Mock() self.function2.inlinecode = None self.function_build_definition1 = FunctionBuildDefinition( "runtime", "codeuri", ZIP, {}) self.function_build_definition1.functions = [ self.function1_1, self.function1_2 ] self.function_build_definition2 = FunctionBuildDefinition( "runtime2", "codeuri", ZIP, {}) self.function_build_definition1.functions = [self.function2] self.build_graph.put_function_build_definition( self.function_build_definition1, Mock(packagetype=ZIP)) self.build_graph.put_function_build_definition( self.function_build_definition2, Mock(packagetype=ZIP)) self.layer_build_definition1 = LayerBuildDefinition( "layer1", "codeuri", "build_method", []) self.layer_build_definition2 = LayerBuildDefinition( "layer2", "codeuri", "build_method", []) self.build_graph.put_layer_build_definition( self.layer_build_definition1, Mock()) self.build_graph.put_layer_build_definition( self.layer_build_definition2, Mock())
def test_functions_should_be_added_existing_build_graph(self): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) build_graph_path = Path(build_dir.parent, "build.toml") build_graph_path.write_text(TestBuildGraph.BUILD_GRAPH_CONTENTS) build_graph = BuildGraph(str(build_dir)) build_definition1 = FunctionBuildDefinition( TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, TestBuildGraph.METADATA, TestBuildGraph.SOURCE_MD5) function1 = generate_function(runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=TestBuildGraph.METADATA) build_graph.put_function_build_definition(build_definition1, function1) self.assertTrue(len(build_graph.get_function_build_definitions()), 1) for build_definition in build_graph.get_function_build_definitions( ): self.assertTrue(len(build_definition.functions), 1) self.assertTrue(build_definition.functions[0], function1) self.assertEqual(build_definition.uuid, TestBuildGraph.UUID) build_definition2 = FunctionBuildDefinition( "another_runtime", "another_codeuri", None, "another_source_md5") function2 = generate_function(name="another_function") build_graph.put_function_build_definition(build_definition2, function2) self.assertTrue(len(build_graph.get_function_build_definitions()), 2)
def test_build_single_function_definition_image_functions_with_same_metadata( self, mock_copy_tree): given_build_function = Mock() built_image = Mock() given_build_function.return_value = built_image given_build_layer = Mock() given_build_dir = "build_dir" default_build_strategy = DefaultBuildStrategy(self.build_graph, given_build_dir, given_build_function, given_build_layer) function1 = Mock() function1.name = "Function" function1.full_path = "Function" function1.packagetype = IMAGE function2 = Mock() function2.name = "Function2" function2.full_path = "Function2" function2.packagetype = IMAGE build_definition = FunctionBuildDefinition("3.7", "codeuri", IMAGE, {}) # since they have the same metadata, they are put into the same build_definition. build_definition.functions = [function1, function2] result = default_build_strategy.build_single_function_definition( build_definition) # both of the function name should show up in results self.assertEqual(result, { "Function": built_image, "Function2": built_image })
def test_different_runtime_codeuri_metadata_should_not_reflect_as_same_object( self, runtime1, codeuri1, metadata1, source_md5_1, runtime2, codeuri2, metadata2, source_md5_2): build_definition1 = FunctionBuildDefinition(runtime1, codeuri1, ZIP, metadata1, source_md5_1) build_definition2 = FunctionBuildDefinition(runtime2, codeuri2, ZIP, metadata2, source_md5_2) self.assertNotEqual(build_definition1, build_definition2)
def test_same_runtime_codeuri_metadata_should_reflect_as_same_object(self): build_definition1 = FunctionBuildDefinition("runtime", "codeuri", ZIP, {"key": "value"}, "source_md5") build_definition2 = FunctionBuildDefinition("runtime", "codeuri", ZIP, {"key": "value"}, "source_md5") self.assertEqual(build_definition1, build_definition2)
def test_different_env_vars_should_not_reflect_as_same_object(self): build_definition1 = FunctionBuildDefinition("runtime", "codeuri", ZIP, {"key": "value"}, "source_md5", {"env_vars": "value1"}) build_definition2 = FunctionBuildDefinition("runtime", "codeuri", ZIP, {"key": "value"}, "source_md5", {"env_vars": "value2"}) self.assertNotEqual(build_definition1, build_definition2)
def build_single_function_definition( self, build_definition: FunctionBuildDefinition) -> Dict[str, str]: """ Build the unique definition and then copy the artifact to the corresponding function folder """ function_build_results = {} LOG.info( "Building codeuri: %s runtime: %s metadata: %s functions: %s", build_definition.codeuri, build_definition.runtime, build_definition.metadata, [function.full_path for function in build_definition.functions], ) # build into one of the functions from this build definition single_full_path = build_definition.get_full_path() single_build_dir = build_definition.get_build_dir(self._build_dir) LOG.debug("Building to following folder %s", single_build_dir) # when a function is passed here, it is ZIP function, codeuri and runtime are not None result = self._build_function( build_definition.get_function_name(), build_definition.codeuri, # type: ignore build_definition.packagetype, build_definition.runtime, # type: ignore build_definition.get_handler_name(), single_build_dir, build_definition.metadata, build_definition.env_vars, ) function_build_results[single_full_path] = result # copy results to other functions if build_definition.packagetype == ZIP: for function in build_definition.functions: if function.full_path != single_full_path: # for zip function we need to copy over the artifacts # artifacts directory will be created by the builder artifacts_dir = function.get_build_dir(self._build_dir) LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir) osutils.copytree(single_build_dir, artifacts_dir) function_build_results[function.full_path] = artifacts_dir elif build_definition.packagetype == IMAGE: for function in build_definition.functions: if function.full_path != single_full_path: # for image function, we just need to copy the image tag function_build_results[function.full_path] = result return function_build_results
def test_function_build_definition_to_toml_table(self): build_definition = FunctionBuildDefinition("runtime", "codeuri", {"key": "value"}, "source_md5") build_definition.add_function(generate_function()) toml_table = _function_build_definition_to_toml_table(build_definition) self.assertEqual(toml_table[CODE_URI_FIELD], build_definition.codeuri) self.assertEqual(toml_table[RUNTIME_FIELD], build_definition.runtime) self.assertEqual(toml_table[METADATA_FIELD], build_definition.metadata) self.assertEqual(toml_table[FUNCTIONS_FIELD], [f.name for f in build_definition.functions]) self.assertEqual(toml_table[SOURCE_MD5_FIELD], build_definition.source_md5)
def _get_build_graph(self): """ Converts list of functions and layers into a build graph, where we can iterate on each unique build and trigger build :return: BuildGraph, which represents list of unique build definitions """ build_graph = BuildGraph(self._build_dir) functions = self._resources_to_build.functions layers = self._resources_to_build.layers for function in functions: function_build_details = FunctionBuildDefinition( function.runtime, function.codeuri, function.packagetype, function.metadata) build_graph.put_function_build_definition(function_build_details, function) for layer in layers: layer_build_details = LayerBuildDefinition( layer.name, layer.codeuri, layer.build_method, layer.compatible_runtimes) build_graph.put_layer_build_definition(layer_build_details, layer) build_graph.clean_redundant_definitions_and_update( not self._is_building_specific_resource) return build_graph
def test_should_instantiate_first_time_and_update(self): with osutils.mkdir_temp() as temp_base_dir: build_dir = Path(temp_base_dir, ".aws-sam", "build") build_dir.mkdir(parents=True) # create a build graph and persist it build_graph1 = BuildGraph(str(build_dir)) build_definition1 = FunctionBuildDefinition( TestBuildGraph.RUNTIME, TestBuildGraph.CODEURI, TestBuildGraph.ZIP, TestBuildGraph.METADATA, TestBuildGraph.SOURCE_MD5, ) function1 = generate_function(runtime=TestBuildGraph.RUNTIME, codeuri=TestBuildGraph.CODEURI, metadata=TestBuildGraph.METADATA) build_graph1.put_function_build_definition(build_definition1, function1) build_graph1.clean_redundant_definitions_and_update(True) # read previously persisted graph and compare build_graph2 = BuildGraph(str(build_dir)) self.assertEqual( len(build_graph1.get_function_build_definitions()), len(build_graph2.get_function_build_definitions())) self.assertEqual( list(build_graph1.get_function_build_definitions())[0], list(build_graph2.get_function_build_definitions())[0], )
def test_str_representation(self): build_definition = FunctionBuildDefinition("runtime", "codeuri", ZIP, None, "source_md5") self.assertEqual( str(build_definition), f"BuildDefinition(runtime, codeuri, Zip, source_md5, {build_definition.uuid}, {{}}, [])", )
def test_no_function_should_raise_exception(self): build_definition = FunctionBuildDefinition("runtime", "codeuri", ZIP, "metadata", "source_md5") self.assertRaises(InvalidBuildGraphException, build_definition.get_handler_name) self.assertRaises(InvalidBuildGraphException, build_definition.get_function_name)
def build_single_function_definition( self, build_definition: FunctionBuildDefinition) -> Dict[str, str]: """ Build the unique definition and then copy the artifact to the corresponding function folder """ function_build_results = {} LOG.info( "Building codeuri: %s runtime: %s metadata: %s functions: %s", build_definition.codeuri, build_definition.runtime, build_definition.metadata, [function.name for function in build_definition.functions], ) # build into one of the functions from this build definition single_function_name = build_definition.get_function_name() single_build_dir = str( pathlib.Path(self._build_dir, single_function_name)) LOG.debug("Building to following folder %s", single_build_dir) result = self._build_function( build_definition.get_function_name(), build_definition.codeuri, build_definition.packagetype, build_definition.runtime, build_definition.get_handler_name(), single_build_dir, build_definition.metadata, ) function_build_results[single_function_name] = result # copy results to other functions if build_definition.packagetype == ZIP: for function in build_definition.functions: if function.name is not single_function_name: # artifacts directory will be created by the builder artifacts_dir = str( pathlib.Path(self._build_dir, function.name)) LOG.debug("Copying artifacts from %s to %s", single_build_dir, artifacts_dir) osutils.copytree(single_build_dir, artifacts_dir) function_build_results[function.name] = artifacts_dir return function_build_results
def test_function_build_definition_to_toml_table(self): build_definition = FunctionBuildDefinition( "runtime", "codeuri", ZIP, {"key": "value"}, "source_md5", env_vars={"env_vars": "value1"}) build_definition.add_function(generate_function()) toml_table = _function_build_definition_to_toml_table(build_definition) self.assertEqual(toml_table[CODE_URI_FIELD], build_definition.codeuri) self.assertEqual(toml_table[PACKAGETYPE_FIELD], build_definition.packagetype) self.assertEqual(toml_table[RUNTIME_FIELD], build_definition.runtime) self.assertEqual(toml_table[METADATA_FIELD], build_definition.metadata) self.assertEqual(toml_table[FUNCTIONS_FIELD], [f.name for f in build_definition.functions]) self.assertEqual(toml_table[SOURCE_MD5_FIELD], build_definition.source_md5) self.assertEqual(toml_table[ENV_VARS_FIELD], build_definition.env_vars)
def test_single_function_should_return_function_and_handler_name(self): build_definition = FunctionBuildDefinition("runtime", "codeuri", ZIP, "metadata", "source_md5") build_definition.add_function(generate_function()) self.assertEqual(build_definition.get_handler_name(), "handler") self.assertEqual(build_definition.get_function_name(), "name")
def setUp(self): # create a build graph with 2 function definitions and 2 layer definitions self.build_graph = BuildGraph("build_dir") self.function1_1 = Mock() self.function1_1.inlinecode = None self.function1_1.get_build_dir = Mock() self.function1_1.full_path = Mock() self.function1_2 = Mock() self.function1_2.inlinecode = None self.function1_2.get_build_dir = Mock() self.function1_2.full_path = Mock() self.function2 = Mock() self.function2.inlinecode = None self.function2.get_build_dir = Mock() self.function2.full_path = Mock() self.function_build_definition1 = FunctionBuildDefinition( "runtime", "codeuri", ZIP, {}) self.function_build_definition2 = FunctionBuildDefinition( "runtime2", "codeuri", ZIP, {}) self.build_graph.put_function_build_definition( self.function_build_definition1, self.function1_1) self.build_graph.put_function_build_definition( self.function_build_definition1, self.function1_2) self.build_graph.put_function_build_definition( self.function_build_definition2, self.function2) self.layer1 = Mock() self.layer2 = Mock() self.layer_build_definition1 = LayerBuildDefinition( "layer1", "codeuri", "build_method", []) self.layer_build_definition2 = LayerBuildDefinition( "layer2", "codeuri", "build_method", []) self.build_graph.put_layer_build_definition( self.layer_build_definition1, self.layer1) self.build_graph.put_layer_build_definition( self.layer_build_definition2, self.layer2)
def build_single_function_definition( self, build_definition: FunctionBuildDefinition) -> Dict[str, str]: """ Builds single function definition with caching """ if build_definition.packagetype == IMAGE: return self._delegate_build_strategy.build_single_function_definition( build_definition) code_dir = str( pathlib.Path(self._base_dir, build_definition.codeuri).resolve()) source_md5 = dir_checksum(code_dir) cache_function_dir = pathlib.Path(self._cache_dir, build_definition.uuid) function_build_results = {} if not cache_function_dir.exists( ) or build_definition.source_md5 != source_md5: LOG.info( "Cache is invalid, running build and copying resources to function build definition of %s", build_definition.uuid, ) build_result = self._delegate_build_strategy.build_single_function_definition( build_definition) function_build_results.update(build_result) if cache_function_dir.exists(): shutil.rmtree(str(cache_function_dir)) build_definition.source_md5 = source_md5 # Since all the build contents are same for a build definition, just copy any one of them into the cache for _, value in build_result.items(): osutils.copytree(value, cache_function_dir) break else: LOG.info( "Valid cache found, copying previously built resources from function build definition of %s", build_definition.uuid, ) for function in build_definition.functions: # artifacts directory will be created by the builder artifacts_dir = str( pathlib.Path(self._build_dir, function.name)) LOG.debug("Copying artifacts from %s to %s", cache_function_dir, artifacts_dir) osutils.copytree(cache_function_dir, artifacts_dir) function_build_results[function.name] = artifacts_dir return function_build_results
def _get_build_graph(self, inline_env_vars: Optional[Dict] = None, env_vars_file: Optional[str] = None) -> BuildGraph: """ Converts list of functions and layers into a build graph, where we can iterate on each unique build and trigger build :return: BuildGraph, which represents list of unique build definitions """ build_graph = BuildGraph(self._build_dir) functions = self._resources_to_build.functions layers = self._resources_to_build.layers file_env_vars = {} if env_vars_file: try: with open(env_vars_file, "r", encoding="utf-8") as fp: file_env_vars = json.load(fp) except Exception as ex: raise IOError( "Could not read environment variables overrides from file {}: {}" .format(env_vars_file, str(ex))) from ex for function in functions: container_env_vars = self._make_env_vars(function, file_env_vars, inline_env_vars) function_build_details = FunctionBuildDefinition( function.runtime, function.codeuri, function.packagetype, function.metadata, env_vars=container_env_vars) build_graph.put_function_build_definition(function_build_details, function) for layer in layers: container_env_vars = self._make_env_vars(layer, file_env_vars, inline_env_vars) layer_build_details = LayerBuildDefinition( layer.name, layer.codeuri, layer.build_method, layer.compatible_runtimes, env_vars=container_env_vars) build_graph.put_layer_build_definition(layer_build_details, layer) build_graph.clean_redundant_definitions_and_update( not self._is_building_specific_resource) return build_graph
def test_euqality_with_another_object(self): build_definition = FunctionBuildDefinition("runtime", "codeuri", ZIP, None, "source_md5") self.assertNotEqual(build_definition, {})