def test_blocks_from_local_graph_module(self): resources_dir = os.path.realpath( os.path.join(TEST_DIRNAME, '../resources/modules/stacks')) hcl_config_parser = Parser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) self.assertEqual( len( list( filter( lambda block: block.block_type == BlockType.RESOURCE and block.name == 'aws_s3_bucket.inner_s3', module.blocks))), 3) self.assertEqual( len( list( filter( lambda block: block.block_type == BlockType.MODULE and block.name == 'inner_module_call', module.blocks))), 3) self.assertEqual( len( list( filter( lambda block: block.block_type == BlockType.MODULE and block.name == 's3', module.blocks))), 3) self.assertEqual( len( list( filter( lambda block: block.block_type == BlockType.MODULE and block.name == 'sub-module', module.blocks))), 1)
def test_encryption_aws(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/encryption')) hcl_config_parser = Parser() module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = LocalGraph(module, module_dependency_map) local_graph._create_vertices() local_graph.calculate_encryption_attribute() all_attributes = [vertex.get_attribute_dict() for vertex in local_graph.vertices] for attribute_dict in all_attributes: [resource_type, resource_name] = decode_graph_property_value( attribute_dict[CustomAttributes.ID]).split(".") if resource_type in ENCRYPTION_BY_RESOURCE_TYPE: is_encrypted = attribute_dict[CustomAttributes.ENCRYPTION] details = attribute_dict[CustomAttributes.ENCRYPTION_DETAILS] self.assertEqual(is_encrypted, EncryptionValues.ENCRYPTED.value if resource_name.startswith("encrypted") else EncryptionValues.UNENCRYPTED.value, f'failed for "{resource_type}.{resource_name}"') if is_encrypted == EncryptionValues.ENCRYPTED.value: if 'kms_key_id' in attribute_dict or 'kms_master_key_id' in attribute_dict: self.assertEqual(details, EncryptionTypes.KMS_VALUE.value, f'Bad encryption details for "{resource_type}.{resource_name}"') else: self.assertIn(details, [EncryptionTypes.AES256.value, EncryptionTypes.KMS_VALUE.value, EncryptionTypes.NODE_TO_NODE.value, EncryptionTypes.DEFAULT_KMS.value], f'Bad encryption details for "{resource_type}.{resource_name}"') else: self.assertEqual(details, "") else: self.assertIsNone(attribute_dict.get(CustomAttributes.ENCRYPTION)) self.assertIsNone(attribute_dict.get(CustomAttributes.ENCRYPTION_DETAILS))
def test_module_dependencies(self): resources_dir = os.path.realpath( os.path.join(TEST_DIRNAME, '../resources/modules/stacks')) hcl_config_parser = Parser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) self.assertEqual(module.module_dependency_map[f'{resources_dir}/prod'], [[]]) self.assertEqual( module.module_dependency_map[f'{resources_dir}/stage'], [[]]) self.assertEqual(module.module_dependency_map[f'{resources_dir}/test'], [[]]) self.assertEqual( module.module_dependency_map[f'{resources_dir}/prod/sub-prod'], [[f'{resources_dir}/prod/main.tf']]) expected_inner_modules = [[ f'{resources_dir}/prod/main.tf', f'{resources_dir}/prod/sub-prod/main.tf' ], [f'{resources_dir}/stage/main.tf'], [f'{resources_dir}/test/main.tf']] self.assertEqual( module.module_dependency_map[ f'{os.path.dirname(resources_dir)}/s3_inner_modules'], expected_inner_modules) self.assertEqual( module.module_dependency_map[ f'{os.path.dirname(resources_dir)}/s3_inner_modules/inner'], list( map( lambda dep_list: dep_list + [ f'{os.path.dirname(resources_dir)}/s3_inner_modules/main.tf' ], expected_inner_modules)))
def test_file_dir_parser_results_match(self): parser = Parser() current_dir = os.path.dirname(os.path.realpath(__file__)) file_path = current_dir + '/resources/parse_file_vs_dir/main.tf' dir_path = current_dir + '/resources/parse_file_vs_dir' tf_definitions_file = parser.parse_file(file_path) _, tf_definitions_dir = parser.parse_hcl_module(dir_path, 'terraform') self.assertDictEqual(tf_definitions_file, tf_definitions_dir.get(list(tf_definitions_dir.keys())[0]))
def test_creating_graph(self): resources_dir = os.path.realpath( os.path.join(TEST_DIRNAME, '../resources/encryption')) hcl_config_parser = Parser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, 'AWS') local_graph = TerraformLocalGraph(module) local_graph._create_vertices() nxc = NetworkxConnector() nxc.save_graph(local_graph)
def test_vertices_from_local_graph_module(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks')) hcl_config_parser = Parser() module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = LocalGraph(module, module_dependency_map) local_graph.build_graph(render_variables=True) self.assertEqual(12, len(local_graph.edges))
def build_graph_from_source_directory(self, source_dir, render_variables=True, local_graph_class=LocalGraph, parsing_errors=None): parser = Parser() module, module_dependency_map, tf_definitions = \ parser.parse_hcl_module(source_dir, self.source, parsing_errors) local_graph = local_graph_class(module, module_dependency_map) local_graph.build_graph(render_variables=render_variables) return local_graph, tf_definitions
def test_vertices_from_local_graph(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_from_module_vpc')) hcl_config_parser = Parser() module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = LocalGraph(module, module_dependency_map) local_graph._create_vertices() tf_definitions, breadcrumbs = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir) self.assertIsNotNone(tf_definitions) self.assertIsNotNone(breadcrumbs)
def build_graph_from_source_directory(self, source_dir, render_variables=True, local_graph_class=LocalGraph, parsing_errors=None, download_external_modules=False, excluded_paths: List[str]=None): parser = Parser() logging.info('Parsing HCL files in source dir') module, module_dependency_map, tf_definitions = \ parser.parse_hcl_module(source_dir, self.source, download_external_modules, parsing_errors, excluded_paths=excluded_paths) logging.info('Building graph from parsed module') local_graph = local_graph_class(module, module_dependency_map) local_graph.build_graph(render_variables=render_variables) return local_graph, tf_definitions
def test_set_variables_values_from_modules(self): resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_from_module_vpc')) hcl_config_parser = Parser() module, module_dependency_map, tf_definitions = hcl_config_parser.parse_hcl_module(resources_dir, source=self.source) local_graph = LocalGraph(module, module_dependency_map) local_graph._create_vertices() variables_before_module_definitions = { "cidr": "0.0.0.0/0", "private_subnets": [], "public_subnets": [], "enable_nat_gateway": False, "single_nat_gateway": False, "enable_dns_hostnames": False, "public_subnet_tags": {}, "private_subnet_tags": {}, } for var_name, var_value in variables_before_module_definitions.items(): vertex_index = local_graph.vertices_block_name_map[BlockType.VARIABLE].get(var_name)[0] vertex = local_graph.vertices[vertex_index] default_val = vertex.attributes['default'] if type(default_val) == list: self.assertEqual(var_value, default_val[0]) else: self.assertEqual(var_value, default_val) local_graph.build_graph(resources_dir) expected_variables_after = { "cidr": "172.16.0.0/16", "private_subnets": ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"], "public_subnets": ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"], "enable_nat_gateway": True, "single_nat_gateway": True, "enable_dns_hostnames": True, "public_subnet_tags": {"kubernetes.io/cluster/${local.cluster_name}": "shared", "kubernetes.io/role/elb": "1"}, "private_subnet_tags": {"kubernetes.io/cluster/${local.cluster_name}" : "shared", "kubernetes.io/role/internal-elb": "1"} } for var_name, var_value in expected_variables_after.items(): vertex_index = local_graph.vertices_block_name_map[BlockType.VARIABLE].get(var_name)[0] vertex = local_graph.vertices[vertex_index] default_val = vertex.attributes['default'] if type(default_val) == list: self.assertEqual(var_value, default_val[0]) else: self.assertEqual(var_value, default_val)
def test_hcl_parsing_sorting(self): source_dir = os.path.realpath( os.path.join( TEST_DIRNAME, '../resources/tf_parsing_comparison/modifications_diff')) config_parser = Parser() _, tf_definitions = config_parser.parse_hcl_module(source_dir, 'AWS') expected = [ 'https://www.googleapis.com/auth/devstorage.read_only', 'https://www.googleapis.com/auth/logging.write', 'https://www.googleapis.com/auth/monitoring.write', 'https://www.googleapis.com/auth/service.management.readonly', 'https://www.googleapis.com/auth/servicecontrol', 'https://www.googleapis.com/auth/trace.append' ] result_resource = tf_definitions[ source_dir + '/main.tf']['resource'][0]['google_compute_instance'][ 'tfer--test3']['service_account'][0]['scopes'][0] self.assertListEqual(result_resource, expected)
def test_variables_same_name_different_modules(self): resources_dir = os.path.realpath( os.path.join(TEST_DIRNAME, '../resources/modules/same_var_names')) hcl_config_parser = Parser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = TerraformLocalGraph(module) local_graph.build_graph(render_variables=True) print(local_graph.edges) self.assertEqual(12, len(local_graph.edges)) self.assertEqual(13, len(local_graph.vertices)) module_variable_edges = [ e for e in local_graph.edges if local_graph.vertices[e.dest].block_type == "module" and local_graph.vertices[e.dest].path.endswith( 'same_var_names/module2/main.tf') ] # Check they point to 2 different modules self.assertEqual(2, len(module_variable_edges)) self.assertNotEqual( local_graph.vertices[module_variable_edges[0].origin], local_graph.vertices[module_variable_edges[1].origin]) module_variable_edges = [ e for e in local_graph.edges if local_graph.vertices[e.dest].block_type == "module" and local_graph.vertices[e.dest].path.endswith( 'same_var_names/module1/main.tf') ] # Check they point to 2 different modules self.assertEqual(2, len(module_variable_edges)) self.assertNotEqual( local_graph.vertices[module_variable_edges[0].origin], local_graph.vertices[module_variable_edges[1].origin])
def test_vertices_from_local_graph_module(self): parent_dir = Path(TEST_DIRNAME).parent resources_dir = str(parent_dir / "resources/modules/stacks") hcl_config_parser = Parser() module, _ = hcl_config_parser.parse_hcl_module(resources_dir, self.source) local_graph = TerraformLocalGraph(module) local_graph.build_graph(render_variables=True) self.assertEqual(12, len(local_graph.edges)) # check vertex breadcrumbs bucket_vertex_1 = next(vertex for vertex in local_graph.vertices if vertex.name == "aws_s3_bucket.inner_s3" and vertex.source_module == {4}) bucket_vertex_2 = next(vertex for vertex in local_graph.vertices if vertex.name == "aws_s3_bucket.inner_s3" and vertex.source_module == {5}) bucket_vertex_3 = next(vertex for vertex in local_graph.vertices if vertex.name == "aws_s3_bucket.inner_s3" and vertex.source_module == {6}) self.assertDictEqual( { "versioning.enabled": [ { "type": "module", "name": "inner_module_call", "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), "module_connection": False, }, { "type": "variable", "name": "versioning", "path": str(parent_dir / "resources/modules/s3_inner_modules/inner/variables.tf" ), "module_connection": False, }, ], "source_module_": [ { "type": "module", "name": "sub-module", "path": str(parent_dir / "resources/modules/stacks/prod/main.tf"), }, { "type": "module", "name": "s3", "path": str(parent_dir / "resources/modules/stacks/prod/sub-prod/main.tf"), }, { "type": "module", "name": "inner_module_call", "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), }, ], }, bucket_vertex_1.breadcrumbs, ) self.assertDictEqual( { "versioning.enabled": [ { "type": "module", "name": "inner_module_call", "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), "module_connection": False, }, { "type": "variable", "name": "versioning", "path": str(parent_dir / "resources/modules/s3_inner_modules/inner/variables.tf" ), "module_connection": False, }, ], "source_module_": [ { "type": "module", "name": "s3", "path": str(parent_dir / "resources/modules/stacks/stage/main.tf"), }, { "type": "module", "name": "inner_module_call", "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), }, ], }, bucket_vertex_2.breadcrumbs, ) self.assertDictEqual( { "versioning.enabled": [ { "type": "module", "name": "inner_module_call", "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), "module_connection": False, }, { "type": "variable", "name": "versioning", "path": str(parent_dir / "resources/modules/s3_inner_modules/inner/variables.tf" ), "module_connection": False, }, ], "source_module_": [ { "type": "module", "name": "s3", "path": str(parent_dir / "resources/modules/stacks/test/main.tf"), }, { "type": "module", "name": "inner_module_call", "path": str(parent_dir / "resources/modules/s3_inner_modules/main.tf"), }, ], }, bucket_vertex_3.breadcrumbs, )