def test_blocks_from_local_graph_module(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/modules/stacks'))
hcl_config_parser = Parser()
module, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
self.assertEqual(
len(
list(
filter(
lambda block: block.block_type == BlockType.RESOURCE
and block.name == 'aws_s3_bucket.inner_s3',
module.blocks))), 3)
self.assertEqual(
len(
list(
filter(
lambda block: block.block_type == BlockType.MODULE and
block.name == 'inner_module_call', module.blocks))), 3)
self.assertEqual(
len(
list(
filter(
lambda block: block.block_type == BlockType.MODULE and
block.name == 's3', module.blocks))), 3)
self.assertEqual(
len(
list(
filter(
lambda block: block.block_type == BlockType.MODULE and
block.name == 'sub-module', module.blocks))), 1)
def test_encryption_aws(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/encryption'))
hcl_config_parser = Parser()
module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
local_graph = LocalGraph(module, module_dependency_map)
local_graph._create_vertices()
local_graph.calculate_encryption_attribute()
all_attributes = [vertex.get_attribute_dict() for vertex in local_graph.vertices]
for attribute_dict in all_attributes:
[resource_type, resource_name] = decode_graph_property_value(
attribute_dict[CustomAttributes.ID]).split(".")
if resource_type in ENCRYPTION_BY_RESOURCE_TYPE:
is_encrypted = attribute_dict[CustomAttributes.ENCRYPTION]
details = attribute_dict[CustomAttributes.ENCRYPTION_DETAILS]
self.assertEqual(is_encrypted, EncryptionValues.ENCRYPTED.value if resource_name.startswith("encrypted")
else EncryptionValues.UNENCRYPTED.value, f'failed for "{resource_type}.{resource_name}"')
if is_encrypted == EncryptionValues.ENCRYPTED.value:
if 'kms_key_id' in attribute_dict or 'kms_master_key_id' in attribute_dict:
self.assertEqual(details, EncryptionTypes.KMS_VALUE.value, f'Bad encryption details for "{resource_type}.{resource_name}"')
else:
self.assertIn(details, [EncryptionTypes.AES256.value, EncryptionTypes.KMS_VALUE.value, EncryptionTypes.NODE_TO_NODE.value, EncryptionTypes.DEFAULT_KMS.value], f'Bad encryption details for "{resource_type}.{resource_name}"')
else:
self.assertEqual(details, "")
else:
self.assertIsNone(attribute_dict.get(CustomAttributes.ENCRYPTION))
self.assertIsNone(attribute_dict.get(CustomAttributes.ENCRYPTION_DETAILS))
def test_module_dependencies(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/modules/stacks'))
hcl_config_parser = Parser()
module, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
self.assertEqual(module.module_dependency_map[f'{resources_dir}/prod'],
[[]])
self.assertEqual(
module.module_dependency_map[f'{resources_dir}/stage'], [[]])
self.assertEqual(module.module_dependency_map[f'{resources_dir}/test'],
[[]])
self.assertEqual(
module.module_dependency_map[f'{resources_dir}/prod/sub-prod'],
[[f'{resources_dir}/prod/main.tf']])
expected_inner_modules = [[
f'{resources_dir}/prod/main.tf',
f'{resources_dir}/prod/sub-prod/main.tf'
], [f'{resources_dir}/stage/main.tf'],
[f'{resources_dir}/test/main.tf']]
self.assertEqual(
module.module_dependency_map[
f'{os.path.dirname(resources_dir)}/s3_inner_modules'],
expected_inner_modules)
self.assertEqual(
module.module_dependency_map[
f'{os.path.dirname(resources_dir)}/s3_inner_modules/inner'],
list(
map(
lambda dep_list: dep_list + [
f'{os.path.dirname(resources_dir)}/s3_inner_modules/main.tf'
], expected_inner_modules)))
def test_file_dir_parser_results_match(self):
parser = Parser()
current_dir = os.path.dirname(os.path.realpath(__file__))
file_path = current_dir + '/resources/parse_file_vs_dir/main.tf'
dir_path = current_dir + '/resources/parse_file_vs_dir'
tf_definitions_file = parser.parse_file(file_path)
_, tf_definitions_dir = parser.parse_hcl_module(dir_path, 'terraform')
self.assertDictEqual(tf_definitions_file, tf_definitions_dir.get(list(tf_definitions_dir.keys())[0]))
def test_creating_graph(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/encryption'))
hcl_config_parser = Parser()
module, _ = hcl_config_parser.parse_hcl_module(resources_dir, 'AWS')
local_graph = TerraformLocalGraph(module)
local_graph._create_vertices()
nxc = NetworkxConnector()
nxc.save_graph(local_graph)
def test_vertices_from_local_graph_module(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks'))
hcl_config_parser = Parser()
module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
local_graph = LocalGraph(module, module_dependency_map)
local_graph.build_graph(render_variables=True)
self.assertEqual(12, len(local_graph.edges))
def build_graph_from_source_directory(self, source_dir, render_variables=True, local_graph_class=LocalGraph,
parsing_errors=None):
parser = Parser()
module, module_dependency_map, tf_definitions = \
parser.parse_hcl_module(source_dir, self.source, parsing_errors)
local_graph = local_graph_class(module, module_dependency_map)
local_graph.build_graph(render_variables=render_variables)
return local_graph, tf_definitions
def test_vertices_from_local_graph(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME,
'../resources/variable_rendering/render_from_module_vpc'))
hcl_config_parser = Parser()
module, module_dependency_map, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
local_graph = LocalGraph(module, module_dependency_map)
local_graph._create_vertices()
tf_definitions, breadcrumbs = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir)
self.assertIsNotNone(tf_definitions)
self.assertIsNotNone(breadcrumbs)
def build_graph_from_source_directory(self, source_dir, render_variables=True, local_graph_class=LocalGraph,
parsing_errors=None, download_external_modules=False, excluded_paths: List[str]=None):
parser = Parser()
logging.info('Parsing HCL files in source dir')
module, module_dependency_map, tf_definitions = \
parser.parse_hcl_module(source_dir, self.source, download_external_modules, parsing_errors, excluded_paths=excluded_paths)
logging.info('Building graph from parsed module')
local_graph = local_graph_class(module, module_dependency_map)
local_graph.build_graph(render_variables=render_variables)
return local_graph, tf_definitions
def test_set_variables_values_from_modules(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME,
'../resources/variable_rendering/render_from_module_vpc'))
hcl_config_parser = Parser()
module, module_dependency_map, tf_definitions = hcl_config_parser.parse_hcl_module(resources_dir,
source=self.source)
local_graph = LocalGraph(module, module_dependency_map)
local_graph._create_vertices()
variables_before_module_definitions = {
"cidr": "0.0.0.0/0",
"private_subnets": [],
"public_subnets": [],
"enable_nat_gateway": False,
"single_nat_gateway": False,
"enable_dns_hostnames": False,
"public_subnet_tags": {},
"private_subnet_tags": {},
}
for var_name, var_value in variables_before_module_definitions.items():
vertex_index = local_graph.vertices_block_name_map[BlockType.VARIABLE].get(var_name)[0]
vertex = local_graph.vertices[vertex_index]
default_val = vertex.attributes['default']
if type(default_val) == list:
self.assertEqual(var_value, default_val[0])
else:
self.assertEqual(var_value, default_val)
local_graph.build_graph(resources_dir)
expected_variables_after = {
"cidr": "172.16.0.0/16",
"private_subnets": ["172.16.1.0/24", "172.16.2.0/24", "172.16.3.0/24"],
"public_subnets": ["172.16.4.0/24", "172.16.5.0/24", "172.16.6.0/24"],
"enable_nat_gateway": True,
"single_nat_gateway": True,
"enable_dns_hostnames": True,
"public_subnet_tags": {"kubernetes.io/cluster/${local.cluster_name}": "shared",
"kubernetes.io/role/elb": "1"},
"private_subnet_tags": {"kubernetes.io/cluster/${local.cluster_name}" : "shared",
"kubernetes.io/role/internal-elb": "1"}
}
for var_name, var_value in expected_variables_after.items():
vertex_index = local_graph.vertices_block_name_map[BlockType.VARIABLE].get(var_name)[0]
vertex = local_graph.vertices[vertex_index]
default_val = vertex.attributes['default']
if type(default_val) == list:
self.assertEqual(var_value, default_val[0])
else:
self.assertEqual(var_value, default_val)
def test_hcl_parsing_sorting(self):
source_dir = os.path.realpath(
os.path.join(
TEST_DIRNAME,
'../resources/tf_parsing_comparison/modifications_diff'))
config_parser = Parser()
_, tf_definitions = config_parser.parse_hcl_module(source_dir, 'AWS')
expected = [
'https://www.googleapis.com/auth/devstorage.read_only',
'https://www.googleapis.com/auth/logging.write',
'https://www.googleapis.com/auth/monitoring.write',
'https://www.googleapis.com/auth/service.management.readonly',
'https://www.googleapis.com/auth/servicecontrol',
'https://www.googleapis.com/auth/trace.append'
]
result_resource = tf_definitions[
source_dir + '/main.tf']['resource'][0]['google_compute_instance'][
'tfer--test3']['service_account'][0]['scopes'][0]
self.assertListEqual(result_resource, expected)
def test_variables_same_name_different_modules(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/modules/same_var_names'))
hcl_config_parser = Parser()
module, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
local_graph = TerraformLocalGraph(module)
local_graph.build_graph(render_variables=True)
print(local_graph.edges)
self.assertEqual(12, len(local_graph.edges))
self.assertEqual(13, len(local_graph.vertices))
module_variable_edges = [
e for e in local_graph.edges
if local_graph.vertices[e.dest].block_type == "module"
and local_graph.vertices[e.dest].path.endswith(
'same_var_names/module2/main.tf')
]
# Check they point to 2 different modules
self.assertEqual(2, len(module_variable_edges))
self.assertNotEqual(
local_graph.vertices[module_variable_edges[0].origin],
local_graph.vertices[module_variable_edges[1].origin])
module_variable_edges = [
e for e in local_graph.edges
if local_graph.vertices[e.dest].block_type == "module"
and local_graph.vertices[e.dest].path.endswith(
'same_var_names/module1/main.tf')
]
# Check they point to 2 different modules
self.assertEqual(2, len(module_variable_edges))
self.assertNotEqual(
local_graph.vertices[module_variable_edges[0].origin],
local_graph.vertices[module_variable_edges[1].origin])
def test_vertices_from_local_graph_module(self):
parent_dir = Path(TEST_DIRNAME).parent
resources_dir = str(parent_dir / "resources/modules/stacks")
hcl_config_parser = Parser()
module, _ = hcl_config_parser.parse_hcl_module(resources_dir,
self.source)
local_graph = TerraformLocalGraph(module)
local_graph.build_graph(render_variables=True)
self.assertEqual(12, len(local_graph.edges))
# check vertex breadcrumbs
bucket_vertex_1 = next(vertex for vertex in local_graph.vertices
if vertex.name == "aws_s3_bucket.inner_s3"
and vertex.source_module == {4})
bucket_vertex_2 = next(vertex for vertex in local_graph.vertices
if vertex.name == "aws_s3_bucket.inner_s3"
and vertex.source_module == {5})
bucket_vertex_3 = next(vertex for vertex in local_graph.vertices
if vertex.name == "aws_s3_bucket.inner_s3"
and vertex.source_module == {6})
self.assertDictEqual(
{
"versioning.enabled": [
{
"type":
"module",
"name":
"inner_module_call",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/main.tf"),
"module_connection":
False,
},
{
"type":
"variable",
"name":
"versioning",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/inner/variables.tf"
),
"module_connection":
False,
},
],
"source_module_": [
{
"type":
"module",
"name":
"sub-module",
"path":
str(parent_dir /
"resources/modules/stacks/prod/main.tf"),
},
{
"type":
"module",
"name":
"s3",
"path":
str(parent_dir /
"resources/modules/stacks/prod/sub-prod/main.tf"),
},
{
"type":
"module",
"name":
"inner_module_call",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/main.tf"),
},
],
},
bucket_vertex_1.breadcrumbs,
)
self.assertDictEqual(
{
"versioning.enabled": [
{
"type":
"module",
"name":
"inner_module_call",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/main.tf"),
"module_connection":
False,
},
{
"type":
"variable",
"name":
"versioning",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/inner/variables.tf"
),
"module_connection":
False,
},
],
"source_module_": [
{
"type":
"module",
"name":
"s3",
"path":
str(parent_dir /
"resources/modules/stacks/stage/main.tf"),
},
{
"type":
"module",
"name":
"inner_module_call",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/main.tf"),
},
],
},
bucket_vertex_2.breadcrumbs,
)
self.assertDictEqual(
{
"versioning.enabled": [
{
"type":
"module",
"name":
"inner_module_call",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/main.tf"),
"module_connection":
False,
},
{
"type":
"variable",
"name":
"versioning",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/inner/variables.tf"
),
"module_connection":
False,
},
],
"source_module_": [
{
"type":
"module",
"name":
"s3",
"path":
str(parent_dir /
"resources/modules/stacks/test/main.tf"),
},
{
"type":
"module",
"name":
"inner_module_call",
"path":
str(parent_dir /
"resources/modules/s3_inner_modules/main.tf"),
},
],
},
bucket_vertex_3.breadcrumbs,
)
