def test_terragoat_db_app(self):
resources_dir = os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_terragoat_db_app')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
self.compare_vertex_attributes(local_graph, expected_terragoat_local_resource_prefix, BlockType.LOCALS.value, 'resource_prefix')
self.compare_vertex_attributes(local_graph, expected_terragoat_db_instance, BlockType.RESOURCE.value, "aws_db_instance.default")
def test_update_vertices_configs_deep_nesting(self):
resources_dir = os.path.join(
TEST_DIRNAME,
'../resources/variable_rendering/render_deep_nesting')
graph_manager = GraphManager(NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=True)
expected_config = {
'aws_s3_bucket': {
'default': {
'server_side_encryption_configuration': [{
'rule': [{
'apply_server_side_encryption_by_default': [{
'sse_algorithm': ['AES256'],
'kms_master_key_id': ['']
}]
}]
}]
}
}
}
actual_config = local_graph.vertices[
local_graph.vertices_by_block_type.get(
BlockType.RESOURCE)[0]].config
self.assertDictEqual(expected_config, actual_config)
print('')
def test_multiple_breadcrumbs(self):
resources_dir = os.path.join(TEST_DIRNAME,
'../resources/general_example')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=True)
vertices = local_graph.vertices
s3_vertex = list(
filter(lambda vertex: vertex.block_type == BlockType.RESOURCE,
vertices))[0]
changed_attributes = list(s3_vertex.changed_attributes.keys())
self.assertListEqual(changed_attributes, ['region', 'bucket'])
bucket_vertices_ids_list = s3_vertex.changed_attributes.get('bucket')
self.assertEqual(2, len(bucket_vertices_ids_list))
self.assertEqual(BlockType.VARIABLE,
vertices[bucket_vertices_ids_list[0]].block_type)
self.assertEqual('bucket_name',
vertices[bucket_vertices_ids_list[0]].name)
self.assertEqual(vertices[bucket_vertices_ids_list[0]].name,
s3_vertex.breadcrumbs['bucket'][0]['name'])
self.assertEqual(BlockType.LOCALS,
vertices[bucket_vertices_ids_list[1]].block_type)
self.assertEqual('bucket_name',
vertices[bucket_vertices_ids_list[1]].name)
self.assertEqual(vertices[bucket_vertices_ids_list[1]].name,
s3_vertex.breadcrumbs['bucket'][1]['name'])
def test_graph_rendering_order(self):
resource_path = os.path.join(TEST_DIRNAME, "..", "resources",
"module_rendering", "example")
graph_manager = GraphManager('m', ['m'])
local_graph, tf_def = graph_manager.build_graph_from_source_directory(
resource_path, render_variables=True)
module_vertices = list(
filter(lambda v: v.block_type == BlockType.MODULE,
local_graph.vertices))
existing = set()
self.assertEqual(6, len(local_graph.edges))
for e in local_graph.edges:
if e in existing:
self.fail(
"No 2 edges should be aimed at the same vertex in this example"
)
else:
existing.add(e)
count = 0
found = 0
for v in module_vertices:
if v.name == 'second-mock':
found += 1
if v.attributes['input'] == ['aws_s3_bucket.some-bucket.arn']:
count += 1
self.assertEqual(
found, count,
f"Expected all instances to have the same value, found {found} instances but only {count} correct values"
)
def test_build_graph_with_linked_registry_modules(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/modules/registry_security_group_inner_module'))
graph_manager = GraphManager(NetworkxConnector())
local_graph, tf_definitions = graph_manager.build_graph_from_source_directory(resources_dir,
render_variables=True,
download_external_modules=True)
outputs_vpcs = self.get_vertex_by_name_and_type(local_graph, BlockType.OUTPUT, 'security_group_vpc_id',
multiple=True)
resource_flow_log = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'aws_flow_log.related_flow_log')
resource_security_group_this = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'aws_security_group.this')
resource_security_group_this_name_prefix = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'aws_security_group.this_name_prefix')
output_this_security_group_vpc_id_inner = [o for o in outputs_vpcs if 'http-80' in o.path][0]
output_this_security_group_vpc_id_outer = [o for o in outputs_vpcs if 'http-80' not in o.path][0]
self.check_edge(local_graph, node_from=resource_flow_log, node_to=output_this_security_group_vpc_id_inner,
expected_label='vpc_id')
self.check_edge(local_graph, node_from=output_this_security_group_vpc_id_inner,
node_to=output_this_security_group_vpc_id_outer, expected_label='value')
self.check_edge(local_graph, node_from=output_this_security_group_vpc_id_outer,
node_to=resource_security_group_this, expected_label='value')
self.check_edge(local_graph, node_from=output_this_security_group_vpc_id_outer,
node_to=resource_security_group_this_name_prefix, expected_label='value')
# cleanup
if os.path.exists(os.path.join(resources_dir, external_modules_download_path)):
shutil.rmtree(os.path.join(resources_dir, external_modules_download_path))
def test_general_example(self):
resources_dir = os.path.join(TEST_DIRNAME,
'../resources/general_example')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=True)
expected_provider = {
'profile': 'default',
'region': 'us-east-1',
'alias': 'east1'
}
expected_local = {'bucket_name': {'val': 'MyBucket'}}
expected_resource = {
'region': 'us-west-2',
'bucket': expected_local['bucket_name']
}
self.compare_vertex_attributes(local_graph, expected_provider,
BlockType.PROVIDER.value, 'aws.east1')
self.compare_vertex_attributes(local_graph, expected_local,
BlockType.LOCALS.value, 'bucket_name')
self.compare_vertex_attributes(local_graph, expected_resource,
BlockType.RESOURCE.value,
'aws_s3_bucket.template_bucket')
def test_render_variable(self):
resources_dir = os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_variable')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
expected_resource = {'region': "us-west-2", 'bucket': "test_bucket_name", "acl": "acl", "force_destroy": True}
self.compare_vertex_attributes(local_graph, expected_resource, BlockType.RESOURCE.value, 'aws_s3_bucket.template_bucket')
def test_render_lambda(self):
resources_dir = os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_lambda')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
expected_aws_lambda_permission = {'count': 0, 'statement_id': 'test_statement_id', 'action': 'lambda:InvokeFunction', 'function_name': 'my-func', 'principal': 'dumbeldor', 'resource_type': 'aws_lambda_permission'}
self.compare_vertex_attributes(local_graph, expected_aws_lambda_permission, BlockType.RESOURCE.value, "aws_lambda_permission.test_lambda_permissions")
def test_render_local_from_variable(self):
resources_dir = os.path.join(TEST_DIRNAME,
'../resources/variable_rendering/render_local_from_variable')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
expected_local = {'bucket_name': 'test_bucket_name'}
self.compare_vertex_attributes(local_graph, expected_local, BlockType.LOCALS.value, 'bucket_name')
def test_render_nested_modules(self):
resources_dir = os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_nested_modules')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
expected_aws_instance = {"instance_type": "bar"}
self.compare_vertex_attributes(local_graph, expected_aws_instance, BlockType.RESOURCE, "aws_instance.example")
expected_output_bucket_acl = {"value": "z"}
self.compare_vertex_attributes(local_graph, expected_output_bucket_acl, BlockType.OUTPUT, "bucket_acl")
def test_render_local(self):
resources_dir = os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_local')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
expected_local = {'bucket_name': 'test_bucket_name'}
expected_resource = {'region': 'us-west-2', 'bucket': expected_local['bucket_name']}
self.compare_vertex_attributes(local_graph, expected_local, BlockType.LOCALS.value, 'bucket_name')
self.compare_vertex_attributes(local_graph, expected_resource, BlockType.RESOURCE.value, 'aws_s3_bucket.template_bucket')
def test_dict_tfvar(self):
resources_dir = os.path.join(TEST_DIRNAME, '../resources/variable_rendering/render_dictionary_tfvars')
graph_manager = GraphManager('d', ['d'])
local_graph, tf_def = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
for v in local_graph.vertices:
expected_v = expected_provider.get(v.block_type, {}).get(v.name)
if expected_v:
for attribute_key, expected_value in expected_v.items():
actual_value = v.attributes.get(attribute_key)
self.assertEqual(expected_value, actual_value,
f'error during comparing {v.block_type} in attribute key: {attribute_key}')
def test_build_graph_with_linked_modules(self):
# see the image to view the expected graph in tests/resources/modules/linked_modules/expected_graph.png
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/modules/linked_modules'))
graph_manager = GraphManager(NetworkxConnector())
local_graph, tf_definitions = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=False)
vertices_by_block_type = local_graph.vertices_by_block_type
expected_vertices_num_by_type = {
BlockType.VARIABLE: 5,
BlockType.RESOURCE: 5,
BlockType.OUTPUT: 3,
BlockType.MODULE: 2,
BlockType.DATA: 1,
}
for block_type, count in expected_vertices_num_by_type.items():
self.assertEqual(count, len(vertices_by_block_type[block_type]))
output_this_lambda_func_arn = self.get_vertex_by_name_and_type(
local_graph, BlockType.OUTPUT, 'this_lambda_function_arn')
output_this_lambda_func_name = self.get_vertex_by_name_and_type(
local_graph, BlockType.OUTPUT, 'this_lambda_function_name')
output_this_s3_bucket_id = self.get_vertex_by_name_and_type(
local_graph, BlockType.OUTPUT, 'this_s3_bucket_id')
resource_aws_lambda_function = self.get_vertex_by_name_and_type(
local_graph, BlockType.RESOURCE, 'aws_lambda_function.this')
resource_aws_s3_bucket_policy = self.get_vertex_by_name_and_type(
local_graph, BlockType.RESOURCE, 'aws_s3_bucket_policy.this')
resource_aws_s3_bucket = self.get_vertex_by_name_and_type(
local_graph, BlockType.RESOURCE, 'aws_s3_bucket.this')
self.check_edge(local_graph,
node_from=output_this_lambda_func_arn,
node_to=resource_aws_lambda_function,
expected_label='value')
self.check_edge(local_graph,
node_from=output_this_lambda_func_name,
node_to=resource_aws_lambda_function,
expected_label='value')
self.check_edge(local_graph,
node_from=output_this_s3_bucket_id,
node_to=resource_aws_s3_bucket_policy,
expected_label='value')
self.check_edge(local_graph,
node_from=output_this_s3_bucket_id,
node_to=resource_aws_s3_bucket,
expected_label='value')
def test_blocks_from_local_graph_module(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks'))
graph_manager = GraphManager(NetworkxConnector())
local_graph, tf = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
tf, _ = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir)
found_results = 0
for key, value in tf.items():
if key.startswith(os.path.join(os.path.dirname(resources_dir), 's3_inner_modules', 'inner', 'main.tf')):
conf = value['resource'][0]['aws_s3_bucket']['inner_s3']
if 'stage/main' in key or 'prod/main' in key:
self.assertTrue(conf['versioning'][0]['enabled'][0])
found_results += 1
elif 'test/main' in key:
self.assertFalse(conf['versioning'][0]['enabled'][0])
found_results += 1
self.assertEqual(found_results, 3)
def test_single_edge_with_same_label(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/k8_service'))
graph_manager = GraphManager(NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir,
render_variables=True)
edges_hash = []
for e in local_graph.edges:
edge_hash = calculate_hash({"origin": e.origin, "dest": e.dest, "label": e.label})
if edge_hash in edges_hash:
origin = local_graph.vertices[e.origin]
dest = local_graph.vertices[e.dest]
self.fail(f'edge {e} == [{origin} - {e.label} -> {dest}] appears more than once in the graph')
else:
edges_hash.append(edge_hash)
def test_build_graph(self):
resources_dir = os.path.join(TEST_DIRNAME,
'../resources/general_example')
graph_manager = GraphManager(db_connector=NetworkxConnector())
graph, tf_definitions = graph_manager.build_graph_from_source_directory(
resources_dir)
expected_num_of_var_nodes = 3
expected_num_of_locals_nodes = 1
expected_num_of_resources_nodes = 1
expected_num_of_provider_nodes = 1
vertices_by_block_type = graph.vertices_by_block_type
self.assertEqual(expected_num_of_var_nodes,
len(vertices_by_block_type[BlockType.VARIABLE]))
self.assertEqual(expected_num_of_locals_nodes,
len(vertices_by_block_type[BlockType.LOCALS]))
self.assertEqual(expected_num_of_resources_nodes,
len(vertices_by_block_type[BlockType.RESOURCE]))
self.assertEqual(expected_num_of_provider_nodes,
len(vertices_by_block_type[BlockType.PROVIDER]))
provider_node = graph.vertices[vertices_by_block_type[
BlockType.PROVIDER][0]]
resource_node = graph.vertices[vertices_by_block_type[
BlockType.RESOURCE][0]]
local_node = graph.vertices[vertices_by_block_type[BlockType.LOCALS]
[0]]
var_bucket_name_node = None
var_region_node = None
var_aws_profile_node = None
for index in vertices_by_block_type[BlockType.VARIABLE]:
var_node = graph.vertices[index]
if var_node.name == 'aws_profile':
var_aws_profile_node = var_node
if var_node.name == 'bucket_name':
var_bucket_name_node = var_node
if var_node.name == 'region':
var_region_node = var_node
self.check_edge(graph, resource_node, local_node, 'bucket')
self.check_edge(graph, resource_node, provider_node, 'provider')
self.check_edge(graph, resource_node, var_region_node, 'region')
self.check_edge(graph, provider_node, var_aws_profile_node, 'profile')
self.check_edge(graph, local_node, var_bucket_name_node, 'bucket_name')
def test_breadcrumbs(self):
resources_dir = os.path.join(TEST_DIRNAME, '../resources/s3_bucket')
graph_manager = GraphManager('acme', ['acme'])
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
vertices = local_graph.vertices
s3_vertex = list(filter(lambda vertex: vertex.block_type == BlockType.RESOURCE, vertices))[0]
changed_attributes = list(s3_vertex.changed_attributes.keys())
self.assertListEqual(changed_attributes, ['versioning.enabled', 'acl'])
for breadcrumbs in s3_vertex.changed_attributes.values():
self.assertEqual(1, len(breadcrumbs))
acl_origin_vertex = s3_vertex.changed_attributes.get('acl')[0]
matching_acl_vertex = vertices[acl_origin_vertex]
self.assertEqual('acl', matching_acl_vertex.name)
versioning_origin_vertex = s3_vertex.changed_attributes.get('versioning.enabled')[0]
matching_versioning_vertex = vertices[versioning_origin_vertex]
self.assertEqual('is_enabled', matching_versioning_vertex.name)
def __init__(self, parser=Parser(), db_connector=NetworkxConnector(), external_registries=None,
source="Terraform", graph_class=LocalGraph, graph_manager=None):
self.external_registries = [] if external_registries is None else external_registries
self.graph_class = graph_class
self.parser = parser
self.tf_definitions = None
self.definitions_context = None
self.breadcrumbs = None
self.definitions_context = {}
self.evaluations_context: Dict[str, Dict[str, EvaluationContext]] = {}
self.graph_manager = graph_manager if graph_manager is not None else GraphManager(source=source,
db_connector=db_connector)
def test_build_graph_with_deep_nested_edges(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/k8_service'))
graph_manager = GraphManager(NetworkxConnector())
local_graph, tf = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
resource_kubernetes_deployment = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'kubernetes_deployment.bazel_remote_cache')
locals_name = self.get_vertex_by_name_and_type(local_graph, BlockType.LOCALS, 'name')
locals_labels = self.get_vertex_by_name_and_type(local_graph, BlockType.LOCALS, 'labels')
self.check_edge(local_graph, node_from=locals_labels, node_to=locals_name,
expected_label="labels.app.kubernetes.io/name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="metadata.name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="spec.template.metadata.name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="spec.template.spec.container.name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="spec.template.spec.volume.1.config_map.name")
def go(self, dir_name, different_expected=None, replace_expected=False):
os.environ['RENDER_VARIABLES_ASYNC'] = 'False'
os.environ['LOG_LEVEL'] = 'INFO'
different_expected = {} if not different_expected else different_expected
resources_dir = os.path.realpath(
os.path.join(
TEST_DIRNAME,
'../../../terraform/parser/resources/parser_scenarios',
dir_name))
graph_manager = GraphManager(dir_name, [dir_name])
local_graph, _ = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=True)
got_tf_definitions, _ = convert_graph_vertices_to_tf_definitions(
local_graph.vertices, resources_dir)
expected = load_expected(replace_expected, dir_name, resources_dir)
for expected_file, expected_block_type_dict in expected.items():
module_removed_path = expected_file
got_file = got_tf_definitions.get(module_removed_path)
self.assertIsNotNone(got_file)
for expected_block_type, expected_block_type_list in expected_block_type_dict.items(
):
got_block_type_list = got_file.get(expected_block_type)
self.assertIsNotNone(got_block_type_list)
for expected_block_dict in expected_block_type_list:
for expected_block_name, expected_block_val in expected_block_dict.items(
):
if expected_block_type != BlockType.RESOURCE:
found = self.match_blocks(expected_block_val,
different_expected,
got_block_type_list,
expected_block_name)
else:
found = self.match_resources(
expected_block_val, different_expected,
got_block_type_list, expected_block_name)
self.assertTrue(
found,
f"expected to find block {expected_block_dict} from file {expected_file} in graph"
)