def test_creating_graph(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/encryption'))
hcl_config_parser = Parser()
module, _ = hcl_config_parser.parse_hcl_module(resources_dir, 'AWS')
local_graph = TerraformLocalGraph(module)
local_graph._create_vertices()
nxc = NetworkxConnector()
nxc.save_graph(local_graph)
def __init__(self, args):
graph_manager = CloudformationGraphManager(
db_connector=NetworkxConnector())
super().__init__(graph_manager,
"checkov/cloudformation/checks/graph_checks",
os.path.dirname(__file__) + "/test_checks",
"cloudformation", __file__, args)
def valiate_render_if(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
ec2instance_expected_attributes = {'InstanceType': 'm1.large'}
s3bucketsuspended_expected_attributes = {'VersioningConfiguration.Status': 'Suspended'}
s3bucketenabled_expected_attributes = {'VersioningConfiguration.Status': 'Enabled'}
self.compare_vertex_attributes(local_graph, ec2instance_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::Instance.EC2Instance')
self.compare_vertex_attributes(local_graph, s3bucketsuspended_expected_attributes, BlockType.RESOURCE, 'AWS::S3::Bucket.S3BucketSuspended')
self.compare_vertex_attributes(local_graph, s3bucketenabled_expected_attributes, BlockType.RESOURCE, 'AWS::S3::Bucket.S3BucketEnabled')
instancesize_breadcrumb = {'type': BlockType.PARAMETERS, 'name': 'InstanceSize', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}
ec2instance_expected_breadcrumbs = {
'InstanceType.Fn::If.2.Fn::If.1': [instancesize_breadcrumb],
'InstanceType.Fn::If.2.Fn::If': [instancesize_breadcrumb],
'InstanceType.Fn::If.2': [instancesize_breadcrumb],
'InstanceType.Fn::If': [instancesize_breadcrumb],
'InstanceType': [instancesize_breadcrumb],
}
s3bucketsuspended_expected_breadcrumbs = {}
s3bucketenabled_expected_breadcrumbs = {}
self.compare_vertex_breadcrumbs(local_graph, ec2instance_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::Instance.EC2Instance')
self.compare_vertex_breadcrumbs(local_graph, s3bucketsuspended_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::S3::Bucket.S3BucketSuspended')
self.compare_vertex_breadcrumbs(local_graph, s3bucketenabled_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::S3::Bucket.S3BucketEnabled')
def test_build_graph_with_dynamic_blocks(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/dynamic_lambda_function'))
graph_manager = TerraformGraphManager(NetworkxConnector())
local_graph, tf = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
lambda_attributes = local_graph.vertices[0].attributes
self.assertIn("dead_letter_config", lambda_attributes.keys())
def test_build_graph_with_linked_registry_modules(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/modules/registry_security_group_inner_module'))
graph_manager = TerraformGraphManager(NetworkxConnector())
local_graph, tf_definitions = graph_manager.build_graph_from_source_directory(resources_dir,
render_variables=True,
download_external_modules=True)
outputs_vpcs = self.get_vertex_by_name_and_type(local_graph, BlockType.OUTPUT, 'security_group_vpc_id',
multiple=True)
resource_flow_log = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'aws_flow_log.related_flow_log')
resource_security_group_this = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'aws_security_group.this')
resource_security_group_this_name_prefix = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'aws_security_group.this_name_prefix')
output_this_security_group_vpc_id_inner = [o for o in outputs_vpcs if 'http-80' in o.path][0]
output_this_security_group_vpc_id_outer = [o for o in outputs_vpcs if 'http-80' not in o.path][0]
self.check_edge(local_graph, node_from=resource_flow_log, node_to=output_this_security_group_vpc_id_inner,
expected_label='vpc_id')
self.check_edge(local_graph, node_from=output_this_security_group_vpc_id_outer,
node_to=resource_security_group_this, expected_label='value')
self.check_edge(local_graph, node_from=output_this_security_group_vpc_id_outer,
node_to=resource_security_group_this_name_prefix, expected_label='value')
# cleanup
if os.path.exists(os.path.join(resources_dir, external_modules_download_path)):
shutil.rmtree(os.path.join(resources_dir, external_modules_download_path))
def test_build_graph_from_source_directory_with_rendering(self):
root_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, "./runner/resources"))
graph_manager = CloudformationGraphManager(
db_connector=NetworkxConnector())
local_graph, definitions = graph_manager.build_graph_from_source_directory(
root_dir, render_variables=True)
sqs_queue_vertex = local_graph.vertices[
local_graph.vertices_block_name_map[
BlockType.RESOURCE]["AWS::SQS::Queue.acmeCWSQueue"][0]]
expected_node = {
'Fn::Join': ['', ['acme', '-acmecws']],
'__startline__': 646,
'__endline__': 656
}
self.assertDictEqual(expected_node,
sqs_queue_vertex.config["QueueName"])
found = False
for d in definitions:
if 'resources/success.json' in d:
found = True
node = definitions[d]['Resources']['acmeCWSQueue'][
'Properties']['QueueName']
self.assertDictEqual(expected_node, node)
self.assertTrue(found,
'Did not find the wanted node, for acmeCWSQueue')
def validate_render_ref(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
db_name_default_value = "db1"
kms_master_key_id_expected_attributes = {'Default': None}
db_name_expected_attributes = {'Default': db_name_default_value}
my_source_queue_expected_attributes = {'KmsMasterKeyId.Ref': 'KmsMasterKeyId'}
my_db_expected_attributes = {'DBName': db_name_default_value}
my_db_instance_name_expected_attributes = {'Value.Ref': 'MyDB'}
self.compare_vertex_attributes(local_graph, kms_master_key_id_expected_attributes, BlockType.PARAMETERS, 'KmsMasterKeyId')
self.compare_vertex_attributes(local_graph, db_name_expected_attributes, BlockType.PARAMETERS, 'DBName')
self.compare_vertex_attributes(local_graph, my_source_queue_expected_attributes, BlockType.RESOURCE, 'AWS::SQS::Queue.MySourceQueue')
self.compare_vertex_attributes(local_graph, my_db_expected_attributes, BlockType.RESOURCE, 'AWS::RDS::DBInstance.MyDB')
self.compare_vertex_attributes(local_graph, my_db_instance_name_expected_attributes, BlockType.OUTPUTS, 'MyDBInstanceName')
kms_master_key_id_expected_breadcrumbs = {}
db_name_expected_breadcrumbs = {}
my_source_queue_expected_breadcrumbs = {}
my_db_expected_breadcrumbs = {'DBName': [{'type': BlockType.PARAMETERS, 'name': 'DBName', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}]}
my_db_instance_name_expected_breadcrumbs = {}
self.compare_vertex_breadcrumbs(local_graph, kms_master_key_id_expected_breadcrumbs, BlockType.PARAMETERS, 'KmsMasterKeyId')
self.compare_vertex_breadcrumbs(local_graph, db_name_expected_breadcrumbs, BlockType.PARAMETERS, 'DBName')
self.compare_vertex_breadcrumbs(local_graph, my_source_queue_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::SQS::Queue.MySourceQueue')
self.compare_vertex_breadcrumbs(local_graph, my_db_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::RDS::DBInstance.MyDB')
self.compare_vertex_breadcrumbs(local_graph, my_db_instance_name_expected_breadcrumbs, BlockType.OUTPUTS, 'MyDBInstanceName')
def test_update_vertices_configs_deep_nesting(self):
resources_dir = os.path.join(
TEST_DIRNAME,
'../resources/variable_rendering/render_deep_nesting')
graph_manager = GraphManager(NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=True)
expected_config = {
'aws_s3_bucket': {
'default': {
'server_side_encryption_configuration': [{
'rule': [{
'apply_server_side_encryption_by_default': [{
'sse_algorithm': ['AES256'],
'kms_master_key_id': ['']
}]
}]
}]
}
}
}
actual_config = local_graph.vertices[
local_graph.vertices_by_block_type.get(
BlockType.RESOURCE)[0]].config
self.assertDictEqual(expected_config, actual_config)
print('')
def __init__(self, args):
graph_manager = KubernetesGraphManager(
db_connector=NetworkxConnector())
super().__init__(graph_manager,
"checkov/kubernetes/checks/graph_checks",
os.path.dirname(__file__) + "/test_checks",
'kubernetes', __file__, args)
def get_policy_results(root_folder, policy):
check_id = policy['metadata']['id']
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(root_folder)
nx_graph = graph_manager.save_graph(local_graph)
registry = get_checks_registry()
checks_results = registry.run_checks(nx_graph, RunnerFilter(checks=[check_id]))
return create_report_from_graph_checks_results(checks_results, policy['metadata'])
def test_build_graph_from_definitions(self):
relative_file_path = "../checks/example_AllowedCapabilities/cronjob-PASSED.yaml"
definitions = {}
file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path))
(definitions[relative_file_path], definitions_raw) = parse(file)
resource = definitions[relative_file_path][0]
graph_manager = KubernetesGraphManager(
db_connector=NetworkxConnector())
local_graph = graph_manager.build_graph_from_definitions(definitions)
self.assertEqual(1, len(local_graph.vertices))
self.assert_vertex(local_graph.vertices[0], resource)
def __init__(self, parser=Parser(), db_connector=NetworkxConnector(), external_registries=None,
source="Terraform", graph_class=TerraformLocalGraph, graph_manager=None):
self.external_registries = [] if external_registries is None else external_registries
self.graph_class = graph_class
self.parser = parser
self.definitions = None
self.context = None
self.breadcrumbs = None
self.evaluations_context: Dict[str, Dict[str, EvaluationContext]] = {}
self.graph_manager = graph_manager if graph_manager is not None else TerraformGraphManager(source=source,
db_connector=db_connector)
self.graph_registry = get_graph_checks_registry(self.check_type)
def test_build_graph_with_linked_modules(self):
# see the image to view the expected graph in tests/resources/modules/linked_modules/expected_graph.png
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/modules/linked_modules'))
graph_manager = GraphManager(NetworkxConnector())
local_graph, tf_definitions = graph_manager.build_graph_from_source_directory(
resources_dir, render_variables=False)
vertices_by_block_type = local_graph.vertices_by_block_type
expected_vertices_num_by_type = {
BlockType.VARIABLE: 5,
BlockType.RESOURCE: 5,
BlockType.OUTPUT: 3,
BlockType.MODULE: 2,
BlockType.DATA: 1,
}
for block_type, count in expected_vertices_num_by_type.items():
self.assertEqual(count, len(vertices_by_block_type[block_type]))
output_this_lambda_func_arn = self.get_vertex_by_name_and_type(
local_graph, BlockType.OUTPUT, 'this_lambda_function_arn')
output_this_lambda_func_name = self.get_vertex_by_name_and_type(
local_graph, BlockType.OUTPUT, 'this_lambda_function_name')
output_this_s3_bucket_id = self.get_vertex_by_name_and_type(
local_graph, BlockType.OUTPUT, 'this_s3_bucket_id')
resource_aws_lambda_function = self.get_vertex_by_name_and_type(
local_graph, BlockType.RESOURCE, 'aws_lambda_function.this')
resource_aws_s3_bucket_policy = self.get_vertex_by_name_and_type(
local_graph, BlockType.RESOURCE, 'aws_s3_bucket_policy.this')
resource_aws_s3_bucket = self.get_vertex_by_name_and_type(
local_graph, BlockType.RESOURCE, 'aws_s3_bucket.this')
self.check_edge(local_graph,
node_from=output_this_lambda_func_arn,
node_to=resource_aws_lambda_function,
expected_label='value')
self.check_edge(local_graph,
node_from=output_this_lambda_func_name,
node_to=resource_aws_lambda_function,
expected_label='value')
self.check_edge(local_graph,
node_from=output_this_s3_bucket_id,
node_to=resource_aws_s3_bucket_policy,
expected_label='value')
self.check_edge(local_graph,
node_from=output_this_s3_bucket_id,
node_to=resource_aws_s3_bucket,
expected_label='value')
def validate_render_subsequent_evals(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
cidr_block_expected_expected_value = "172.16.0.0/16"
cidr_block_expected_attributes = {'Default': cidr_block_expected_expected_value}
web_vpc_expected_attributes = {'CidrBlock': cidr_block_expected_expected_value}
my_sg_expected_attributes = {'SecurityGroupIngress.CidrIp': cidr_block_expected_expected_value}
self.compare_vertex_attributes(local_graph, cidr_block_expected_attributes, BlockType.PARAMETERS, 'CidrBlock')
self.compare_vertex_attributes(local_graph, web_vpc_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::VPC.WebVPC')
self.compare_vertex_attributes(local_graph, my_sg_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::SecurityGroup.MySG')
cidr_block_expected_breadcrumbs = {}
web_vpc_expected_breadcrumbs = {'CidrBlock': [{'type': BlockType.PARAMETERS, 'name': 'CidrBlock', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}, {'type': BlockType.RESOURCE, 'name': 'AWS::EC2::VPC.WebVPC', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'CidrBlock'}]}
my_sg_expected_breadcrumbs = {
"SecurityGroupIngress.0.CidrIp": [
{
"type": BlockType.PARAMETERS,
"name": "CidrBlock",
"path": os.path.join(test_dir, f"test.{file_ext}"),
"attribute_key": "Default",
},
{
"type": BlockType.RESOURCE,
"name": "AWS::EC2::VPC.WebVPC",
"path": os.path.join(test_dir, f"test.{file_ext}"),
"attribute_key": "CidrBlock",
},
],
"SecurityGroupIngress.0": [
{
"type": BlockType.PARAMETERS,
"name": "CidrBlock",
"path": os.path.join(test_dir, f"test.{file_ext}"),
"attribute_key": "Default",
},
{
"type": BlockType.RESOURCE,
"name": "AWS::EC2::VPC.WebVPC",
"path": os.path.join(test_dir, f"test.{file_ext}"),
"attribute_key": "CidrBlock",
},
],
}
self.compare_vertex_breadcrumbs(local_graph, cidr_block_expected_breadcrumbs, BlockType.PARAMETERS, 'CidrBlock')
self.compare_vertex_breadcrumbs(local_graph, web_vpc_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::VPC.WebVPC')
self.compare_vertex_breadcrumbs(local_graph, my_sg_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::SecurityGroup.MySG')
def test_build_graph_from_definitions(self):
relative_file_path = "./checks/resource/aws/example_APIGatewayXray/APIGatewayXray-PASSED.yaml"
definitions = {}
file = os.path.realpath(os.path.join(TEST_DIRNAME, relative_file_path))
(definitions[relative_file_path], definitions_raw) = parse(file)
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph = graph_manager.build_graph_from_definitions(definitions)
self.assertEqual(1, len(local_graph.vertices))
resource_vertex = local_graph.vertices[0]
self.assertEqual("AWS::ApiGateway::Stage.Enabled", resource_vertex.name)
self.assertEqual("AWS::ApiGateway::Stage.Enabled", resource_vertex.id)
self.assertEqual(BlockType.RESOURCE, resource_vertex.block_type)
self.assertEqual("CloudFormation", resource_vertex.source)
self.assertDictEqual(definitions[relative_file_path]["Resources"]["Enabled"]["Properties"], resource_vertex.attributes)
def __init__(self,
graph_class=KubernetesLocalGraph,
db_connector=NetworkxConnector(),
source="Kubernetes",
graph_manager=None,
external_registries=None):
self.external_registries = [] if external_registries is None else external_registries
self.check_type = "kubernetes"
self.graph_class = graph_class
self.graph_manager = \
graph_manager if graph_manager else KubernetesGraphManager(source=source, db_connector=db_connector)
self.graph_registry = get_graph_checks_registry(self.check_type)
self.definitions_raw = {}
def test_build_graph_from_source_directory_no_rendering(self):
root_dir = os.path.realpath(os.path.join(TEST_DIRNAME, "./runner/resources"))
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, definitions = graph_manager.build_graph_from_source_directory(root_dir, render_variables=False)
expected_resources_by_file = {
os.path.join(root_dir, "tags.yaml"): [
"AWS::S3::Bucket.DataBucket",
"AWS::S3::Bucket.NoTags",
"AWS::EKS::Nodegroup.EKSClusterNodegroup",
"AWS::AutoScaling::AutoScalingGroup.TerraformServerAutoScalingGroup",
],
os.path.join(root_dir, "cfn_newline_at_end.yaml"): [
"AWS::RDS::DBInstance.MyDB",
"AWS::S3::Bucket.MyBucket",
],
os.path.join(root_dir, "success.json"): [
"AWS::S3::Bucket.acmeCWSBucket",
"AWS::S3::Bucket.acmeCWSBucket2",
"AWS::S3::BucketPolicy.acmeCWSBucketPolicy",
"AWS::SNS::Topic.acmeCWSTopic",
"AWS::SNS::TopicPolicy.acmeCWSTopicPolicy",
"AWS::CloudTrail::Trail.acmeCWSTrail",
"AWS::KMS::Key.CloudtrailKMSKey",
"AWS::KMS::Alias.CloudtrailKMSKeyAlias",
"AWS::SQS::Queue.acmeCWSQueue",
"AWS::SQS::QueuePolicy.acmeCWSQueuePolicy",
"AWS::SNS::Subscription.acmeCWSSubscription",
"AWS::IAM::Role.acmeCWSSACrossAccountAccessRole",
"AWS::EKS::Cluster.eksCluster",
"Custom::acmeSnsCustomResource.acmeSnsCustomResource",
],
os.path.join(root_dir, "fail.yaml"): [
"AWS::SQS::Queue.UnencryptedQueue",
]
}
self.assertEqual(41, len(local_graph.vertices))
self.assertEqual(21, len(local_graph.vertices_by_block_type[BlockType.RESOURCE]))
self.assertEqual(9, len(local_graph.vertices_by_block_type[BlockType.PARAMETERS]))
self.assertEqual(6, len(local_graph.vertices_by_block_type[BlockType.OUTPUTS]))
self.assertEqual(4, len(local_graph.vertices_by_block_type[BlockType.CONDITIONS]))
self.assertEqual(1, len(local_graph.vertices_by_block_type[BlockType.MAPPINGS]))
for v in local_graph.vertices:
if v.block_type == BlockType.RESOURCE:
self.assertIn(v.name, expected_resources_by_file[v.path])
sqs_queue_vertex = local_graph.vertices[local_graph.vertices_block_name_map[BlockType.RESOURCE]["AWS::SQS::Queue.acmeCWSQueue"][0]]
self.assertDictEqual({'Fn::Join': ['', [{'Ref': 'ResourceNamePrefix', '__startline__': 650, '__endline__': 652}, '-acmecws']], '__startline__': 646, '__endline__': 656}, sqs_queue_vertex.attributes["QueueName"])
def __init__(
self,
db_connector=NetworkxConnector(),
source="CloudFormation",
graph_class=CloudformationLocalGraph,
graph_manager=None,
external_registries=None,
):
self.external_registries = [] if external_registries is None else external_registries
self.graph_class = graph_class
self.graph_manager = (graph_manager if graph_manager is not None else
CloudformationGraphManager(
source=source, db_connector=db_connector))
self.definitions_raw = {}
self.graph_registry = get_graph_checks_registry(self.check_type)
def __init__(
self,
graph_class=KubernetesLocalGraph,
db_connector=NetworkxConnector(),
source="Kubernetes",
graph_manager=None,
):
self.check_type = "kubernetes"
self.graph_class = graph_class
# TODO: uncomment it in order to create the graph
# self.graph_manager = \
# graph_manager if graph_manager else KubernetesGraphManager(source=source, db_connector=db_connector)
self.graph_registry = get_graph_checks_registry(self.check_type)
self.definitions_raw = {}
def __init__(
self,
graph_class: Type[LocalGraph] = KubernetesLocalGraph,
db_connector: NetworkxConnector = NetworkxConnector(),
source: str = "Kubernetes",
graph_manager: Optional[GraphManager] = None,
external_registries: Optional[List[BaseRegistry]] = None) -> None:
self.external_registries = [] if external_registries is None else external_registries
self.check_type = "kubernetes"
self.graph_class = graph_class
self.graph_manager = \
graph_manager if graph_manager else KubernetesGraphManager(source=source, db_connector=db_connector)
self.graph_registry = get_graph_checks_registry(self.check_type)
self.definitions_raw = {}
def validate_render_join(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
s3bucket1_expected_attributes = {'BucketName': 'a:b:c'}
s3bucket2_expected_attributes = {'BucketName': 'my_bucket_name_test'}
self.compare_vertex_attributes(local_graph, s3bucket1_expected_attributes, BlockType.RESOURCE, 'AWS::S3::Bucket.S3Bucket1')
self.compare_vertex_attributes(local_graph, s3bucket2_expected_attributes, BlockType.RESOURCE, 'AWS::S3::Bucket.S3Bucket2')
s3bucket1_expected_breadcrumbs = {}
s3bucket2_expected_breadcrumbs = {'BucketName.Fn::Join.1.0': [{'type': BlockType.PARAMETERS, 'name': 'BucketName', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}], 'BucketName.Fn::Join.1': [{'type': BlockType.PARAMETERS, 'name': 'BucketName', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}], 'BucketName.Fn::Join': [{'type': BlockType.PARAMETERS, 'name': 'BucketName', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}]}
self.compare_vertex_breadcrumbs(local_graph, s3bucket1_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::S3::Bucket.S3Bucket1')
self.compare_vertex_breadcrumbs(local_graph, s3bucket2_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::S3::Bucket.S3Bucket2')
def test_blocks_from_local_graph_module(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/modules/stacks'))
graph_manager = TerraformGraphManager(NetworkxConnector())
local_graph, tf = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
tf, _ = convert_graph_vertices_to_tf_definitions(local_graph.vertices, resources_dir)
found_results = 0
for key, value in tf.items():
if key.startswith(os.path.join(os.path.dirname(resources_dir), 's3_inner_modules', 'inner', 'main.tf')):
conf = value['resource'][0]['aws_s3_bucket']['inner_s3']
if 'stage/main' in key or 'prod/main' in key:
self.assertTrue(conf['versioning'][0]['enabled'][0])
found_results += 1
elif 'test/main' in key:
self.assertFalse(conf['versioning'][0]['enabled'][0])
found_results += 1
self.assertEqual(found_results, 3)
def test_single_edge_with_same_label(self):
resources_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, '../resources/k8_service'))
graph_manager = GraphManager(NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(resources_dir,
render_variables=True)
edges_hash = []
for e in local_graph.edges:
edge_hash = calculate_hash({"origin": e.origin, "dest": e.dest, "label": e.label})
if edge_hash in edges_hash:
origin = local_graph.vertices[e.origin]
dest = local_graph.vertices[e.dest]
self.fail(f'edge {e} == [{origin} - {e.label} -> {dest}] appears more than once in the graph')
else:
edges_hash.append(edge_hash)
def validate_render_findinmap(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
region_map_expected_ami_value = "ami-0ff8a91507f77f867"
region_map_expected_attributes = {'us-east-1.AMI': region_map_expected_ami_value}
ec2instance_expected_attributes = {'ImageId': region_map_expected_ami_value}
self.compare_vertex_attributes(local_graph, region_map_expected_attributes, BlockType.MAPPINGS, 'RegionMap')
self.compare_vertex_attributes(local_graph, ec2instance_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::Instance.EC2Instance')
region_map_expected_breadcrumbs = {}
ec2instance_expected_breadcrumbs = {'ImageId': [{'type': BlockType.MAPPINGS, 'name': 'RegionMap', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'us-east-1.AMI'}]}
self.compare_vertex_breadcrumbs(local_graph, region_map_expected_breadcrumbs, BlockType.MAPPINGS, 'RegionMap')
self.compare_vertex_breadcrumbs(local_graph, ec2instance_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::Instance.EC2Instance')
def test_build_graph(self):
resources_dir = os.path.join(TEST_DIRNAME,
'../resources/general_example')
graph_manager = GraphManager(db_connector=NetworkxConnector())
graph, tf_definitions = graph_manager.build_graph_from_source_directory(
resources_dir)
expected_num_of_var_nodes = 3
expected_num_of_locals_nodes = 1
expected_num_of_resources_nodes = 1
expected_num_of_provider_nodes = 1
vertices_by_block_type = graph.vertices_by_block_type
self.assertEqual(expected_num_of_var_nodes,
len(vertices_by_block_type[BlockType.VARIABLE]))
self.assertEqual(expected_num_of_locals_nodes,
len(vertices_by_block_type[BlockType.LOCALS]))
self.assertEqual(expected_num_of_resources_nodes,
len(vertices_by_block_type[BlockType.RESOURCE]))
self.assertEqual(expected_num_of_provider_nodes,
len(vertices_by_block_type[BlockType.PROVIDER]))
provider_node = graph.vertices[vertices_by_block_type[
BlockType.PROVIDER][0]]
resource_node = graph.vertices[vertices_by_block_type[
BlockType.RESOURCE][0]]
local_node = graph.vertices[vertices_by_block_type[BlockType.LOCALS]
[0]]
var_bucket_name_node = None
var_region_node = None
var_aws_profile_node = None
for index in vertices_by_block_type[BlockType.VARIABLE]:
var_node = graph.vertices[index]
if var_node.name == 'aws_profile':
var_aws_profile_node = var_node
if var_node.name == 'bucket_name':
var_bucket_name_node = var_node
if var_node.name == 'region':
var_region_node = var_node
self.check_edge(graph, resource_node, local_node, 'bucket')
self.check_edge(graph, resource_node, provider_node, 'provider')
self.check_edge(graph, resource_node, var_region_node, 'region')
self.check_edge(graph, provider_node, var_aws_profile_node, 'profile')
self.check_edge(graph, local_node, var_bucket_name_node, 'bucket_name')
def validate_render_sub(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
company_name_expected_value = "acme"
web_vpc_expected_cidr_block = "172.16.0.0/16"
# Parameters
company_name_expected_attributes = {'Default': company_name_expected_value}
environment_expected_attributes = {'Default': None}
# Resources
web_vpc_expected_attributes = {'CidrBlock': web_vpc_expected_cidr_block}
default_db_expected_attributes = {'DBName': {'Fn::Sub': 'rds-${AWS::AccountId}-${CompanyName}-${Environment}'}}
# Outputs
db_endpoint_sg_expected_attributes = {'Value.Fn::Sub': "${DefaultDB.Endpoint.Address}:${DefaultDB.Endpoint.Port}"}
web_vpc_cidr_block_expected_attributes = {'Value': web_vpc_expected_cidr_block}
cidr_block_associations_expected_attributes = {'Value.Fn::Sub': "${WebVPC.CidrBlockAssociations}"}
default_db_name_expected_attributes = {'Value': {'Fn::Sub': 'rds-${AWS::AccountId}-${CompanyName}-${Environment}'}}
self.compare_vertex_attributes(local_graph, company_name_expected_attributes, BlockType.PARAMETERS, 'CompanyName')
self.compare_vertex_attributes(local_graph, environment_expected_attributes, BlockType.PARAMETERS, 'Environment')
self.compare_vertex_attributes(local_graph, web_vpc_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::VPC.WebVPC')
self.compare_vertex_attributes(local_graph, default_db_expected_attributes, BlockType.RESOURCE, 'AWS::RDS::DBInstance.DefaultDB')
self.compare_vertex_attributes(local_graph, db_endpoint_sg_expected_attributes, BlockType.OUTPUTS, 'DBEndpoint')
self.compare_vertex_attributes(local_graph, web_vpc_cidr_block_expected_attributes, BlockType.OUTPUTS, 'WebVPCCidrBlock')
self.compare_vertex_attributes(local_graph, cidr_block_associations_expected_attributes, BlockType.OUTPUTS, 'CidrBlockAssociations')
self.compare_vertex_attributes(local_graph, default_db_name_expected_attributes, BlockType.OUTPUTS, 'DefaultDBName')
company_name_expected_breadcrumbs = {}
environment_expected_breadcrumbs = {}
web_vpc_expected_breadcrumbs = {}
default_db_expected_breadcrumbs = {}
db_endpoint_sg_expected_breadcrumbs = {}
web_vpc_cidr_block_expected_breadcrumbs = {'Value': [{'type': BlockType.RESOURCE, 'name': 'AWS::EC2::VPC.WebVPC', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'CidrBlock'}]}
cidr_block_associations_expected_breadcrumbs = {}
default_db_name_expected_breadcrumbs = {}
self.compare_vertex_breadcrumbs(local_graph, company_name_expected_breadcrumbs, BlockType.PARAMETERS, 'CompanyName')
self.compare_vertex_breadcrumbs(local_graph, environment_expected_breadcrumbs, BlockType.PARAMETERS, 'Environment')
self.compare_vertex_breadcrumbs(local_graph, web_vpc_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::VPC.WebVPC')
self.compare_vertex_breadcrumbs(local_graph, default_db_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::RDS::DBInstance.DefaultDB')
self.compare_vertex_breadcrumbs(local_graph, db_endpoint_sg_expected_breadcrumbs, BlockType.OUTPUTS, 'DBEndpoint')
self.compare_vertex_breadcrumbs(local_graph, web_vpc_cidr_block_expected_breadcrumbs, BlockType.OUTPUTS, 'WebVPCCidrBlock')
self.compare_vertex_breadcrumbs(local_graph, cidr_block_associations_expected_breadcrumbs, BlockType.OUTPUTS, 'CidrBlockAssociations')
self.compare_vertex_breadcrumbs(local_graph, default_db_name_expected_breadcrumbs, BlockType.OUTPUTS, 'DefaultDBName')
def test_build_graph_from_source_directory_no_rendering(self):
root_dir = os.path.realpath(
os.path.join(TEST_DIRNAME, "../runner/resources"))
graph_manager = KubernetesGraphManager(
db_connector=NetworkxConnector())
local_graph, definitions = graph_manager.build_graph_from_source_directory(
root_dir, render_variables=False)
expected_resources_by_file = {
os.path.join("/example.yaml"): ["Service.default.kafka-hs"],
os.path.join("/example_multiple.yaml"):
["PodDisruptionBudget.a.a", "Service.default.a"]
}
self.assertEqual(3, len(local_graph.vertices))
self.assertEqual(
3, len(local_graph.vertices_by_block_type[BlockType.RESOURCE]))
for v in local_graph.vertices:
self.assertIn(v.name, expected_resources_by_file[v.path])
def validate_render_select(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
subnet0_expected_attributes = {'CidrBlock': '10.0.48.0/24'}
grapes_select_expected_attributes = {'Value': 'grapes'}
out_of_bound_select_expected_attributes = {'Value.Fn::Select': ['7', ['apples', 'grapes', 'oranges', 'mangoes']]}
self.compare_vertex_attributes(local_graph, subnet0_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::Subnet.Subnet0')
self.compare_vertex_attributes(local_graph, grapes_select_expected_attributes, BlockType.OUTPUTS, 'GrapesSelect')
self.compare_vertex_attributes(local_graph, out_of_bound_select_expected_attributes, BlockType.OUTPUTS, 'OutOfBoundSelect')
subnet0_expected_breadcrumbs = {'CidrBlock.Fn::Select.1': [{'type': BlockType.PARAMETERS, 'name': 'DbSubnetIpBlocks', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}], 'CidrBlock.Fn::Select': [{'type': BlockType.PARAMETERS, 'name': 'DbSubnetIpBlocks', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'Default'}]}
grapes_select_expected_breadcrumbs = {}
out_of_bound_select_expected_breadcrumbs = {}
self.compare_vertex_breadcrumbs(local_graph, subnet0_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::Subnet.Subnet0')
self.compare_vertex_breadcrumbs(local_graph, grapes_select_expected_breadcrumbs, BlockType.OUTPUTS, 'GrapesSelect')
self.compare_vertex_breadcrumbs(local_graph, out_of_bound_select_expected_breadcrumbs, BlockType.OUTPUTS, 'OutOfBoundSelect')
def validate_render_getatt(self, test_dir: str, file_ext: str):
graph_manager = CloudformationGraphManager(db_connector=NetworkxConnector())
local_graph, _ = graph_manager.build_graph_from_source_directory(test_dir, render_variables=True)
web_vpc_expected_cidr_block = "172.16.0.0/16"
web_vpc_expected_attributes = {'CidrBlock': web_vpc_expected_cidr_block}
my_sg_expected_attributes = {'SecurityGroupIngress.CidrIp': web_vpc_expected_cidr_block}
web_vpc_default_sg_expected_attributes = {'Value.Fn::GetAtt': ['WebVPC', 'DefaultSecurityGroup']}
self.compare_vertex_attributes(local_graph, web_vpc_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::VPC.WebVPC')
self.compare_vertex_attributes(local_graph, my_sg_expected_attributes, BlockType.RESOURCE, 'AWS::EC2::SecurityGroup.MySG')
self.compare_vertex_attributes(local_graph, web_vpc_default_sg_expected_attributes, BlockType.OUTPUTS, 'WebVPCDefaultSg')
web_vpc_expected_breadcrumbs = {}
my_sg_expected_breadcrumbs = {'SecurityGroupIngress.CidrIp': [{'type': BlockType.RESOURCE, 'name': 'AWS::EC2::VPC.WebVPC', 'path': os.path.join(test_dir, f'test.{file_ext}'), 'attribute_key': 'CidrBlock'}]}
web_vpc_default_sg_expected_breadcrumbs = {}
self.compare_vertex_breadcrumbs(local_graph, web_vpc_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::VPC.WebVPC')
self.compare_vertex_breadcrumbs(local_graph, my_sg_expected_breadcrumbs, BlockType.RESOURCE, 'AWS::EC2::SecurityGroup.MySG')
self.compare_vertex_breadcrumbs(local_graph, web_vpc_default_sg_expected_breadcrumbs, BlockType.OUTPUTS, 'WebVPCDefaultSg')
def test_build_graph_with_deep_nested_edges(self):
resources_dir = os.path.realpath(os.path.join(TEST_DIRNAME, '../resources/k8_service'))
graph_manager = TerraformGraphManager(NetworkxConnector())
local_graph, tf = graph_manager.build_graph_from_source_directory(resources_dir, render_variables=True)
resource_kubernetes_deployment = self.get_vertex_by_name_and_type(local_graph, BlockType.RESOURCE,
'kubernetes_deployment.bazel_remote_cache')
locals_name = self.get_vertex_by_name_and_type(local_graph, BlockType.LOCALS, 'name')
locals_labels = self.get_vertex_by_name_and_type(local_graph, BlockType.LOCALS, 'labels')
self.check_edge(local_graph, node_from=locals_labels, node_to=locals_name,
expected_label="labels.app.kubernetes.io/name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="metadata.name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="spec.template.metadata.name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="spec.template.spec.container.name")
self.check_edge(local_graph, node_from=resource_kubernetes_deployment, node_to=locals_name,
expected_label="spec.template.spec.volume.1.config_map.name")
