def main(argv: Optional[List[str]] = None) -> int: if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("input_json_filepaths", type=Path, nargs="+") args_ns = parser.parse_args(argv) input_json_filepaths = args_ns.input_json_filepaths if len(input_json_filepaths) > 1: raise NotImplementedError("Only one input supported at this time") # create a dict of scan ids to GraphSets. This contains all of the data in the provided input. scan_ids_graph_sets: Dict[int, GraphSet] = { scan_id: GraphSet.from_json_file(filepath) for scan_id, filepath in enumerate(input_json_filepaths) } # discover tables which need to be created by iterating over resources and finding the maximum # set of predicates used for each type table_defns = build_table_defns(scan_ids_graph_sets.values()) # build data table_names_datas = build_data(scan_ids_graph_sets.values(), table_defns) table_names_tables: Dict[str, tableauhyperapi.TableDefinition] = {} with tableauhyperapi.HyperProcess( telemetry=tableauhyperapi.Telemetry. DO_NOT_SEND_USAGE_DATA_TO_TABLEAU) as hyper: with tableauhyperapi.Connection( endpoint=hyper.endpoint, database="altimeter.hyper", create_mode=tableauhyperapi.CreateMode.CREATE_AND_REPLACE, ) as connection: # create tables for table_name, columns in table_defns.items(): table = tableauhyperapi.TableDefinition( table_name=table_name, columns=[column.to_hyper() for column in columns]) connection.catalog.create_table(table) table_names_tables[table_name] = table for table_name, datas in table_names_datas.items(): with tableauhyperapi.Inserter( connection, table_names_tables[table_name]) as inserter: inserter.add_rows(datas) inserter.execute() return 0
def test(self): with tempfile.TemporaryDirectory() as temp_dir: resource_region_name = "us-east-1" # get moto"s enabled regions ec2_client = boto3.client("ec2", region_name=resource_region_name) all_regions = ec2_client.describe_regions( Filters=[{ "Name": "opt-in-status", "Values": ["opt-in-not-required", "opted-in"] }])["Regions"] account_id = get_account_id() all_region_names = tuple(region["RegionName"] for region in all_regions) enabled_region_names = tuple( region["RegionName"] for region in all_regions if region["OptInStatus"] != "not-opted-in") delete_vpcs(all_region_names) # add a diverse set of resources which are supported by moto ## dynamodb # TODO moto is not returning TableId in list/describe # dynamodb_table_1_arn = create_dynamodb_table( # name="test_table_1", # attr_name="test_hash_key_attr_1", # attr_type="S", # key_type="HASH", # region_name=region_name, # ) ## s3 bucket_1_name = "test_bucket" bucket_1_arn, bucket_1_creation_date = create_bucket( name=bucket_1_name, account_id=account_id, region_name=resource_region_name) ## ec2 vpc_1_cidr = "10.0.0.0/16" vpc_1_id = create_vpc(cidr_block=vpc_1_cidr, region_name=resource_region_name) vpc_1_arn = VPCResourceSpec.generate_arn( resource_id=vpc_1_id, account_id=account_id, region=resource_region_name) subnet_1_cidr = "10.0.0.0/24" subnet_1_cidr_network = ipaddress.IPv4Network(subnet_1_cidr, strict=False) subnet_1_first_ip, subnet_1_last_ip = ( int(subnet_1_cidr_network[0]), int(subnet_1_cidr_network[-1]), ) subnet_1_id = create_subnet(cidr_block=subnet_1_cidr, vpc_id=vpc_1_id, region_name=resource_region_name) subnet_1_arn = SubnetResourceSpec.generate_arn( resource_id=subnet_1_id, account_id=account_id, region=resource_region_name) fixed_bucket_1_arn = f"arn:aws:s3:::{bucket_1_name}" flow_log_1_id, flow_log_1_creation_time = create_flow_log( vpc_id=vpc_1_id, dest_bucket_arn=fixed_bucket_1_arn, region_name=resource_region_name, ) flow_log_1_arn = FlowLogResourceSpec.generate_arn( resource_id=flow_log_1_id, account_id=account_id, region=resource_region_name) ebs_volume_1_size = 128 ebs_volume_1_az = f"{resource_region_name}a" ebs_volume_1_arn, ebs_volume_1_create_time = create_volume( size=ebs_volume_1_size, az=ebs_volume_1_az, region_name=resource_region_name) ## iam policy_1_name = "test_policy_1" policy_1_arn, policy_1_id = create_iam_policy( name=policy_1_name, policy_doc={ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "logs:CreateLogGroup", "Resource": "*" }, ], }, ) role_1_name = "test_role_1" role_1_assume_role_policy_doc = { "Version": "2012-10-17", "Statement": [{ "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "lambda.amazonaws.com" }, "Sid": "", }], } role_1_description = "Test Role 1" role_1_max_session_duration = 3600 role_1_arn = create_iam_role( name=role_1_name, assume_role_policy_doc=role_1_assume_role_policy_doc, description=role_1_description, max_session_duration=role_1_max_session_duration, ) ## lambda lambda_function_1_name = "test_lambda_function_1" lambda_function_1_runtime = "python3.7" lambda_function_1_handler = "lambda_function.lambda_handler" lambda_function_1_description = "Test Lambda Function 1" lambda_function_1_timeout = 30 lambda_function_1_memory_size = 256 lambda_function_1_arn = create_lambda_function( name=lambda_function_1_name, runtime=lambda_function_1_runtime, role_name=role_1_arn, handler=lambda_function_1_handler, description=lambda_function_1_description, timeout=lambda_function_1_timeout, memory_size=lambda_function_1_memory_size, publish=False, region_name=resource_region_name, ) # scan test_scan_id = "test_scan_id" aws_config = AWSConfig( artifact_path=temp_dir, pruner_max_age_min=4320, graph_name="alti", concurrency=ConcurrencyConfig(max_account_scan_threads=1, max_svc_scan_threads=1, max_account_scan_tries=2), scan=ScanConfig( accounts=(), regions=(), scan_sub_accounts=False, preferred_account_scan_regions=( "us-west-1", "us-west-2", "us-east-1", "us-east-2", ), ), accessor=Accessor( credentials_cache=AWSCredentialsCache(cache={}), multi_hop_accessors=[], cache_creds=True, ), write_master_json=True, ) resource_spec_classes = ( # DynamoDbTableResourceSpec, TODO moto EBSVolumeResourceSpec, FlowLogResourceSpec, IAMPolicyResourceSpec, IAMRoleResourceSpec, LambdaFunctionResourceSpec, S3BucketResourceSpec, SubnetResourceSpec, VPCResourceSpec, ) muxer = LocalAWSScanMuxer( scan_id=test_scan_id, config=aws_config, resource_spec_classes=resource_spec_classes, ) with unittest.mock.patch( "altimeter.aws.scan.account_scanner.get_all_enabled_regions" ) as mock_get_all_enabled_regions: mock_get_all_enabled_regions.return_value = enabled_region_names aws2n_result = aws2n( scan_id=test_scan_id, config=aws_config, muxer=muxer, load_neptune=False, ) graph_set = GraphSet.from_json_file( Path(aws2n_result.json_path)) self.assertEqual(len(graph_set.errors), 0) self.assertEqual(graph_set.name, "alti") self.assertEqual(graph_set.version, "2") # now check each resource type self.maxDiff = None ## Accounts expected_account_resources = [ Resource( resource_id=f"arn:aws::::account/{account_id}", type="aws:account", link_collection=LinkCollection( simple_links=(SimpleLink(pred="account_id", obj=account_id), ), ), ) ] account_resources = [ resource for resource in graph_set.resources if resource.type == "aws:account" ] self.assertCountEqual(account_resources, expected_account_resources) ## Regions expected_region_resources = [ Resource( resource_id= f"arn:aws:::{account_id}:region/{region['RegionName']}", type="aws:region", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=region["RegionName"]), SimpleLink(pred="opt_in_status", obj=region["OptInStatus"]), ), resource_links=(ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ), ), ) for region in all_regions ] region_resources = [ resource for resource in graph_set.resources if resource.type == "aws:region" ] self.assertCountEqual(region_resources, expected_region_resources) ## IAM Policies expected_iam_policy_resources = [ Resource( resource_id=policy_1_arn, type="aws:iam:policy", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=policy_1_name), SimpleLink(pred="policy_id", obj=policy_1_id), SimpleLink(pred="default_version_id", obj="v1"), SimpleLink( pred="default_version_policy_document_text", obj= '{"Statement": [{"Action": "logs:CreateLogGroup", "Effect": "Allow", "Resource": "*"}], "Version": "2012-10-17"}', ), ), resource_links=(ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ), ), ) ] iam_policy_resources = [ resource for resource in graph_set.resources if resource.type == "aws:iam:policy" ] self.assertCountEqual(iam_policy_resources, expected_iam_policy_resources) ## IAM Roles expected_iam_role_resources = [ Resource( resource_id=role_1_arn, type="aws:iam:role", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=role_1_name), SimpleLink(pred="max_session_duration", obj=role_1_max_session_duration), SimpleLink(pred="description", obj=role_1_description), SimpleLink( pred="assume_role_policy_document_text", obj=policy_doc_dict_to_sorted_str( role_1_assume_role_policy_doc), ), ), multi_links=(MultiLink( pred="assume_role_policy_document", obj=LinkCollection( simple_links=(SimpleLink( pred="version", obj="2012-10-17"), ), multi_links=(MultiLink( pred="statement", obj=LinkCollection( simple_links=( SimpleLink(pred="effect", obj="Allow"), SimpleLink( pred="action", obj="sts:AssumeRole"), ), multi_links=(MultiLink( pred="principal", obj=LinkCollection( simple_links=(SimpleLink( pred="service", obj= "lambda.amazonaws.com", ), )), ), ), ), ), ), ), ), ), resource_links=(ResourceLink( pred="account", obj="arn:aws::::account/123456789012"), ), ), ) ] iam_role_resources = [ resource for resource in graph_set.resources if resource.type == "aws:iam:role" ] self.assertCountEqual(iam_role_resources, expected_iam_role_resources) ## Lambda functions expected_lambda_function_resources = [ Resource( resource_id=lambda_function_1_arn, type="aws:lambda:function", link_collection=LinkCollection( simple_links=( SimpleLink(pred="function_name", obj=lambda_function_1_name), SimpleLink(pred="runtime", obj=lambda_function_1_runtime), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), transient_resource_links=(ResourceLink( pred="role", obj="arn:aws:iam::123456789012:role/test_role_1" ), ), ), ), ] lambda_function_resources = [ resource for resource in graph_set.resources if resource.type == "aws:lambda:function" ] self.assertCountEqual(lambda_function_resources, expected_lambda_function_resources) ## EC2 VPCs expected_ec2_vpc_resources = [ Resource( resource_id=vpc_1_arn, type="aws:ec2:vpc", link_collection=LinkCollection( simple_links=( SimpleLink(pred="is_default", obj=True), SimpleLink(pred="cidr_block", obj=vpc_1_cidr), SimpleLink(pred="state", obj="available"), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] ec2_vpc_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:vpc" ] self.assertCountEqual(ec2_vpc_resources, expected_ec2_vpc_resources) ## EC2 VPC Flow Logs expected_ec2_vpc_flow_log_resources = [ Resource( resource_id=flow_log_1_arn, type="aws:ec2:flow-log", link_collection=LinkCollection( simple_links=( SimpleLink( pred="creation_time", obj=flow_log_1_creation_time.replace( tzinfo=datetime.timezone.utc). isoformat(), ), SimpleLink(pred="deliver_logs_status", obj="SUCCESS"), SimpleLink(pred="flow_log_status", obj="ACTIVE"), SimpleLink(pred="traffic_type", obj="ALL"), SimpleLink(pred="log_destination_type", obj="s3"), SimpleLink(pred="log_destination", obj=fixed_bucket_1_arn), SimpleLink( pred="log_format", obj= "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}", ), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), transient_resource_links=(TransientResourceLink( pred="vpc", obj=vpc_1_arn, ), ), ), ) ] ec2_vpc_flow_log_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:flow-log" ] self.assertCountEqual(ec2_vpc_flow_log_resources, expected_ec2_vpc_flow_log_resources) ## EC2 Subnets expected_ec2_subnet_resources = [ Resource( resource_id=subnet_1_arn, type="aws:ec2:subnet", link_collection=LinkCollection( simple_links=( SimpleLink(pred="cidr_block", obj=subnet_1_cidr), SimpleLink(pred="first_ip", obj=subnet_1_first_ip), SimpleLink(pred="last_ip", obj=subnet_1_last_ip), SimpleLink(pred="state", obj="available"), ), resource_links=( ResourceLink(pred="vpc", obj=vpc_1_arn), ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] ec2_subnet_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:subnet" ] self.assertCountEqual(ec2_subnet_resources, expected_ec2_subnet_resources) ## EC2 EBS Volumes expected_ec2_ebs_volume_resources = [ Resource( resource_id=ebs_volume_1_arn, type="aws:ec2:volume", link_collection=LinkCollection( simple_links=( SimpleLink(pred="availability_zone", obj=ebs_volume_1_az), SimpleLink( pred="create_time", obj=ebs_volume_1_create_time.replace( tzinfo=datetime.timezone.utc). isoformat(), ), SimpleLink(pred="size", obj=ebs_volume_1_size), SimpleLink(pred="state", obj="available"), SimpleLink(pred="volume_type", obj="gp2"), SimpleLink(pred="encrypted", obj=False), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] ec2_ebs_volume_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:volume" ] self.assertCountEqual(ec2_ebs_volume_resources, expected_ec2_ebs_volume_resources) ## S3 Buckets expected_s3_bucket_resources = [ Resource( resource_id=bucket_1_arn, type="aws:s3:bucket", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=bucket_1_name), SimpleLink( pred="creation_date", obj=bucket_1_creation_date.replace( tzinfo=datetime.timezone.utc). isoformat(), ), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] s3_bucket_resources = [ resource for resource in graph_set.resources if resource.type == "aws:s3:bucket" ] self.assertCountEqual(s3_bucket_resources, expected_s3_bucket_resources) expected_num_graph_set_resources = ( 0 + len(expected_account_resources) + len(expected_region_resources) + len(expected_iam_policy_resources) + len(expected_iam_role_resources) + len(expected_lambda_function_resources) + len(expected_ec2_ebs_volume_resources) + len(expected_ec2_subnet_resources) + len(expected_ec2_vpc_resources) + len(expected_ec2_vpc_flow_log_resources) + len(expected_s3_bucket_resources)) self.assertEqual(len(graph_set.resources), expected_num_graph_set_resources)