def setUp(self): resource_a1 = Resource( resource_id="123", type="test:a", link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]), ) resource_a2 = Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),) resource_b1 = Resource( resource_id="abc", type="test:b", link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-a", obj="123")]), ) resource_b2 = Resource( resource_id="def", type="test:b", link_collection=LinkCollection(simple_links=[SimpleLink(pred="name", obj="sue")]), ) resources = (resource_a1, resource_a2, resource_b1, resource_b2) self.validated_graph_set = ValidatedGraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=["test err 1", "test err 2"], )
def test_graph_content(self): expected_resources = ( Resource( resource_id="123", type="test:a", link_collection=LinkCollection( simple_links=(SimpleLink(pred="has-foo", obj="goo"),), ), ), Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),), Resource( resource_id="abc", type="test:b", link_collection=LinkCollection( simple_links=(SimpleLink(pred="has-a", obj="123"),), ), ), Resource( resource_id="def", type="test:b", link_collection=LinkCollection(simple_links=(SimpleLink(pred="name", obj="sue"),),), ), ) expected_errors = ["test err 1", "test err 2"] self.assertEqual(self.validated_graph_set.resources, expected_resources) self.assertEqual(self.validated_graph_set.errors, expected_errors)
def scan(cls: Type["TestResourceSpecB"], scan_accessor: Any) -> List[Resource]: resources = [ Resource(resource_id="abc", type_name=cls.type_name, links=[]), Resource(resource_id="def", type_name=cls.type_name, links=[]), ] return resources
def test_scan(self): scan_accessor = TestScanAccessor() graph_spec = GraphSpec( name="test-name", version="1", resource_spec_classes=(TestResourceSpecA, TestResourceSpecB), scan_accessor=scan_accessor, ) resources = graph_spec.scan() expected_resources = [ Resource(resource_id="123", type="a", link_collection=LinkCollection()), Resource(resource_id="456", type="a", link_collection=LinkCollection()), Resource(resource_id="abc", type="b", link_collection=LinkCollection()), Resource(resource_id="def", type="b", link_collection=LinkCollection()), ] self.assertEqual(resources, expected_resources)
def test_scan(self): scan_accessor = TestScanAccessor() graph_spec = GraphSpec( name="test-name", version="1", resource_spec_classes=(TestResourceSpecA, TestResourceSpecB), scan_accessor=scan_accessor, ) graph_set = graph_spec.scan() self.assertEqual(graph_set.name, "test-name") self.assertEqual(graph_set.version, "1") self.assertEqual(graph_set.errors, []) expected_resource_dicts = [ Resource(resource_id="123", type_name=TestResourceSpecA.type_name).to_dict(), Resource(resource_id="456", type_name=TestResourceSpecA.type_name).to_dict(), Resource(resource_id="abc", type_name=TestResourceSpecB.type_name).to_dict(), Resource(resource_id="def", type_name=TestResourceSpecB.type_name).to_dict(), ] self.assertCountEqual( [resource.to_dict() for resource in graph_set.resources], expected_resource_dicts)
def scan(cls: Type["TestResourceSpecB"], scan_accessor: Any) -> ResourceScanResult: resources = [ Resource(resource_id="abc", type_name=cls.type_name, links=[]), Resource(resource_id="def", type_name=cls.type_name, links=[]), ] return ResourceScanResult(resources=resources, stats=MultilevelCounter(), errors=[])
def scan(cls: Type["TestResourceSpecB"], scan_accessor: Any) -> List[Resource]: resources = [ Resource(resource_id="abc", type=cls.type_name, link_collection=LinkCollection()), Resource(resource_id="def", type=cls.type_name, link_collection=LinkCollection()), ] return resources
def test_scan(self): account_id = "123456789012" region_name = "us-east-1" session = boto3.Session() ec2_client = session.client("ec2", region_name=region_name) list_resp = ec2_client.describe_vpcs() present_vpcs = list_resp["Vpcs"] self.assertEqual(len(present_vpcs), 1) present_vpc_id = present_vpcs[0]["VpcId"] present_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{present_vpc_id}" create_resp = ec2_client.create_vpc(CidrBlock="10.0.0.0/16") created_vpc_id = create_resp["Vpc"]["VpcId"] created_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{created_vpc_id}" scan_accessor = AWSAccessor(session=session, account_id=account_id, region_name=region_name) resources = VPCResourceSpec.scan(scan_accessor=scan_accessor) expected_resources = [ Resource( resource_id=present_vpc_arn, type="aws:ec2:vpc", link_collection=LinkCollection( simple_links=( SimpleLink(pred="is_default", obj=True), SimpleLink(pred="cidr_block", obj="172.31.0.0/16"), SimpleLink(pred="state", obj="available"), ), resource_links=( ResourceLink(pred="account", obj="arn:aws::::account/123456789012"), ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"), ), ), ), Resource( resource_id=created_vpc_arn, type="aws:ec2:vpc", link_collection=LinkCollection( simple_links=( SimpleLink(pred="is_default", obj=False), SimpleLink(pred="cidr_block", obj="10.0.0.0/16"), SimpleLink(pred="state", obj="available"), ), resource_links=( ResourceLink(pred="account", obj="arn:aws::::account/123456789012"), ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"), ), ), ), ] self.assertEqual(resources, expected_resources)
def test_invalid_resources_dupes_same_class_conflicting_types_no_allow_clobber(self): resources = [ Resource(resource_id="123", type="test:a", link_collection=LinkCollection()), Resource(resource_id="123", type="test:b", link_collection=LinkCollection()), ] with self.assertRaises(UnmergableDuplicateResourceIdsFoundException): ValidatedGraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=[], )
def test_unknown_type_name(self): resources = [ Resource(resource_id="xyz", type="test:a", link_collection=LinkCollection()), Resource(resource_id="xyz", type="test:c", link_collection=LinkCollection()), ] with self.assertRaises(ResourceSpecClassNotFoundException): ValidatedGraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=[], )
def test_unknown_type_name(self): resources = [ Resource(resource_id="xyz", type_name="test:a"), Resource(resource_id="xyz", type_name="test:c"), ] with self.assertRaises(ResourceSpecClassNotFoundException): GraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=[], stats=MultilevelCounter(), )
def _list_from_aws_result_to_resources( cls: Type["AWSResourceSpec"], list_from_aws_result: ListFromAWSResult, context: Dict[str, str], ) -> List[Resource]: resources: List[Resource] = [] for arn, resource_dict in list_from_aws_result.resources.items(): try: links = cls.schema.parse(resource_dict, context) except Exception as ex: raise SchemaParseException( ( f"Error parsing {cls.__name__} : " f"{arn}:\n\nData: {resource_dict}\n\nError: {ex}" ) ) resource_account_id = arn.split(":")[4] if resource_account_id: if resource_account_id != "aws": account_link = ResourceLinkLink( pred="account", obj=f"arn:aws::::account/{resource_account_id}" ) links.append(account_link) resource_region_name = arn.split(":")[3] if resource_region_name: region_link = ResourceLinkLink( pred="region", obj=f"arn:aws:::{resource_account_id}:region/{resource_region_name}", ) links.append(region_link) resource = Resource(resource_id=arn, type_name=cls.get_full_type_name(), links=links) resources.append(resource) return resources
def test_invalid_resources_dupes_same_class_conflicting_types_no_allow_clobber( self): resources = [ Resource(resource_id="123", type_name="test:a"), Resource(resource_id="123", type_name="test:b"), ] with self.assertRaises(UnmergableDuplicateResourceIdsFoundException): GraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=[], stats=MultilevelCounter(), )
def test_schema_parse(self): resource_arn = "arn:aws:ec2:us-west-2:111122223333:vpc-endpoint-service/com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01" aws_resource_dict = { "ServiceType": [{"ServiceType": "Interface"}], "ServiceId": "vpce-svc-01234abcd5678ef01", "ServiceName": "com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01", "ServiceState": "Available", "AvailabilityZones": ["us-west-2a", "us-west-2b"], "AcceptanceRequired": True, "ManagesVpcEndpoints": False, "NetworkLoadBalancerArns": [ "arn:aws:elasticloadbalancing:us-west-2:111122223333:loadbalancer/net/splunk-hwf-lb/1a7ff9c18eeaaf9b" ], "BaseEndpointDnsNames": ["vpce-svc-01234abcd5678ef01.us-west-2.vpce.amazonaws.com"], "PrivateDnsNameConfiguration": {}, "Tags": [{"Key": "Name", "Value": "Splunk HEC"}], } link_collection = VpcEndpointServiceResourceSpec.schema.parse( data=aws_resource_dict, context={"account_id": "111122223333", "region": "us-west-2"} ) resource = Resource( resource_id=resource_arn, type=VpcEndpointServiceResourceSpec.type_name, link_collection=link_collection, ) expected_resource = Resource( resource_id="arn:aws:ec2:us-west-2:111122223333:vpc-endpoint-service/com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01", type="vpc-endpoint-service", link_collection=LinkCollection( simple_links=( SimpleLink(pred="service_type", obj="Interface"), SimpleLink( pred="service_name", obj="com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01", ), SimpleLink(pred="service_state", obj="Available"), SimpleLink(pred="acceptance_required", obj=True), SimpleLink(pred="availability_zones", obj="us-west-2a"), SimpleLink(pred="availability_zones", obj="us-west-2b"), ), tag_links=(TagLink(pred="Name", obj="Splunk HEC"),), ), ) self.assertEqual(resource, expected_resource)
def test_schema_parse(self): resource_arn = "arn:aws:iam:us-west-2:111122223333:account-password-policy/default" aws_resource_dict = { "MinimumPasswordLength": 12, "RequireSymbols": True, "RequireNumbers": True, "RequireUppercaseCharacters": True, "RequireLowercaseCharacters": True, "AllowUsersToChangePassword": True, "ExpirePasswords": True, "MaxPasswordAge": 90, "PasswordReusePrevention": 5, "HardExpiry": True, } link_collection = IAMAccountPasswordPolicyResourceSpec.schema.parse( data=aws_resource_dict, context={ "account_id": "111122223333", "region": "us-west-2" }) resource = Resource( resource_id=resource_arn, type=IAMAccountPasswordPolicyResourceSpec.type_name, link_collection=link_collection, ) expected_resource = Resource( resource_id= "arn:aws:iam:us-west-2:111122223333:account-password-policy/default", type="account-password-policy", link_collection=LinkCollection(simple_links=( SimpleLink(pred="minimum_password_length", obj=12), SimpleLink(pred="require_symbols", obj=True), SimpleLink(pred="require_numbers", obj=True), SimpleLink(pred="require_uppercase_characters", obj=True), SimpleLink(pred="require_lowercase_characters", obj=True), SimpleLink(pred="allow_users_to_change_password", obj=True), SimpleLink(pred="expire_passwords", obj=True), SimpleLink(pred="max_password_age", obj=90), SimpleLink(pred="password_reuse_prevention", obj=5), SimpleLink(pred="hard_expiry", obj=True), )), ) self.assertEqual(resource, expected_resource)
def test_scan(self): account_id = "123456789012" region_name = "us-east-1" session = boto3.Session() client = session.client("iam") oidc_url = "https://oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E" oidc_client_ids = ["sts.amazonaws.com"] oidc_thumbprints = ["9999999999999999999999999999999999999999"] _ = client.create_open_id_connect_provider( Url=oidc_url, ClientIDList=oidc_client_ids, ThumbprintList=oidc_thumbprints, ) scan_accessor = AWSAccessor(session=session, account_id=account_id, region_name=region_name) resources = IAMOIDCProviderResourceSpec.scan( scan_accessor=scan_accessor) expected_resources = [ Resource( resource_id= "arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E", type="aws:iam:oidc-provider", link_collection=LinkCollection( simple_links=( SimpleLink( pred="url", obj= "oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E", ), SimpleLink( pred="create_date", obj=resources[0].link_collection.simple_links[1]. obj, ), SimpleLink(pred="client_id", obj="sts.amazonaws.com"), SimpleLink( pred="thumbprint", obj="9999999999999999999999999999999999999999"), ), multi_links=None, tag_links=None, resource_links=(ResourceLink( pred="account", obj="arn:aws::::account/123456789012"), ), transient_resource_links=None, ), ) ] self.assertEqual(resources, expected_resources)
def test_orphaned_ref(self): resource_a1 = Resource(resource_id="123", type_name="test:a", links=[SimpleLink(pred="has-foo", obj="goo")]) resource_b1 = Resource( resource_id="abc", type_name="test:b", links=[ResourceLinkLink(pred="has-a", obj="456")]) resources = [resource_a1, resource_b1] graph_set = GraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=["test err 1", "test err 2"], stats=MultilevelCounter(), ) with self.assertRaises(GraphSetOrphanedReferencesException): graph_set.validate()
def create_resource(cls: Type["UnscannedAccountResourceSpec"], account_id: str, errors: List[str]) -> Resource: links: List[Link] = [] links.append(SimpleLink(pred="account_id", obj=account_id)) for error in errors: link = SimpleLink(pred="error", obj=error) links.append(link) return Resource( resource_id=cls.generate_arn("", "", account_id), type_name=cls.get_full_type_name(), links=links, )
def create_resource(cls: Type["UnscannedAccountResourceSpec"], account_id: str, errors: List[str]) -> Resource: links: List[Link] = [] links.append(SimpleLink(pred="account_id", obj=account_id)) if errors: error = "\n".join(errors) links.append( SimpleLink(pred="error", obj=f"{error} - {uuid.uuid4()}")) return Resource( resource_id=cls.generate_arn(resource_id=account_id), type_name=cls.get_full_type_name(), links=links, )
def test_orphaned_ref(self): resource_a1 = Resource( resource_id="123", type="test:a", link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]), ) resource_b1 = Resource( resource_id="abc", type="test:b", link_collection=LinkCollection(resource_links=[ResourceLink(pred="has-a", obj="456")]), ) resources = [resource_a1, resource_b1] graph_set = GraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=["test err 1", "test err 2"], ) with self.assertRaises(GraphSetOrphanedReferencesException): ValidatedGraphSet.from_graph_set(graph_set)
def setUp(self): resource_a1 = Resource(resource_id="123", type_name="test:a", links=[SimpleLink(pred="has-foo", obj="goo")]) resource_a2 = Resource(resource_id="456", type_name="test:a") resource_b1 = Resource( resource_id="abc", type_name="test:b", links=[ResourceLinkLink(pred="has-a", obj="123")]) resource_b2 = Resource(resource_id="def", type_name="test:b", links=[SimpleLink(pred="name", obj="sue")]) resources = [resource_a1, resource_a2, resource_b1, resource_b2] self.graph_set = GraphSet( name="test-name", version="1", start_time=1234, end_time=4567, resources=resources, errors=["test err 1", "test err 2"], stats=MultilevelCounter(), )
def test_scan(self): account_id = "123456789012" region_name = "us-east-1" session = boto3.Session() scan_accessor = AWSAccessor(session=session, account_id=account_id, region_name=region_name) resources = AccountResourceSpec.scan(scan_accessor=scan_accessor) expected_resources = [ Resource( resource_id="arn:aws::::account/123456789012", type="aws:account", link_collection=LinkCollection(simple_links=(SimpleLink( pred="account_id", obj="123456789012"), ), ), ) ] self.assertEqual(resources, expected_resources)
def test_scan(self): account_id = "123456789012" region_name = "us-east-1" session = boto3.Session() ec2_client = session.client("ec2", region_name=region_name) resp = ec2_client.create_volume(Size=1, AvailabilityZone="us-east-1a") create_time = resp["CreateTime"] created_volume_id = resp["VolumeId"] created_volume_arn = f"arn:aws:ec2:us-east-1:123456789012:volume/{created_volume_id}" scan_accessor = AWSAccessor(session=session, account_id=account_id, region_name=region_name) resources = EBSVolumeResourceSpec.scan(scan_accessor=scan_accessor) expected_resources = [ Resource( resource_id=created_volume_arn, type="aws:ec2:volume", link_collection=LinkCollection( simple_links=( SimpleLink(pred="availability_zone", obj="us-east-1a"), SimpleLink(pred="create_time", obj=create_time), SimpleLink(pred="size", obj=True), SimpleLink(pred="state", obj="available"), SimpleLink(pred="volume_type", obj="standard"), SimpleLink(pred="encrypted", obj=False), ), resource_links=( ResourceLink(pred="account", obj="arn:aws::::account/123456789012"), ResourceLink( pred="region", obj="arn:aws:::123456789012:region/us-east-1"), ), ), ) ] self.assertEqual(resources, expected_resources)
def test_schema_parse(self): resource_arn = "arn:aws:ec2:us-east-2:111122223333:route-table/rtb-099c7b032f2bbddda" aws_resource_dict = { "Associations": [ { "Main": False, "RouteTableAssociationId": "rtbassoc-069d59127bf10a728", "RouteTableId": "rtb-099c7b032f2bbddda", "SubnetId": "subnet-00f9fe55b9d7ca4fb", }, { "Main": False, "RouteTableAssociationId": "rtbassoc-07bfd170c4ece33c8", "RouteTableId": "rtb-099c7b032f2bbddda", "SubnetId": "subnet-0b98092b454c882cf", }, ], "PropagatingVgws": [], "RouteTableId": "rtb-099c7b032f2bbddda", "Routes": [ { "DestinationCidrBlock": "172.31.0.0/16", "GatewayId": "local", "Origin": "CreateRouteTable", "State": "active", }, { "DestinationCidrBlock": "0.0.0.0/0", "GatewayId": "igw-092e5ec1685fd0c0b", "Origin": "CreateRoute", "State": "active", }, { "DestinationPrefixListId": "pl-68a54001", "GatewayId": "vpce-0678bce2b63b8ad0f", "Origin": "CreateRoute", "State": "active", }, ], "VpcId": "vpc-03c33051f57d21ff0", "OwnerId": "210554966933", } links = EC2RouteTableResourceSpec.schema.parse(data=aws_resource_dict, context={ "account_id": "111122223333", "region": "us-west-2" }) resource = Resource(resource_id=resource_arn, type_name=EC2RouteTableResourceSpec.type_name, links=links) alti_resource_dict = resource.to_dict() expected_alti_resource_dict = { "type": "route-table", "links": [ { "pred": "route_table_id", "obj": "rtb-099c7b032f2bbddda", "type": "simple" }, { "pred": "vpc", "obj": "arn:aws:ec2:us-west-2:111122223333:vpc/vpc-03c33051f57d21ff0", "type": "resource_link", }, { "pred": "owner_id", "obj": "210554966933", "type": "simple" }, { "pred": "route", "obj": [ { "pred": "destination_cidr_block", "obj": "172.31.0.0/16", "type": "simple", }, { "pred": "gateway_id", "obj": "local", "type": "simple" }, { "pred": "origin", "obj": "CreateRouteTable", "type": "simple" }, { "pred": "state", "obj": "active", "type": "simple" }, ], "type": "multi", }, { "pred": "route", "obj": [ { "pred": "destination_cidr_block", "obj": "0.0.0.0/0", "type": "simple" }, { "pred": "gateway_id", "obj": "igw-092e5ec1685fd0c0b", "type": "simple" }, { "pred": "origin", "obj": "CreateRoute", "type": "simple" }, { "pred": "state", "obj": "active", "type": "simple" }, ], "type": "multi", }, { "pred": "route", "obj": [ { "pred": "destination_prefix_list_id", "obj": "pl-68a54001", "type": "simple", }, { "pred": "gateway_id", "obj": "vpce-0678bce2b63b8ad0f", "type": "simple" }, { "pred": "origin", "obj": "CreateRoute", "type": "simple" }, { "pred": "state", "obj": "active", "type": "simple" }, ], "type": "multi", }, { "pred": "association", "obj": [ { "pred": "main", "obj": False, "type": "simple" }, { "pred": "route_table_association_id", "obj": "rtbassoc-069d59127bf10a728", "type": "simple", }, { "pred": "route_table_id", "obj": "rtb-099c7b032f2bbddda", "type": "simple", }, { "pred": "subnet_id", "obj": "subnet-00f9fe55b9d7ca4fb", "type": "simple" }, ], "type": "multi", }, { "pred": "association", "obj": [ { "pred": "main", "obj": False, "type": "simple" }, { "pred": "route_table_association_id", "obj": "rtbassoc-07bfd170c4ece33c8", "type": "simple", }, { "pred": "route_table_id", "obj": "rtb-099c7b032f2bbddda", "type": "simple", }, { "pred": "subnet_id", "obj": "subnet-0b98092b454c882cf", "type": "simple" }, ], "type": "multi", }, ], } self.assertDictEqual(alti_resource_dict, expected_alti_resource_dict)
def test_valid_merge(self): resource_a1 = Resource(resource_id="123", type_name="test:a", links=[SimpleLink(pred="has-foo", obj="goo")]) resource_a2 = Resource(resource_id="456", type_name="test:a") resource_b1 = Resource( resource_id="abc", type_name="test:b", links=[ResourceLinkLink(pred="has-a", obj="123")]) resource_b2 = Resource(resource_id="def", type_name="test:b", links=[SimpleLink(pred="name", obj="sue")]) graph_set_1 = GraphSet( name="graph-1", version="1", start_time=10, end_time=20, resources=[resource_a1, resource_a2], errors=["errora1", "errora2"], stats=MultilevelCounter(), ) graph_set_2 = GraphSet( name="graph-1", version="1", start_time=15, end_time=25, resources=[resource_b1, resource_b2], errors=["errorb1", "errorb2"], stats=MultilevelCounter(), ) graph_set_1.merge(graph_set_2) self.assertEqual(graph_set_1.name, "graph-1") self.assertEqual(graph_set_1.version, "1") self.assertEqual(graph_set_1.start_time, 10) self.assertEqual(graph_set_1.end_time, 25) self.assertCountEqual(graph_set_1.errors, ["errora1", "errora2", "errorb1", "errorb2"]) expected_resource_dicts = [ { "type": "test:a", "links": [{ "pred": "has-foo", "obj": "goo", "type": "simple" }] }, { "type": "test:a" }, { "type": "test:b", "links": [{ "pred": "has-a", "obj": "123", "type": "resource_link" }] }, { "type": "test:b", "links": [{ "pred": "name", "obj": "sue", "type": "simple" }] }, ] resource_dicts = [ resource.to_dict() for resource in graph_set_1.resources ] self.assertCountEqual(expected_resource_dicts, resource_dicts)
def merge_resources(resource_id: str, resources: List[Resource]) -> Resource: """Merge multiple resources with the same id into one. This is permissible in two cases: 1) If all resources have the same value for 'get_full_type' and no key/values conflict they will be merged into a single resource by combining all key/values into a single resource 2) If all resources do not have the same value for 'get_full_type', if classes have the allow_clobber attribute set depending on the values a resource may be created. Args: resource_id: common resource id resources: list of Resources to merge Returns: merged Resource object Raises: UnmergableDuplicateResourceIdsFoundException if resources could not be merged. """ full_type_names = {resource.type for resource in resources} if len(full_type_names) > 1: # in this case conflicting resources have different full_type_names, this # can be a permissible if allow_clobber is used. Here we determine this by # building a dict where the keys are resource spec classes which have # appeared in the duplicate resources allow_clobber list and the # values are the classes where they appeared. If at the end in # this dict there are any values which match the complete list of # spec classes, we use the resource of that type spec_classes_resources: DefaultDict[ Type[ResourceSpec], List[Resource]] = defaultdict(list) allow_clobber_classes_classes: DefaultDict[ Type[ResourceSpec], Set[Type[ResourceSpec]]] = defaultdict(set) for resource in resources: full_type_name = resource.type spec_classes: List[ Type[ResourceSpec]] = ResourceSpec.get_by_full_type_name( full_type_name) for spec_class in spec_classes: spec_classes_resources[spec_class].append(resource) for allow_clobber_class in spec_class.allow_clobber: allow_clobber_classes_classes[allow_clobber_class].add( spec_class) all_spec_classes = set(spec_classes_resources.keys()) winning_class = None for allow_clobber_class, classes in allow_clobber_classes_classes.items( ): if len(classes) == len(all_spec_classes) - 1: winning_class = (all_spec_classes - classes).pop() if not winning_class: raise UnmergableDuplicateResourceIdsFoundException( (f"Multiple resources for {resource_id} with " f"different types that aren't clobberable: " f"{[resource.dict() for resource in resources]}")) resources = spec_classes_resources[winning_class] full_type_names = {resource.type for resource in resources} merged_resource_type_name = full_type_names.pop() merged_link_keys_links: Dict[str, Link] = {} for duplicate_resource in resources: for link in duplicate_resource.link_collection.get_links(): duplicate_link = merged_link_keys_links.get(link.pred) if duplicate_link: if type(duplicate_link) != type(link): raise UnmergableDuplicateResourceIdsFoundException( f"Conflicting link types {type(link)}, {type(duplicate_link)} found in duplicate #s {resources}" ) if duplicate_link.obj != link.obj: raise UnmergableDuplicateResourceIdsFoundException( f"Conflicting link values {link.obj}, {duplicate_link.obj} found in duplicate #s {resources}" ) else: merged_link_keys_links[link.pred] = link merged_links = list(merged_link_keys_links.values()) link_collection = LinkCollection.from_links(merged_links) return Resource( resource_id=resource_id, type=merged_resource_type_name, link_collection=link_collection, )
def test_schema_parse(self): self.maxDiff = None resource_arn = "arn:aws:ec2:us-east-2:111122223333:transit-gateway-vpc-attachment/tgw-attach-09ece7878ee9ab7a4" aws_resource_dict = { "TransitGatewayAttachmentId": "tgw-attach-09ece7878ee9ab7a4", "TransitGatewayId": "tgw-086b599bebfee5d40", "VpcId": "vpc-01e8457e8c00c40a7", "VpcOwnerId": "123456789012", "State": "available", "SubnetIds": ["subnet-07697f82fe4c6a8d6", "subnet-0396137c18d6c30ef"], "CreationTime": datetime.datetime(2019, 8, 23, 15, 59, 46, tzinfo=tzutc()), "Options": { "DnsSupport": "enable", "Ipv6Support": "disable" }, "Tags": [{ "Key": "Name", "Value": "customer-dev-tgw-attachment" }], } link_collection = TransitGatewayVpcAttachmentResourceSpec.schema.parse( data=aws_resource_dict, context={ "account_id": "111122223333", "region": "us-west-2" }) resource = Resource( resource_id=resource_arn, type=TransitGatewayVpcAttachmentResourceSpec.type_name, link_collection=link_collection, ) expected_resource = Resource( resource_id= "arn:aws:ec2:us-east-2:111122223333:transit-gateway-vpc-attachment/tgw-attach-09ece7878ee9ab7a4", type="transit-gateway-vpc-attachment", link_collection=LinkCollection(simple_links=( SimpleLink(pred="transit_gateway_attachment_id", obj="tgw-attach-09ece7878ee9ab7a4"), SimpleLink(pred="transit_gateway_id", obj="tgw-086b599bebfee5d40"), SimpleLink(pred="vpc_id", obj="vpc-01e8457e8c00c40a7"), SimpleLink(pred="vpc_owner_id", obj="123456789012"), SimpleLink(pred="state", obj="available"), SimpleLink( pred="creation_time", obj=datetime.datetime(2019, 8, 23, 15, 59, 46, tzinfo=tzutc()), ), SimpleLink(pred="subnet_id", obj="subnet-07697f82fe4c6a8d6"), SimpleLink(pred="subnet_id", obj="subnet-0396137c18d6c30ef"), SimpleLink(pred="dns_support", obj="enable"), SimpleLink(pred="ipv6_support", obj="disable"), ), ), ) self.assertEqual(resource, expected_resource)
def test_schema_parse(self): resource_arn = "arn:aws:ec2:us-west-2:111122223333:vpc-endpoint-service/com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01" aws_resource_dict = { "ServiceType": [{ "ServiceType": "Interface" }], "ServiceId": "vpce-svc-01234abcd5678ef01", "ServiceName": "com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01", "ServiceState": "Available", "AvailabilityZones": ["us-west-2a", "us-west-2b"], "AcceptanceRequired": True, "ManagesVpcEndpoints": False, "NetworkLoadBalancerArns": [ "arn:aws:elasticloadbalancing:us-west-2:111122223333:loadbalancer/net/splunk-hwf-lb/1a7ff9c18eeaaf9b" ], "BaseEndpointDnsNames": ["vpce-svc-01234abcd5678ef01.us-west-2.vpce.amazonaws.com"], "PrivateDnsNameConfiguration": {}, "Tags": [{ "Key": "Name", "Value": "Splunk HEC" }], } links = VpcEndpointServiceResourceSpec.schema.parse( data=aws_resource_dict, context={ "account_id": "111122223333", "region": "us-west-2" }) resource = Resource( resource_id=resource_arn, type_name=VpcEndpointServiceResourceSpec.type_name, links=links, ) alti_resource_dict = resource.to_dict() expected_alti_resource_dict = { "type": "vpc-endpoint-service", "links": [ { "pred": "service_type", "obj": "Interface", "type": "simple" }, { "pred": "service_name", "obj": "com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01", "type": "simple", }, { "pred": "service_state", "obj": "Available", "type": "simple" }, { "pred": "acceptance_required", "obj": True, "type": "simple" }, { "pred": "availability_zones", "obj": "us-west-2a", "type": "simple" }, { "pred": "availability_zones", "obj": "us-west-2b", "type": "simple" }, { "pred": "Name", "obj": "Splunk HEC", "type": "tag" }, ], } self.assertDictEqual(alti_resource_dict, expected_alti_resource_dict)
def test(self): with tempfile.TemporaryDirectory() as temp_dir: resource_region_name = "us-east-1" # get moto"s enabled regions ec2_client = boto3.client("ec2", region_name=resource_region_name) all_regions = ec2_client.describe_regions( Filters=[{ "Name": "opt-in-status", "Values": ["opt-in-not-required", "opted-in"] }])["Regions"] account_id = get_account_id() all_region_names = tuple(region["RegionName"] for region in all_regions) enabled_region_names = tuple( region["RegionName"] for region in all_regions if region["OptInStatus"] != "not-opted-in") delete_vpcs(all_region_names) # add a diverse set of resources which are supported by moto ## dynamodb # TODO moto is not returning TableId in list/describe # dynamodb_table_1_arn = create_dynamodb_table( # name="test_table_1", # attr_name="test_hash_key_attr_1", # attr_type="S", # key_type="HASH", # region_name=region_name, # ) ## s3 bucket_1_name = "test_bucket" bucket_1_arn, bucket_1_creation_date = create_bucket( name=bucket_1_name, account_id=account_id, region_name=resource_region_name) ## ec2 vpc_1_cidr = "10.0.0.0/16" vpc_1_id = create_vpc(cidr_block=vpc_1_cidr, region_name=resource_region_name) vpc_1_arn = VPCResourceSpec.generate_arn( resource_id=vpc_1_id, account_id=account_id, region=resource_region_name) subnet_1_cidr = "10.0.0.0/24" subnet_1_cidr_network = ipaddress.IPv4Network(subnet_1_cidr, strict=False) subnet_1_first_ip, subnet_1_last_ip = ( int(subnet_1_cidr_network[0]), int(subnet_1_cidr_network[-1]), ) subnet_1_id = create_subnet(cidr_block=subnet_1_cidr, vpc_id=vpc_1_id, region_name=resource_region_name) subnet_1_arn = SubnetResourceSpec.generate_arn( resource_id=subnet_1_id, account_id=account_id, region=resource_region_name) fixed_bucket_1_arn = f"arn:aws:s3:::{bucket_1_name}" flow_log_1_id, flow_log_1_creation_time = create_flow_log( vpc_id=vpc_1_id, dest_bucket_arn=fixed_bucket_1_arn, region_name=resource_region_name, ) flow_log_1_arn = FlowLogResourceSpec.generate_arn( resource_id=flow_log_1_id, account_id=account_id, region=resource_region_name) ebs_volume_1_size = 128 ebs_volume_1_az = f"{resource_region_name}a" ebs_volume_1_arn, ebs_volume_1_create_time = create_volume( size=ebs_volume_1_size, az=ebs_volume_1_az, region_name=resource_region_name) ## iam policy_1_name = "test_policy_1" policy_1_arn, policy_1_id = create_iam_policy( name=policy_1_name, policy_doc={ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "logs:CreateLogGroup", "Resource": "*" }, ], }, ) role_1_name = "test_role_1" role_1_assume_role_policy_doc = { "Version": "2012-10-17", "Statement": [{ "Action": "sts:AssumeRole", "Effect": "Allow", "Principal": { "Service": "lambda.amazonaws.com" }, "Sid": "", }], } role_1_description = "Test Role 1" role_1_max_session_duration = 3600 role_1_arn = create_iam_role( name=role_1_name, assume_role_policy_doc=role_1_assume_role_policy_doc, description=role_1_description, max_session_duration=role_1_max_session_duration, ) ## lambda lambda_function_1_name = "test_lambda_function_1" lambda_function_1_runtime = "python3.7" lambda_function_1_handler = "lambda_function.lambda_handler" lambda_function_1_description = "Test Lambda Function 1" lambda_function_1_timeout = 30 lambda_function_1_memory_size = 256 lambda_function_1_arn = create_lambda_function( name=lambda_function_1_name, runtime=lambda_function_1_runtime, role_name=role_1_arn, handler=lambda_function_1_handler, description=lambda_function_1_description, timeout=lambda_function_1_timeout, memory_size=lambda_function_1_memory_size, publish=False, region_name=resource_region_name, ) # scan test_scan_id = "test_scan_id" aws_config = AWSConfig( artifact_path=temp_dir, pruner_max_age_min=4320, graph_name="alti", concurrency=ConcurrencyConfig(max_account_scan_threads=1, max_svc_scan_threads=1, max_account_scan_tries=2), scan=ScanConfig( accounts=(), regions=(), scan_sub_accounts=False, preferred_account_scan_regions=( "us-west-1", "us-west-2", "us-east-1", "us-east-2", ), ), accessor=Accessor( credentials_cache=AWSCredentialsCache(cache={}), multi_hop_accessors=[], cache_creds=True, ), write_master_json=True, ) resource_spec_classes = ( # DynamoDbTableResourceSpec, TODO moto EBSVolumeResourceSpec, FlowLogResourceSpec, IAMPolicyResourceSpec, IAMRoleResourceSpec, LambdaFunctionResourceSpec, S3BucketResourceSpec, SubnetResourceSpec, VPCResourceSpec, ) muxer = LocalAWSScanMuxer( scan_id=test_scan_id, config=aws_config, resource_spec_classes=resource_spec_classes, ) with unittest.mock.patch( "altimeter.aws.scan.account_scanner.get_all_enabled_regions" ) as mock_get_all_enabled_regions: mock_get_all_enabled_regions.return_value = enabled_region_names aws2n_result = aws2n( scan_id=test_scan_id, config=aws_config, muxer=muxer, load_neptune=False, ) graph_set = GraphSet.from_json_file( Path(aws2n_result.json_path)) self.assertEqual(len(graph_set.errors), 0) self.assertEqual(graph_set.name, "alti") self.assertEqual(graph_set.version, "2") # now check each resource type self.maxDiff = None ## Accounts expected_account_resources = [ Resource( resource_id=f"arn:aws::::account/{account_id}", type="aws:account", link_collection=LinkCollection( simple_links=(SimpleLink(pred="account_id", obj=account_id), ), ), ) ] account_resources = [ resource for resource in graph_set.resources if resource.type == "aws:account" ] self.assertCountEqual(account_resources, expected_account_resources) ## Regions expected_region_resources = [ Resource( resource_id= f"arn:aws:::{account_id}:region/{region['RegionName']}", type="aws:region", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=region["RegionName"]), SimpleLink(pred="opt_in_status", obj=region["OptInStatus"]), ), resource_links=(ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ), ), ) for region in all_regions ] region_resources = [ resource for resource in graph_set.resources if resource.type == "aws:region" ] self.assertCountEqual(region_resources, expected_region_resources) ## IAM Policies expected_iam_policy_resources = [ Resource( resource_id=policy_1_arn, type="aws:iam:policy", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=policy_1_name), SimpleLink(pred="policy_id", obj=policy_1_id), SimpleLink(pred="default_version_id", obj="v1"), SimpleLink( pred="default_version_policy_document_text", obj= '{"Statement": [{"Action": "logs:CreateLogGroup", "Effect": "Allow", "Resource": "*"}], "Version": "2012-10-17"}', ), ), resource_links=(ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ), ), ) ] iam_policy_resources = [ resource for resource in graph_set.resources if resource.type == "aws:iam:policy" ] self.assertCountEqual(iam_policy_resources, expected_iam_policy_resources) ## IAM Roles expected_iam_role_resources = [ Resource( resource_id=role_1_arn, type="aws:iam:role", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=role_1_name), SimpleLink(pred="max_session_duration", obj=role_1_max_session_duration), SimpleLink(pred="description", obj=role_1_description), SimpleLink( pred="assume_role_policy_document_text", obj=policy_doc_dict_to_sorted_str( role_1_assume_role_policy_doc), ), ), multi_links=(MultiLink( pred="assume_role_policy_document", obj=LinkCollection( simple_links=(SimpleLink( pred="version", obj="2012-10-17"), ), multi_links=(MultiLink( pred="statement", obj=LinkCollection( simple_links=( SimpleLink(pred="effect", obj="Allow"), SimpleLink( pred="action", obj="sts:AssumeRole"), ), multi_links=(MultiLink( pred="principal", obj=LinkCollection( simple_links=(SimpleLink( pred="service", obj= "lambda.amazonaws.com", ), )), ), ), ), ), ), ), ), ), resource_links=(ResourceLink( pred="account", obj="arn:aws::::account/123456789012"), ), ), ) ] iam_role_resources = [ resource for resource in graph_set.resources if resource.type == "aws:iam:role" ] self.assertCountEqual(iam_role_resources, expected_iam_role_resources) ## Lambda functions expected_lambda_function_resources = [ Resource( resource_id=lambda_function_1_arn, type="aws:lambda:function", link_collection=LinkCollection( simple_links=( SimpleLink(pred="function_name", obj=lambda_function_1_name), SimpleLink(pred="runtime", obj=lambda_function_1_runtime), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), transient_resource_links=(ResourceLink( pred="role", obj="arn:aws:iam::123456789012:role/test_role_1" ), ), ), ), ] lambda_function_resources = [ resource for resource in graph_set.resources if resource.type == "aws:lambda:function" ] self.assertCountEqual(lambda_function_resources, expected_lambda_function_resources) ## EC2 VPCs expected_ec2_vpc_resources = [ Resource( resource_id=vpc_1_arn, type="aws:ec2:vpc", link_collection=LinkCollection( simple_links=( SimpleLink(pred="is_default", obj=True), SimpleLink(pred="cidr_block", obj=vpc_1_cidr), SimpleLink(pred="state", obj="available"), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] ec2_vpc_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:vpc" ] self.assertCountEqual(ec2_vpc_resources, expected_ec2_vpc_resources) ## EC2 VPC Flow Logs expected_ec2_vpc_flow_log_resources = [ Resource( resource_id=flow_log_1_arn, type="aws:ec2:flow-log", link_collection=LinkCollection( simple_links=( SimpleLink( pred="creation_time", obj=flow_log_1_creation_time.replace( tzinfo=datetime.timezone.utc). isoformat(), ), SimpleLink(pred="deliver_logs_status", obj="SUCCESS"), SimpleLink(pred="flow_log_status", obj="ACTIVE"), SimpleLink(pred="traffic_type", obj="ALL"), SimpleLink(pred="log_destination_type", obj="s3"), SimpleLink(pred="log_destination", obj=fixed_bucket_1_arn), SimpleLink( pred="log_format", obj= "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}", ), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), transient_resource_links=(TransientResourceLink( pred="vpc", obj=vpc_1_arn, ), ), ), ) ] ec2_vpc_flow_log_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:flow-log" ] self.assertCountEqual(ec2_vpc_flow_log_resources, expected_ec2_vpc_flow_log_resources) ## EC2 Subnets expected_ec2_subnet_resources = [ Resource( resource_id=subnet_1_arn, type="aws:ec2:subnet", link_collection=LinkCollection( simple_links=( SimpleLink(pred="cidr_block", obj=subnet_1_cidr), SimpleLink(pred="first_ip", obj=subnet_1_first_ip), SimpleLink(pred="last_ip", obj=subnet_1_last_ip), SimpleLink(pred="state", obj="available"), ), resource_links=( ResourceLink(pred="vpc", obj=vpc_1_arn), ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] ec2_subnet_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:subnet" ] self.assertCountEqual(ec2_subnet_resources, expected_ec2_subnet_resources) ## EC2 EBS Volumes expected_ec2_ebs_volume_resources = [ Resource( resource_id=ebs_volume_1_arn, type="aws:ec2:volume", link_collection=LinkCollection( simple_links=( SimpleLink(pred="availability_zone", obj=ebs_volume_1_az), SimpleLink( pred="create_time", obj=ebs_volume_1_create_time.replace( tzinfo=datetime.timezone.utc). isoformat(), ), SimpleLink(pred="size", obj=ebs_volume_1_size), SimpleLink(pred="state", obj="available"), SimpleLink(pred="volume_type", obj="gp2"), SimpleLink(pred="encrypted", obj=False), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] ec2_ebs_volume_resources = [ resource for resource in graph_set.resources if resource.type == "aws:ec2:volume" ] self.assertCountEqual(ec2_ebs_volume_resources, expected_ec2_ebs_volume_resources) ## S3 Buckets expected_s3_bucket_resources = [ Resource( resource_id=bucket_1_arn, type="aws:s3:bucket", link_collection=LinkCollection( simple_links=( SimpleLink(pred="name", obj=bucket_1_name), SimpleLink( pred="creation_date", obj=bucket_1_creation_date.replace( tzinfo=datetime.timezone.utc). isoformat(), ), ), resource_links=( ResourceLink( pred="account", obj=f"arn:aws::::account/{account_id}"), ResourceLink( pred="region", obj= f"arn:aws:::{account_id}:region/{resource_region_name}", ), ), ), ) ] s3_bucket_resources = [ resource for resource in graph_set.resources if resource.type == "aws:s3:bucket" ] self.assertCountEqual(s3_bucket_resources, expected_s3_bucket_resources) expected_num_graph_set_resources = ( 0 + len(expected_account_resources) + len(expected_region_resources) + len(expected_iam_policy_resources) + len(expected_iam_role_resources) + len(expected_lambda_function_resources) + len(expected_ec2_ebs_volume_resources) + len(expected_ec2_subnet_resources) + len(expected_ec2_vpc_resources) + len(expected_ec2_vpc_flow_log_resources) + len(expected_s3_bucket_resources)) self.assertEqual(len(graph_set.resources), expected_num_graph_set_resources)
def test_valid_merge(self): resource_a1 = Resource( resource_id="123", type="test:a", link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]), ) resource_a2 = Resource(resource_id="456", type="test:a", link_collection=LinkCollection()) resource_b1 = Resource( resource_id="abc", type="test:b", link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-a", obj="123")]), ) resource_b2 = Resource( resource_id="def", type="test:b", link_collection=LinkCollection(simple_links=[SimpleLink(pred="name", obj="sue")]), ) graph_set_1 = GraphSet( name="graph-1", version="1", start_time=10, end_time=20, resources=[resource_a1, resource_a2], errors=["errora1", "errora2"], ) graph_set_2 = GraphSet( name="graph-1", version="1", start_time=15, end_time=25, resources=[resource_b1, resource_b2], errors=["errorb1", "errorb2"], ) merged_graph_set = ValidatedGraphSet.from_graph_sets([graph_set_1, graph_set_2]) self.assertEqual(merged_graph_set.name, "graph-1") self.assertEqual(merged_graph_set.version, "1") self.assertEqual(merged_graph_set.start_time, 10) self.assertEqual(merged_graph_set.end_time, 25) self.assertCountEqual(merged_graph_set.errors, ["errora1", "errora2", "errorb1", "errorb2"]) expected_resources = ( Resource( resource_id="123", type="test:a", link_collection=LinkCollection( simple_links=(SimpleLink(pred="has-foo", obj="goo"),), ), ), Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),), Resource( resource_id="abc", type="test:b", link_collection=LinkCollection( simple_links=(SimpleLink(pred="has-a", obj="123"),), ), ), Resource( resource_id="def", type="test:b", link_collection=LinkCollection(simple_links=(SimpleLink(pred="name", obj="sue"),),), ), ) expected_errors = ["errora1", "errora2", "errorb1", "errorb2"] self.assertCountEqual(merged_graph_set.resources, expected_resources) self.assertCountEqual(merged_graph_set.errors, expected_errors)