def test_dict_conversion(cloudwanderer_resource): assert dict(cloudwanderer_resource) == { "urn": URN( cloud_name="aws", account_id="111111111111", region="eu-west-1", service="ec2", resource_type="vpc", resource_id_parts=["vpc-111111"], ), "relationships": [ URN( cloud_name="aws", account_id="unknown", region="unknown", service="ec2", resource_type="vpc", resource_id_parts=["vpc-111111"], ) ], "dependent_resource_urns": [], "parent_urn": None, "cloudwanderer_metadata": { "VpcId": "vpc-111111" }, "discovery_time": datetime(2021, 10, 23, 0, 0), "vpc_id": "vpc-111111", }
def test_write_resources(cloudwanderer_aws, aws_interface, default_test_discovery_actions): create_iam_role() create_s3_buckets(regions=["eu-west-2", "us-east-1"]) aws_interface.get_resource_discovery_actions = MagicMock( return_value=default_test_discovery_actions) thread_results = list( cloudwanderer_aws.write_resources_concurrently( concurrency=2, cloud_interface_generator=lambda: aws_interface, storage_connector_generator=lambda: [MemoryStorageConnector()], )) connector_results = [ resource_record for result in thread_results for connector in result.storage_connectors for resource_record in connector.read_all() ] result_summary = set([(URN.from_string(result["urn"]).region, URN.from_string(result["urn"]).resource_type) for result in connector_results]) assert result_summary == { ("eu-west-2", "bucket"), ("eu-west-2", "vpc"), ("us-east-1", "bucket"), ("us-east-1", "role"), ("us-east-1", "role_policy"), ("us-east-1", "vpc"), }
def iam_role(): return CloudWandererResource( urn=URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], ), resource_data={ "RoleName": "test-role", "InlinePolicyAttachments": [{ "PolicyNames": ["test-role"] }] }, dependent_resource_urns=[ URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role_policy", resource_id_parts=["test-role", "test-role-policy-1"], ) ], )
def iam_instance_profile(): return CloudWandererResource( urn=URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="instance_profile", resource_id_parts=["my-test-profile"], ), resource_data={}, dependent_resource_urns=[], relationships=[ Relationship( partial_urn=PartialUrn( cloud_name="aws", account_id="unknown", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], ), direction=RelationshipDirection.INBOUND, ) ], )
def test_get_resources_specific_resource(cloud_wanderer: CloudWanderer): cloud_wanderer.write_resources(service_resource_types=[ServiceResourceType(service="ec2", resource_type="vpc")]) cloud_wanderer.cloud_interface.get_resource_discovery_actions.assert_called() cloud_wanderer.cloud_interface.get_resources.assert_called_with( region="eu-west-1", service_name="ec2", resource_type="vpc", service_resource_type_filters=ANY ) cloud_wanderer.storage_connectors[0].write_resource.assert_called_with( CloudWandererResource( urn=URN( cloud_name="aws", account_id="111111111111", region="eu-west-1", service="ec2", resource_type="vpc", resource_id_parts=["vpc-11111111"], ), dependent_resource_urns=[], resource_data={}, ) ) cloud_wanderer.storage_connectors[0].delete_resource_of_type_in_account_region.assert_called_with( cloud_name="aws", account_id="111111111111", region="eu-west-1", service="ec2", resource_type="vpc", cutoff=datetime.datetime(1986, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), )
def get_inferred_ec2_instances(cloudwanderer_boto3_session): vpcs = list(cloudwanderer_boto3_session.resource("ec2").vpcs.all()) return [ CloudWandererResource( urn=URN( account_id="111111111111", region="eu-west-2", service="ec2", resource_type="instance", resource_id_parts=[instance.instance_id], ), resource_data=instance.meta.data, relationships=[ Relationship( partial_urn=PartialUrn( cloud_name="aws", account_id="unknown", region="eu-west-2", service="ec2", resource_type="vpc", resource_id_parts=[vpcs[0].vpc_id], ), direction=RelationshipDirection.INBOUND, ) ], ) for instance in cloudwanderer_boto3_session.resource( "ec2").instances.all() ]
def urn(): return URN( account_id="111111111111", region="eu-west-1", service="ec2", resource_type="vpc", resource_id_parts=["vpc-111111"], )
def partial_urn(): return URN( account_id="unknown", region="unknown", service="ec2", resource_type="vpc", resource_id_parts=["vpc-111111"], )
def cloudwanderer_resource_urn() -> URN: return URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], )
def test_get_urn(service_resource_s3_bucket): urn = service_resource_s3_bucket.get_urn() assert urn == URN( account_id="111111111111", region="eu-west-1", service="s3", resource_type="bucket", resource_id_parts=["my_bucket"], )
def test_dependent_resource_urn(loaded_memory_connector): result = next( loaded_memory_connector.read_resources(urn=URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role_policy", resource_id_parts=["test-role", "test-role-policy"], )), None, ) assert isinstance(result.parent_urn, URN) assert result.parent_urn == URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], )
def test_get_urn_regex_pattern(iam_service): single_iam_policy = list(iam_service.collection("policy").all())[0] single_iam_policy.fetch_secondary_attributes() assert single_iam_policy.get_urn() == URN( cloud_name="aws", account_id="aws", region="us-east-1", service="iam", resource_type="policy", resource_id_parts=["arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy"], )
def inferred_ec2_vpcs(cloudwanderer_boto3_session): return [ CloudWandererResource( urn=URN( account_id="111111111111", region="eu-west-2", service="ec2", resource_type="vpc", resource_id_parts=[vpc.vpc_id], ), resource_data=vpc.meta.data, ) for vpc in cloudwanderer_boto3_session.resource("ec2").vpcs.all() ]
def get_inferred_ec2_instances(cloudwanderer_boto3_session): return [ CloudWandererResource( urn=URN( account_id="111111111111", region="eu-west-2", service="ec2", resource_type="instance", resource_id_parts=[instance.instance_id], ), resource_data=instance.meta.data, ) for instance in cloudwanderer_boto3_session.resource( "ec2").instances.all() ]
def iam_role_policies(): return [ CloudWandererResource( urn=URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role_policy", resource_id_parts=["test-role", "test-role-policy-1"], ), resource_data={}, parent_urn=URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], ), ), CloudWandererResource( urn=URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role_policy", resource_id_parts=["test-role", "test-role-policy-2"], ), resource_data={}, parent_urn=URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], ), ), ]
def cloud_wanderer() -> CloudWanderer: mock_storage_connector = MagicMock(**{}) mock_cloud_interface = MagicMock( spec_set=CloudWandererAWSInterface, **{ "get_resource_discovery_actions.return_value": [ ActionSet( get_urns=[ PartialUrn( cloud_name="aws", account_id="111111111111", region="eu-west-1", service="ec2", resource_type="vpc", resource_id_parts=["ALL"], ) ], delete_urns=[ PartialUrn( cloud_name="aws", account_id="111111111111", region="eu-west-1", service="ec2", resource_type="vpc", resource_id_parts=["ALL"], ) ], ) ], "get_resources.return_value": [ CloudWandererResource( URN( cloud_name="aws", account_id="111111111111", region="eu-west-1", service="ec2", resource_type="vpc", resource_id_parts=["vpc-11111111"], ), resource_data={}, discovery_time=datetime.datetime(1986, 1, 1, tzinfo=datetime.timezone.utc), ) ], } ) return CloudWanderer(storage_connectors=[mock_storage_connector], cloud_interface=mock_cloud_interface)
def test_write_resource_and_attribute(dynamodb_connnector, iam_role): dynamodb_connnector.write_resource(resource=iam_role) result = dynamodb_connnector.read_resource(urn=iam_role.urn) assert result.urn == iam_role.urn assert result.role_name == "test-role" logger.info(result.cloudwanderer_metadata.resource_data) assert result.inline_policy_attachments == [{"PolicyNames": ["test-role"]}] assert result.dependent_resource_urns == [ URN( account_id="111111111111", region="us-east-1", service="iam", resource_type="role_policy", resource_id_parts=["test-role", "test-role-policy-1"], ) ]
def test_resource_urn(loaded_dynamodb_connector): result: List[CloudWandererResource] = list( loaded_dynamodb_connector.read_resources(urn=URN( cloud_name="aws", account_id="111111111111", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], ))) expected_urns = [ { "account_id": "111111111111", "region": "us-east-1", "service": "iam", "resource_type": "role", "resource_id": "test-role", }, ] not_expected_urns = [ { "account_id": "111111111111", "region": "us-east-1", "service": "ec2", "resource_type": "vpc" }, { "account_id": "111111111111", "region": "eu-west-2", "service": "ec2", "resource_type": "vpc" }, { "account_id": "123456789012", "region": "us-east-1", "service": "ec2", "resource_type": "vpc" }, { "account_id": "123456789012", "region": "eu-west-2", "service": "ec2", "resource_type": "vpc" }, { "account_id": "123456789012", "region": "us-east-1", "service": "iam", "resource_type": "role", "resource_id": "test-role", }, { "account_id": "123456789012", "region": "eu-west-2", "service": "s3", "resource_type": "bucket" }, { "account_id": "123456789012", "region": "us-east-1", "service": "s3", "resource_type": "bucket" }, { "account_id": "111111111111", "region": "eu-west-2", "service": "s3", "resource_type": "bucket" }, { "account_id": "111111111111", "region": "us-east-1", "service": "s3", "resource_type": "bucket" }, ] assert_has_matching_urns(result, expected_urns) assert_does_not_have_matching_urns(result, not_expected_urns)
def test_all_custom_resources(file_name, aws_interface): with open(file_name) as f: test_spec = json.load(f) mock, paginator_side_effects = build_mock(test_spec["mockData"]) mock_client = MagicMock( **{ **{ "meta": aws_interface.cloudwanderer_boto3_session.client(test_spec["service"]).meta, "get_caller_identity.return_value": { "UserId": "111111111111111111111", "Account": "0123456789012", "Arn": "arn:aws:iam::0123456789012:user/CloudWanderer", }, }, **mock, } ) aws_interface.cloudwanderer_boto3_session.client = MagicMock(return_value=mock_client) if "getResource" in test_spec: urn = URN.from_string(test_spec["getResource"]["urn"]) logger.info("Getting resource %s", urn) try: result = prepare_for_comparison(aws_interface.get_resource(urn=urn)) except botocore.model.NoShapeFoundError as ex: raise ValueError( "Boto3 raised a NoShapeFoundError, this suggests your test has the wrong service name specified" ) from ex if "getResources" in test_spec: get_resources = test_spec["getResources"] logger.info("Getting resources %s", get_resources) try: result = prepare_for_comparison( aws_interface.get_resources( service_name=get_resources["serviceName"], resource_type=get_resources["resourceType"], region=get_resources["region"], ) ) except NotImplementedError as ex: raise ValueError( "Boto3 raised a NotImplementedError, usually this means " "you forgot to wrap your paginate.return_results in a list or " "include the key at the top level (e.g. Instances)" ) from ex except botocore.model.NoShapeFoundError as ex: raise ValueError( "Boto3 raised a NoShapeFoundError, this suggests your test has the wrong service name specified" ) from ex compare_list_of_dicts_allow_any( test_spec["expectedResults"], [dict(x) for x in result], allow_partial_match_first=True ) for method_path, calls in test_spec["expectedCalls"].items(): logger.info("Assert calls %s on %s", calls, method_path) method = mock_client for attr in method_path.split("."): index_match = re.search(r"\[(?P<index>\d)\](?P<attr>.*)", attr) if index_match: # This will only work if it's an index of `get_paginator.side_effect` index = index_match.groupdict()["index"] attr = index_match.groupdict()["attr"] method = paginator_side_effects[int(index)] if not attr: continue method = getattr(method, attr) logger.info("Assert %s calls on %s", calls, method_path) logger.info([call(*call_dict["args"], **call_dict["kwargs"]) for call_dict in calls]) method.assert_has_calls([call(*call_dict["args"], **call_dict["kwargs"]) for call_dict in calls])
def test_cleanup_resources_of_type_us_east_1(cloudwanderer_aws): create_iam_role() cloudwanderer_aws.write_resources( regions=["us-east-1"], service_resource_types=[ ServiceResourceType(service="iam", resource_type="role"), ], ) compare_list_of_dicts_allow_any( [ { "PolicyDocument": { "Statement": { "Action": "s3:ListBucket", "Effect": "Allow", "Resource": "arn:aws:s3:::example_bucket", }, "Version": "2012-10-17", }, "PolicyName": "test-role-policy", "RoleName": "test-role", "attr": "BaseResource", "urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy", }, { "attr": "ParentUrn", "urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy", "value": URN( cloud_name="aws", account_id="123456789012", region="us-east-1", service="iam", resource_type="role", resource_id_parts=["test-role"], ), }, { "attr": "DependentResourceUrns", "urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy", "value": [], }, { "Arn": "arn:aws:iam::123456789012:role/test-role", "AssumeRolePolicyDocument": {}, "CreateDate": ANY, "Description": None, "ManagedPolicyAttachments": [ { "PolicyArn": "arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy", "PolicyName": "APIGatewayServiceRolePolicy", } ], "MaxSessionDuration": ANY, "Path": "/", "PermissionsBoundary": None, "RoleId": ANY, "RoleLastUsed": None, "RoleName": "test-role", "Tags": None, "attr": "BaseResource", "urn": "urn:aws:123456789012:us-east-1:iam:role:test-role", }, {"attr": "ParentUrn", "urn": "urn:aws:123456789012:us-east-1:iam:role:test-role", "value": None}, { "attr": "DependentResourceUrns", "urn": "urn:aws:123456789012:us-east-1:iam:role:test-role", "value": [ URN( cloud_name="aws", account_id="123456789012", region="us-east-1", service="iam", resource_type="role_policy", resource_id_parts=["test-role", "test-role-policy"], ) ], }, ], list(cloudwanderer_aws.storage_connectors[0].read_all()), ) # Delete the role iam_resource = boto3.resource("iam") iam_resource.Role("test-role").detach_policy( PolicyArn="arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy" ) iam_resource.Role("test-role").Policy("test-role-policy").delete() iam_resource.Role("test-role").delete() cloudwanderer_aws.write_resources( regions=["us-east-1"], service_resource_types=[ ServiceResourceType(service="iam", resource_type="role"), ], ) assert list(cloudwanderer_aws.storage_connectors[0].read_all()) == []