def test_dict_conversion(cloudwanderer_resource):
assert dict(cloudwanderer_resource) == {
"urn":
URN(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-111111"],
),
"relationships": [
URN(
cloud_name="aws",
account_id="unknown",
region="unknown",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-111111"],
)
],
"dependent_resource_urns": [],
"parent_urn":
None,
"cloudwanderer_metadata": {
"VpcId": "vpc-111111"
},
"discovery_time":
datetime(2021, 10, 23, 0, 0),
"vpc_id":
"vpc-111111",
}
def test_write_resources(cloudwanderer_aws, aws_interface,
default_test_discovery_actions):
create_iam_role()
create_s3_buckets(regions=["eu-west-2", "us-east-1"])
aws_interface.get_resource_discovery_actions = MagicMock(
return_value=default_test_discovery_actions)
thread_results = list(
cloudwanderer_aws.write_resources_concurrently(
concurrency=2,
cloud_interface_generator=lambda: aws_interface,
storage_connector_generator=lambda: [MemoryStorageConnector()],
))
connector_results = [
resource_record for result in thread_results
for connector in result.storage_connectors
for resource_record in connector.read_all()
]
result_summary = set([(URN.from_string(result["urn"]).region,
URN.from_string(result["urn"]).resource_type)
for result in connector_results])
assert result_summary == {
("eu-west-2", "bucket"),
("eu-west-2", "vpc"),
("us-east-1", "bucket"),
("us-east-1", "role"),
("us-east-1", "role_policy"),
("us-east-1", "vpc"),
}
def iam_role():
return CloudWandererResource(
urn=URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
),
resource_data={
"RoleName": "test-role",
"InlinePolicyAttachments": [{
"PolicyNames": ["test-role"]
}]
},
dependent_resource_urns=[
URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role_policy",
resource_id_parts=["test-role", "test-role-policy-1"],
)
],
)
def iam_instance_profile():
return CloudWandererResource(
urn=URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="instance_profile",
resource_id_parts=["my-test-profile"],
),
resource_data={},
dependent_resource_urns=[],
relationships=[
Relationship(
partial_urn=PartialUrn(
cloud_name="aws",
account_id="unknown",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
),
direction=RelationshipDirection.INBOUND,
)
],
)
def test_get_resources_specific_resource(cloud_wanderer: CloudWanderer):
cloud_wanderer.write_resources(service_resource_types=[ServiceResourceType(service="ec2", resource_type="vpc")])
cloud_wanderer.cloud_interface.get_resource_discovery_actions.assert_called()
cloud_wanderer.cloud_interface.get_resources.assert_called_with(
region="eu-west-1", service_name="ec2", resource_type="vpc", service_resource_type_filters=ANY
)
cloud_wanderer.storage_connectors[0].write_resource.assert_called_with(
CloudWandererResource(
urn=URN(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-11111111"],
),
dependent_resource_urns=[],
resource_data={},
)
)
cloud_wanderer.storage_connectors[0].delete_resource_of_type_in_account_region.assert_called_with(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
cutoff=datetime.datetime(1986, 1, 1, 0, 0, tzinfo=datetime.timezone.utc),
)
def get_inferred_ec2_instances(cloudwanderer_boto3_session):
vpcs = list(cloudwanderer_boto3_session.resource("ec2").vpcs.all())
return [
CloudWandererResource(
urn=URN(
account_id="111111111111",
region="eu-west-2",
service="ec2",
resource_type="instance",
resource_id_parts=[instance.instance_id],
),
resource_data=instance.meta.data,
relationships=[
Relationship(
partial_urn=PartialUrn(
cloud_name="aws",
account_id="unknown",
region="eu-west-2",
service="ec2",
resource_type="vpc",
resource_id_parts=[vpcs[0].vpc_id],
),
direction=RelationshipDirection.INBOUND,
)
],
) for instance in cloudwanderer_boto3_session.resource(
"ec2").instances.all()
]
def urn():
return URN(
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-111111"],
)
def partial_urn():
return URN(
account_id="unknown",
region="unknown",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-111111"],
)
def cloudwanderer_resource_urn() -> URN:
return URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
)
def test_get_urn(service_resource_s3_bucket):
urn = service_resource_s3_bucket.get_urn()
assert urn == URN(
account_id="111111111111",
region="eu-west-1",
service="s3",
resource_type="bucket",
resource_id_parts=["my_bucket"],
)
def test_dependent_resource_urn(loaded_memory_connector):
result = next(
loaded_memory_connector.read_resources(urn=URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role_policy",
resource_id_parts=["test-role", "test-role-policy"],
)),
None,
)
assert isinstance(result.parent_urn, URN)
assert result.parent_urn == URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
)
def test_get_urn_regex_pattern(iam_service):
single_iam_policy = list(iam_service.collection("policy").all())[0]
single_iam_policy.fetch_secondary_attributes()
assert single_iam_policy.get_urn() == URN(
cloud_name="aws",
account_id="aws",
region="us-east-1",
service="iam",
resource_type="policy",
resource_id_parts=["arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy"],
)
def inferred_ec2_vpcs(cloudwanderer_boto3_session):
return [
CloudWandererResource(
urn=URN(
account_id="111111111111",
region="eu-west-2",
service="ec2",
resource_type="vpc",
resource_id_parts=[vpc.vpc_id],
),
resource_data=vpc.meta.data,
) for vpc in cloudwanderer_boto3_session.resource("ec2").vpcs.all()
]
def get_inferred_ec2_instances(cloudwanderer_boto3_session):
return [
CloudWandererResource(
urn=URN(
account_id="111111111111",
region="eu-west-2",
service="ec2",
resource_type="instance",
resource_id_parts=[instance.instance_id],
),
resource_data=instance.meta.data,
) for instance in cloudwanderer_boto3_session.resource(
"ec2").instances.all()
]
def iam_role_policies():
return [
CloudWandererResource(
urn=URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role_policy",
resource_id_parts=["test-role", "test-role-policy-1"],
),
resource_data={},
parent_urn=URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
),
),
CloudWandererResource(
urn=URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role_policy",
resource_id_parts=["test-role", "test-role-policy-2"],
),
resource_data={},
parent_urn=URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
),
),
]
def cloud_wanderer() -> CloudWanderer:
mock_storage_connector = MagicMock(**{})
mock_cloud_interface = MagicMock(
spec_set=CloudWandererAWSInterface,
**{
"get_resource_discovery_actions.return_value": [
ActionSet(
get_urns=[
PartialUrn(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["ALL"],
)
],
delete_urns=[
PartialUrn(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["ALL"],
)
],
)
],
"get_resources.return_value": [
CloudWandererResource(
URN(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-11111111"],
),
resource_data={},
discovery_time=datetime.datetime(1986, 1, 1, tzinfo=datetime.timezone.utc),
)
],
}
)
return CloudWanderer(storage_connectors=[mock_storage_connector], cloud_interface=mock_cloud_interface)
def test_write_resource_and_attribute(dynamodb_connnector, iam_role):
dynamodb_connnector.write_resource(resource=iam_role)
result = dynamodb_connnector.read_resource(urn=iam_role.urn)
assert result.urn == iam_role.urn
assert result.role_name == "test-role"
logger.info(result.cloudwanderer_metadata.resource_data)
assert result.inline_policy_attachments == [{"PolicyNames": ["test-role"]}]
assert result.dependent_resource_urns == [
URN(
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role_policy",
resource_id_parts=["test-role", "test-role-policy-1"],
)
]
def test_resource_urn(loaded_dynamodb_connector):
result: List[CloudWandererResource] = list(
loaded_dynamodb_connector.read_resources(urn=URN(
cloud_name="aws",
account_id="111111111111",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
)))
expected_urns = [
{
"account_id": "111111111111",
"region": "us-east-1",
"service": "iam",
"resource_type": "role",
"resource_id": "test-role",
},
]
not_expected_urns = [
{
"account_id": "111111111111",
"region": "us-east-1",
"service": "ec2",
"resource_type": "vpc"
},
{
"account_id": "111111111111",
"region": "eu-west-2",
"service": "ec2",
"resource_type": "vpc"
},
{
"account_id": "123456789012",
"region": "us-east-1",
"service": "ec2",
"resource_type": "vpc"
},
{
"account_id": "123456789012",
"region": "eu-west-2",
"service": "ec2",
"resource_type": "vpc"
},
{
"account_id": "123456789012",
"region": "us-east-1",
"service": "iam",
"resource_type": "role",
"resource_id": "test-role",
},
{
"account_id": "123456789012",
"region": "eu-west-2",
"service": "s3",
"resource_type": "bucket"
},
{
"account_id": "123456789012",
"region": "us-east-1",
"service": "s3",
"resource_type": "bucket"
},
{
"account_id": "111111111111",
"region": "eu-west-2",
"service": "s3",
"resource_type": "bucket"
},
{
"account_id": "111111111111",
"region": "us-east-1",
"service": "s3",
"resource_type": "bucket"
},
]
assert_has_matching_urns(result, expected_urns)
assert_does_not_have_matching_urns(result, not_expected_urns)
def test_all_custom_resources(file_name, aws_interface):
with open(file_name) as f:
test_spec = json.load(f)
mock, paginator_side_effects = build_mock(test_spec["mockData"])
mock_client = MagicMock(
**{
**{
"meta": aws_interface.cloudwanderer_boto3_session.client(test_spec["service"]).meta,
"get_caller_identity.return_value": {
"UserId": "111111111111111111111",
"Account": "0123456789012",
"Arn": "arn:aws:iam::0123456789012:user/CloudWanderer",
},
},
**mock,
}
)
aws_interface.cloudwanderer_boto3_session.client = MagicMock(return_value=mock_client)
if "getResource" in test_spec:
urn = URN.from_string(test_spec["getResource"]["urn"])
logger.info("Getting resource %s", urn)
try:
result = prepare_for_comparison(aws_interface.get_resource(urn=urn))
except botocore.model.NoShapeFoundError as ex:
raise ValueError(
"Boto3 raised a NoShapeFoundError, this suggests your test has the wrong service name specified"
) from ex
if "getResources" in test_spec:
get_resources = test_spec["getResources"]
logger.info("Getting resources %s", get_resources)
try:
result = prepare_for_comparison(
aws_interface.get_resources(
service_name=get_resources["serviceName"],
resource_type=get_resources["resourceType"],
region=get_resources["region"],
)
)
except NotImplementedError as ex:
raise ValueError(
"Boto3 raised a NotImplementedError, usually this means "
"you forgot to wrap your paginate.return_results in a list or "
"include the key at the top level (e.g. Instances)"
) from ex
except botocore.model.NoShapeFoundError as ex:
raise ValueError(
"Boto3 raised a NoShapeFoundError, this suggests your test has the wrong service name specified"
) from ex
compare_list_of_dicts_allow_any(
test_spec["expectedResults"], [dict(x) for x in result], allow_partial_match_first=True
)
for method_path, calls in test_spec["expectedCalls"].items():
logger.info("Assert calls %s on %s", calls, method_path)
method = mock_client
for attr in method_path.split("."):
index_match = re.search(r"\[(?P<index>\d)\](?P<attr>.*)", attr)
if index_match:
# This will only work if it's an index of `get_paginator.side_effect`
index = index_match.groupdict()["index"]
attr = index_match.groupdict()["attr"]
method = paginator_side_effects[int(index)]
if not attr:
continue
method = getattr(method, attr)
logger.info("Assert %s calls on %s", calls, method_path)
logger.info([call(*call_dict["args"], **call_dict["kwargs"]) for call_dict in calls])
method.assert_has_calls([call(*call_dict["args"], **call_dict["kwargs"]) for call_dict in calls])
def test_cleanup_resources_of_type_us_east_1(cloudwanderer_aws):
create_iam_role()
cloudwanderer_aws.write_resources(
regions=["us-east-1"],
service_resource_types=[
ServiceResourceType(service="iam", resource_type="role"),
],
)
compare_list_of_dicts_allow_any(
[
{
"PolicyDocument": {
"Statement": {
"Action": "s3:ListBucket",
"Effect": "Allow",
"Resource": "arn:aws:s3:::example_bucket",
},
"Version": "2012-10-17",
},
"PolicyName": "test-role-policy",
"RoleName": "test-role",
"attr": "BaseResource",
"urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy",
},
{
"attr": "ParentUrn",
"urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy",
"value": URN(
cloud_name="aws",
account_id="123456789012",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
),
},
{
"attr": "DependentResourceUrns",
"urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy",
"value": [],
},
{
"Arn": "arn:aws:iam::123456789012:role/test-role",
"AssumeRolePolicyDocument": {},
"CreateDate": ANY,
"Description": None,
"ManagedPolicyAttachments": [
{
"PolicyArn": "arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy",
"PolicyName": "APIGatewayServiceRolePolicy",
}
],
"MaxSessionDuration": ANY,
"Path": "/",
"PermissionsBoundary": None,
"RoleId": ANY,
"RoleLastUsed": None,
"RoleName": "test-role",
"Tags": None,
"attr": "BaseResource",
"urn": "urn:aws:123456789012:us-east-1:iam:role:test-role",
},
{"attr": "ParentUrn", "urn": "urn:aws:123456789012:us-east-1:iam:role:test-role", "value": None},
{
"attr": "DependentResourceUrns",
"urn": "urn:aws:123456789012:us-east-1:iam:role:test-role",
"value": [
URN(
cloud_name="aws",
account_id="123456789012",
region="us-east-1",
service="iam",
resource_type="role_policy",
resource_id_parts=["test-role", "test-role-policy"],
)
],
},
],
list(cloudwanderer_aws.storage_connectors[0].read_all()),
)
# Delete the role
iam_resource = boto3.resource("iam")
iam_resource.Role("test-role").detach_policy(
PolicyArn="arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy"
)
iam_resource.Role("test-role").Policy("test-role-policy").delete()
iam_resource.Role("test-role").delete()
cloudwanderer_aws.write_resources(
regions=["us-east-1"],
service_resource_types=[
ServiceResourceType(service="iam", resource_type="role"),
],
)
assert list(cloudwanderer_aws.storage_connectors[0].read_all()) == []
