def loaded_dynamodb_connector():
create_iam_role()
create_s3_buckets(regions=["us-east-1", "eu-west-2"])
create_ec2_instances(regions=["us-east-1", "eu-west-2"])
connector = DynamoDbConnector(
boto3_session=boto3.Session(
# Take advantage of the fact that dynamodb local creates a separate db instance for each access key
# to allow github action tests to run multiple python version tests in parallel.
aws_access_key_id=platform.python_version(),
aws_secret_access_key="1",
region_name="eu-west-1",
),
endpoint_url="http://localhost:8000",
)
connector.init()
wanderer = CloudWanderer(
storage_connectors=[connector],
cloud_interface=CloudWandererAWSInterface(
cloudwanderer_boto3_session=CloudWandererBoto3Session(
aws_access_key_id="aaaa", aws_secret_access_key="aaaaaa")),
)
for account_id in ["123456789012", "111111111111"]:
wanderer.cloud_interface.cloudwanderer_boto3_session.get_account_id = MagicMock(
return_value=account_id)
wanderer.write_resources(
regions=["us-east-1", "eu-west-2"],
service_resource_types=[
ServiceResourceType(service="ec2", resource_type="instance"),
ServiceResourceType(service="ec2", resource_type="vpc"),
ServiceResourceType(service="s3", resource_type="bucket"),
ServiceResourceType(service="iam", resource_type="role"),
],
)
return connector
def loaded_memory_connector():
create_iam_role()
create_s3_buckets(regions=["us-east-1", "eu-west-2"])
create_ec2_instances(regions=["us-east-1", "eu-west-2"])
connector = MemoryStorageConnector()
connector.init()
wanderer = CloudWanderer(
storage_connectors=[connector],
cloud_interface=CloudWandererAWSInterface(
cloudwanderer_boto3_session=CloudWandererBoto3Session(
aws_access_key_id="aaaa", aws_secret_access_key="aaaaaa")),
)
for account_id in ["123456789012", "111111111111"]:
wanderer.cloud_interface.cloudwanderer_boto3_session.get_account_id = MagicMock(
return_value=account_id)
wanderer.write_resources(
regions=["us-east-1", "eu-west-2"],
service_resource_types=[
ServiceResourceType(service="ec2", resource_type="instance"),
ServiceResourceType(service="ec2", resource_type="vpc"),
ServiceResourceType(service="s3", resource_type="bucket"),
ServiceResourceType(service="iam", resource_type="role"),
],
)
return connector
def test_get_resources_specific_resource(cloud_wanderer: CloudWanderer):
cloud_wanderer.write_resources(service_resource_types=[ServiceResourceType(service="ec2", resource_type="vpc")])
cloud_wanderer.cloud_interface.get_resource_discovery_actions.assert_called()
cloud_wanderer.cloud_interface.get_resources.assert_called_with(
region="eu-west-1", service_name="ec2", resource_type="vpc", service_resource_type_filters=ANY
)
cloud_wanderer.storage_connectors[0].write_resource.assert_called_with(
CloudWandererResource(
urn=URN(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-11111111"],
),
dependent_resource_urns=[],
resource_data={},
)
)
cloud_wanderer.storage_connectors[0].delete_resource_of_type_in_account_region.assert_called_with(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
cutoff=datetime.datetime(1986, 1, 1, 0, 0, tzinfo=datetime.timezone.utc),
)
def test_get_resource_discovery_actions_specific_resource_type(
aws_interface: CloudWandererAWSInterface, mock_action_set_vpc):
aws_interface._get_discovery_action_templates_for_service = MagicMock(
return_value=[mock_action_set_vpc])
result = aws_interface.get_resource_discovery_actions(
service_resource_types=[
ServiceResourceType(service="ec2", resource_type="vpc")
])
assert len(result) == 1
assert isinstance(result[0], ActionSet)
assert result[0].delete_urns == [
PartialUrn(account_id="111111111111",
region=region,
service="ec2",
resource_type="vpc",
resource_id_parts=["ALL"])
for region in ["eu-west-1", "us-east-1"]
]
assert result[0].get_urns == [
PartialUrn(account_id="111111111111",
region=region,
service="ec2",
resource_type="vpc",
resource_id_parts=["ALL"])
for region in ["eu-west-1", "us-east-1"]
]
aws_interface._get_discovery_action_templates_for_service.assert_called_with(
service=ANY,
resource_types=["vpc"],
discovery_regions=["us-east-1", "eu-west-1"])
def test_filters(cloudwanderer_aws):
create_iam_policy()
cloudwanderer_aws.write_resources(
regions=["us-east-1"],
service_resource_types=[ServiceResourceType(service="iam", resource_type="policy")],
service_resource_type_filters=[
AWSResourceTypeFilter(service="iam", resource_type="policy", botocore_filters={"Scope": "Local"})
],
)
assert set(r["urn"] for r in cloudwanderer_aws.storage_connectors[0].read_all()) == {
"urn:aws:123456789012:us-east-1:iam:policy:arn\\:aws\\:iam\\:\\:123456789012\\:policy\\/test-policy",
"urn:aws:123456789012:us-east-1:iam:policy_version:arn\\:aws\\:iam\\:\\:123456789012\\:policy\\/test-policy/v1",
}
def test_get_resource_discovery_actions(aws_interface, s3_cleanup_actions):
result = aws_interface.get_resource_discovery_actions(
regions=["us-east-1"],
service_resource_types=[
ServiceResourceType(service="s3", resource_type="bucket")
])[0]
assert result.get_urns == [
PartialUrn(
cloud_name="aws",
account_id="123456789012",
region="us-east-1",
service="s3",
resource_type="bucket",
resource_id_parts=[],
)
]
for action in s3_cleanup_actions:
assert action in result.delete_urns
def test_write_resource_type(self):
"""It is sufficient for this not to throw an exception."""
self.wanderer.write_resources(
regions=["eu-west-1"],
service_resource_types=[ServiceResourceType("iam", "role")])
def test_write_resources(cloudwanderer_aws):
create_s3_buckets()
create_ec2_instances()
cloudwanderer_aws.write_resources(
regions=["eu-west-2", "us-east-1"],
service_resource_types=[
ServiceResourceType(service="ec2", resource_type="instance"),
ServiceResourceType(service="s3", resource_type="bucket"),
],
)
result = list(cloudwanderer_aws.storage_connectors[0].read_all())
compare_list_of_dicts_allow_any(
result,
[
{
"AmiLaunchIndex": 0,
"Architecture": "x86_64",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/sda1",
"Ebs": {
"AttachTime": ANY,
"DeleteOnTermination": True,
"Status": "in-use",
"VolumeId": ANY,
},
}
],
"BootMode": None,
"CapacityReservationId": None,
"CapacityReservationSpecification": None,
"ClientToken": "ABCDE1234567890123",
"CpuOptions": None,
"EbsOptimized": False,
"ElasticGpuAssociations": None,
"ElasticInferenceAcceleratorAssociations": None,
"EnaSupport": None,
"EnclaveOptions": None,
"HibernationOptions": None,
"Hypervisor": "xen",
"IamInstanceProfile": None,
"ImageId": ANY,
"InstanceId": ANY,
"InstanceLifecycle": None,
"InstanceType": "m1.small",
"KernelId": "None",
"KeyName": "None",
"LaunchTime": ANY,
"Licenses": None,
"MetadataOptions": None,
"Monitoring": {"State": "disabled"},
"NetworkInterfaces": [
{
"Association": {"IpOwnerId": "123456789012", "PublicIp": ANY},
"Attachment": {
"AttachTime": "2015-01-01T00:00:00+00:00",
"AttachmentId": ANY,
"DeleteOnTermination": True,
"DeviceIndex": 0,
"Status": "attached",
},
"Description": "Primary network interface",
"Groups": [{"GroupId": ANY, "GroupName": "default"}],
"MacAddress": "1b:2b:3c:4d:5e:6f",
"NetworkInterfaceId": ANY,
"OwnerId": ANY,
"PrivateIpAddress": ANY,
"PrivateIpAddresses": [
{
"Association": {"IpOwnerId": "123456789012", "PublicIp": ANY},
"Primary": True,
"PrivateIpAddress": ANY,
}
],
"SourceDestCheck": True,
"Status": "in-use",
"SubnetId": ANY,
"VpcId": ANY,
}
],
"OutpostArn": None,
"Placement": {"AvailabilityZone": "eu-west-2a", "GroupName": None, "Tenancy": "default"},
"Platform": "windows",
"PlatformDetails": None,
"PrivateDnsName": ANY,
"PrivateIpAddress": ANY,
"ProductCodes": [],
"PublicDnsName": ANY,
"PublicIpAddress": ANY,
"RamdiskId": None,
"RootDeviceName": "/dev/sda1",
"RootDeviceType": "ebs",
"SecurityGroups": [],
"SourceDestCheck": True,
"SpotInstanceRequestId": None,
"SriovNetSupport": None,
"State": {"Code": 16, "Name": "running"},
"StateReason": {"Code": None, "Message": None},
"StateTransitionReason": None,
"SubnetId": ANY,
"Tags": None,
"UsageOperation": None,
"UsageOperationUpdateTime": None,
"VirtualizationType": "hvm",
"VpcId": ANY,
"attr": "BaseResource",
"urn": ANY,
},
{
"attr": "ParentUrn",
"urn": ANY,
"value": None,
},
{
"attr": "DependentResourceUrns",
"urn": ANY,
"value": [],
},
{
"CreationDate": ANY,
"Name": "test-eu-west-2",
"attr": "BaseResource",
"urn": ANY,
},
{
"attr": "ParentUrn",
"urn": ANY,
"value": None,
},
{
"attr": "DependentResourceUrns",
"urn": "urn:aws:123456789012:eu-west-2:s3:bucket:test-eu-west-2",
"value": [],
},
],
allow_partial_match_first=True,
)
def test_cleanup_resources_of_type_us_east_1(cloudwanderer_aws):
create_iam_role()
cloudwanderer_aws.write_resources(
regions=["us-east-1"],
service_resource_types=[
ServiceResourceType(service="iam", resource_type="role"),
],
)
compare_list_of_dicts_allow_any(
[
{
"PolicyDocument": {
"Statement": {
"Action": "s3:ListBucket",
"Effect": "Allow",
"Resource": "arn:aws:s3:::example_bucket",
},
"Version": "2012-10-17",
},
"PolicyName": "test-role-policy",
"RoleName": "test-role",
"attr": "BaseResource",
"urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy",
},
{
"attr": "ParentUrn",
"urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy",
"value": URN(
cloud_name="aws",
account_id="123456789012",
region="us-east-1",
service="iam",
resource_type="role",
resource_id_parts=["test-role"],
),
},
{
"attr": "DependentResourceUrns",
"urn": "urn:aws:123456789012:us-east-1:iam:role_policy:test-role/test-role-policy",
"value": [],
},
{
"Arn": "arn:aws:iam::123456789012:role/test-role",
"AssumeRolePolicyDocument": {},
"CreateDate": ANY,
"Description": None,
"ManagedPolicyAttachments": [
{
"PolicyArn": "arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy",
"PolicyName": "APIGatewayServiceRolePolicy",
}
],
"MaxSessionDuration": ANY,
"Path": "/",
"PermissionsBoundary": None,
"RoleId": ANY,
"RoleLastUsed": None,
"RoleName": "test-role",
"Tags": None,
"attr": "BaseResource",
"urn": "urn:aws:123456789012:us-east-1:iam:role:test-role",
},
{"attr": "ParentUrn", "urn": "urn:aws:123456789012:us-east-1:iam:role:test-role", "value": None},
{
"attr": "DependentResourceUrns",
"urn": "urn:aws:123456789012:us-east-1:iam:role:test-role",
"value": [
URN(
cloud_name="aws",
account_id="123456789012",
region="us-east-1",
service="iam",
resource_type="role_policy",
resource_id_parts=["test-role", "test-role-policy"],
)
],
},
],
list(cloudwanderer_aws.storage_connectors[0].read_all()),
)
# Delete the role
iam_resource = boto3.resource("iam")
iam_resource.Role("test-role").detach_policy(
PolicyArn="arn:aws:iam::aws:policy/aws-service-role/APIGatewayServiceRolePolicy"
)
iam_resource.Role("test-role").Policy("test-role-policy").delete()
iam_resource.Role("test-role").delete()
cloudwanderer_aws.write_resources(
regions=["us-east-1"],
service_resource_types=[
ServiceResourceType(service="iam", resource_type="role"),
],
)
assert list(cloudwanderer_aws.storage_connectors[0].read_all()) == []
def test_write_resource_type(cloudwanderer):
"""It is sufficient for this not to throw an exception."""
cloudwanderer.write_resources(regions=["us-east-1"], service_resource_types=[ServiceResourceType("iam", "policy")])