def test_get_resources_specific_resource(cloud_wanderer: CloudWanderer):
cloud_wanderer.write_resources(service_resource_types=[ServiceResourceType(service="ec2", resource_type="vpc")])
cloud_wanderer.cloud_interface.get_resource_discovery_actions.assert_called()
cloud_wanderer.cloud_interface.get_resources.assert_called_with(
region="eu-west-1", service_name="ec2", resource_type="vpc", service_resource_type_filters=ANY
)
cloud_wanderer.storage_connectors[0].write_resource.assert_called_with(
CloudWandererResource(
urn=URN(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-11111111"],
),
dependent_resource_urns=[],
resource_data={},
)
)
cloud_wanderer.storage_connectors[0].delete_resource_of_type_in_account_region.assert_called_with(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
cutoff=datetime.datetime(1986, 1, 1, 0, 0, tzinfo=datetime.timezone.utc),
)
def loaded_dynamodb_connector():
create_iam_role()
create_s3_buckets(regions=["us-east-1", "eu-west-2"])
create_ec2_instances(regions=["us-east-1", "eu-west-2"])
connector = DynamoDbConnector(
boto3_session=boto3.Session(
# Take advantage of the fact that dynamodb local creates a separate db instance for each access key
# to allow github action tests to run multiple python version tests in parallel.
aws_access_key_id=platform.python_version(),
aws_secret_access_key="1",
region_name="eu-west-1",
),
endpoint_url="http://localhost:8000",
)
connector.init()
wanderer = CloudWanderer(
storage_connectors=[connector],
cloud_interface=CloudWandererAWSInterface(
cloudwanderer_boto3_session=CloudWandererBoto3Session(
aws_access_key_id="aaaa", aws_secret_access_key="aaaaaa")),
)
for account_id in ["123456789012", "111111111111"]:
wanderer.cloud_interface.cloudwanderer_boto3_session.get_account_id = MagicMock(
return_value=account_id)
wanderer.write_resources(
regions=["us-east-1", "eu-west-2"],
service_resource_types=[
ServiceResourceType(service="ec2", resource_type="instance"),
ServiceResourceType(service="ec2", resource_type="vpc"),
ServiceResourceType(service="s3", resource_type="bucket"),
ServiceResourceType(service="iam", resource_type="role"),
],
)
return connector
def loaded_memory_connector():
create_iam_role()
create_s3_buckets(regions=["us-east-1", "eu-west-2"])
create_ec2_instances(regions=["us-east-1", "eu-west-2"])
connector = MemoryStorageConnector()
connector.init()
wanderer = CloudWanderer(
storage_connectors=[connector],
cloud_interface=CloudWandererAWSInterface(
cloudwanderer_boto3_session=CloudWandererBoto3Session(
aws_access_key_id="aaaa", aws_secret_access_key="aaaaaa")),
)
for account_id in ["123456789012", "111111111111"]:
wanderer.cloud_interface.cloudwanderer_boto3_session.get_account_id = MagicMock(
return_value=account_id)
wanderer.write_resources(
regions=["us-east-1", "eu-west-2"],
service_resource_types=[
ServiceResourceType(service="ec2", resource_type="instance"),
ServiceResourceType(service="ec2", resource_type="vpc"),
ServiceResourceType(service="s3", resource_type="bucket"),
ServiceResourceType(service="iam", resource_type="role"),
],
)
return connector
def setUp(self):
self.storage_connector = DynamoDbConnector(
endpoint_url="http://localhost:8000",
client_args={
"config": botocore.config.Config(max_pool_connections=100, )
},
)
self.storage_connector.init()
self.wanderer = CloudWanderer(
storage_connectors=[self.storage_connector])
def setUp(self):
self.storage_connector = MemoryStorageConnector()
self.wanderer = CloudWanderer(
storage_connectors=[self.storage_connector],
cloud_interface=CloudWandererAWSInterface(DEFAULT_SESSION),
)
self.boto3_client_mock = MagicMock()
self.add_service_mock()
self.add_caller_identity()
self.client_patch = patch("boto3.session.Session.client", new=self.boto3_client_mock)
self.client_patch.start()
def cloud_wanderer() -> CloudWanderer:
mock_storage_connector = MagicMock(**{})
mock_cloud_interface = MagicMock(
spec_set=CloudWandererAWSInterface,
**{
"get_resource_discovery_actions.return_value": [
ActionSet(
get_urns=[
PartialUrn(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["ALL"],
)
],
delete_urns=[
PartialUrn(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["ALL"],
)
],
)
],
"get_resources.return_value": [
CloudWandererResource(
URN(
cloud_name="aws",
account_id="111111111111",
region="eu-west-1",
service="ec2",
resource_type="vpc",
resource_id_parts=["vpc-11111111"],
),
resource_data={},
discovery_time=datetime.datetime(1986, 1, 1, tzinfo=datetime.timezone.utc),
)
],
}
)
return CloudWanderer(storage_connectors=[mock_storage_connector], cloud_interface=mock_cloud_interface)
class TestFunctional(unittest.TestCase):
identifier_mapping = {
"layer_version:version":
"1",
"policy:arn":
"arn:aws:iam::1234567890:policy/APIGatewayLambdaExecPolicy",
"policy_version:arn":
"arn:aws:iam::1234567890:policy/APIGatewayLambdaExecPolicy",
}
resources_not_supporting_load = [
"lambda:layer", "iam:virtual_mfa_device", "iam:signing_certificate"
]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
logging.basicConfig(level="debug")
def setUp(self):
self.storage_connector = DynamoDbConnector(
endpoint_url="http://localhost:8000",
client_args={
"config": botocore.config.Config(max_pool_connections=100, )
},
)
self.storage_connector.init()
self.wanderer = CloudWanderer(
storage_connectors=[self.storage_connector])
# The _a_ in this test name ensures this test runs first so that subsequent read tests have values to read.
def test_a_write_resources(self):
thread_results = self.wanderer.write_resources_concurrently(
concurrency=12,
cloud_interface_generator=lambda: CloudWandererAWSInterface(
cloudwanderer_boto3_session=CloudWandererBoto3Session()),
storage_connector_generator=lambda: [
DynamoDbConnector(
endpoint_url="http://localhost:8000",
client_args={
"config":
botocore.config.Config(max_pool_connections=100, )
},
),
],
)
for result in thread_results:
assert isinstance(result, CloudWandererConcurrentWriteThreadResult)
# def test_a_write_resources(self):
# """It is sufficient for this not to throw an exception."""
# self.wanderer.write_resources()
def test_write_resources_in_region(self):
"""It is sufficient for this not to throw an exception."""
self.wanderer.write_resources(regions=["us-east-1"],
exclude_resources=[])
def test_write_resource_type(self):
"""It is sufficient for this not to throw an exception."""
self.wanderer.write_resources(
regions=["eu-west-1"],
service_resource_types=[ServiceResourceType("iam", "role")])
def test_write_custom_resource_definition(self):
"""It is sufficient for this not to throw an exception."""
self.wanderer.write_resources(service_names=["lambda"])
def test_write_single_non_existent_resource_of_every_type(self):
for service_name in self.wanderer.cloud_interface.boto3_services.available_services:
service = self.wanderer.cloud_interface.boto3_services.get_service(
service_name)
for resource_type in service.resource_types:
logging.info("Testing %s %s", service_name, resource_type)
resource = service._get_empty_resource(resource_type)
identifiers = resource.boto3_resource.meta.resource_model.identifiers
args = [
self.identifier_mapping.get(
f"{resource_type}:{identifier.name}",
f"{resource_type}:{identifier.name}")
for identifier in identifiers
]
print(args)
self._write_resource_with_fake_id(service_name, resource_type,
args)
def test_write_single_non_existent_subresource_of_every_type(self):
for service_name in self.wanderer.cloud_interface.boto3_services.available_services:
service = self.wanderer.cloud_interface.boto3_services.get_service(
service_name)
for resource_summary in service.resource_summary:
for subresource in resource_summary.subresources:
resource_type = subresource.resource_type
resource = service._get_empty_resource(resource_type)
identifiers = resource.boto3_resource.meta.resource_model.identifiers
args = [
self.identifier_mapping.get(
f"{resource_type}:{identifier.name}",
f"{resource_type}:{identifier.name}")
for identifier in identifiers
]
logging.info("Testing %s %s", service_name, resource_type)
print(args)
self._write_resource_with_fake_id(service_name,
resource_type, args)
def _write_resource_with_fake_id(self, service_name, resource_type,
args) -> None:
urn = URN(
account_id=self.wanderer.cloud_interface.account_id,
region="eu-west-2",
service=service_name,
resource_type=resource_type,
resource_id_parts=args,
)
try:
self.wanderer.write_resource(urn=urn)
except (BadUrnSubResourceError, BadUrnRegionError):
pass
except UnsupportedResourceTypeError as ex:
if f"{service_name}:{resource_type}" in self.resources_not_supporting_load:
return
raise ex
def test_write_single_resource_of_every_found_type(self):
for service_name in self.wanderer.cloud_interface.boto3_services.available_services:
service = self.wanderer.cloud_interface.boto3_services.get_service(
service_name)
for resource_type in service.resource_types:
try:
resource = next(
self.storage_connector.read_resources(
service=service_name, resource_type=resource_type))
except StopIteration:
logging.info(
"No %s %s resources were found, skipping testing write_resource for this type",
service_name,
resource_type,
)
continue
logging.info("Found %s, testing write_resource", resource.urn)
try:
self.wanderer.write_resource(urn=resource.urn)
except UnsupportedResourceTypeError as ex:
if f"{service_name}:{resource_type}" in self.resources_not_supporting_load:
continue
raise ex
def test_read_all(self):
results = list(self.storage_connector.read_all())
assert len(results) > 0
for result in results:
assert isinstance(result, dict)
def test_read_resource_of_type(self):
vpcs = list(
self.storage_connector.read_resources(service="ec2",
resource_type="vpc"))
vpcs[0].load()
assert len(vpcs) > 0
assert isinstance(
vpcs[0].get_secondary_attribute(name="vpc_enable_dns_support")[0],
dict)
assert isinstance(vpcs[0].is_default, bool)
def test_read_all_resources_in_account(self):
resources = list(
self.storage_connector.read_resources(
account_id=self.wanderer.cloud_interface.account_id))
service_resource_types = set()
for resource in resources:
service_resource_types.add(
f"{resource.urn.service}:{resource.urn.resource_type}")
expected_resource_types = {
"apigateway:rest_api",
"cloudformation:stack",
"cloudwatch:alarm",
"cloudwatch:metric",
"dynamodb:table",
"ec2:dhcp_options",
"ec2:internet_gateway",
"ec2:network_acl",
"ec2:route_table",
"ec2:security_group",
"ec2:subnet",
"ec2:vpc",
"iam:group",
"iam:role",
"iam:role_policy",
"iam:user",
"lambda:function",
"s3:bucket",
"secretsmanager:secret",
"sns:subscription",
"sns:topic",
}
for resource_type in expected_resource_types:
assert resource_type in service_resource_types
def test_read_resource_of_type_in_account(self):
vpc = next(
self.storage_connector.read_resources(
service="ec2",
resource_type="vpc",
account_id=self.wanderer.cloud_interface.account_id))
vpc.load()
assert isinstance(
vpc.get_secondary_attribute(name="vpc_enable_dns_support")[0],
dict)
assert isinstance(vpc.is_default, bool)
def test_documentation_resource_example(self):
"""It's sufficient for this not to throw an exception."""
storage_connector = self.storage_connector
resources = storage_connector.read_resources(service="ec2",
resource_type="vpc")
for resource in resources:
resource.load()
print(resource.urn)
print(resource.cidr_block)
print(resource.cidr_block_association_set)
print(resource.dhcp_options_id)
print(resource.instance_tenancy)
print(resource.ipv6_cidr_block_association_set)
print(resource.is_default)
print(resource.owner_id)
print(resource.state)
print(resource.tags)
print(resource.vpc_id)
def test_documentation_secondary_attribute_example(self):
"""It's sufficient for this not to throw an exception."""
storage_connector = self.storage_connector
resources = storage_connector.read_resources(service="ec2",
resource_type="vpc")
for resource in resources:
resource.get_secondary_attribute(name="vpc_enable_dns_support")
def test_documentation_subresource_example(self):
"""It's sufficient for this not to throw an exception."""
storage_connector = self.storage_connector
resources = storage_connector.read_resources(service="iam",
resource_type="role")
for resource in resources:
resource.load()
print(resource.urn)
print(resource.arn)
print(resource.assume_role_policy_document)
print(resource.create_date)
print(resource.description)
print(resource.max_session_duration)
print(resource.path)
print(resource.permissions_boundary)
print(resource.role_id)
print(resource.role_last_used)
print(resource.role_name)
print(resource.tags)
class NoMotoMock(ABC, GenericAssertionHelpers):
single_resource_scenarios: List["SingleResourceScenario"]
multiple_resource_scenarios = List["MultipleResourceScenario"]
def setUp(self):
self.storage_connector = MemoryStorageConnector()
self.wanderer = CloudWanderer(
storage_connectors=[self.storage_connector],
cloud_interface=CloudWandererAWSInterface(DEFAULT_SESSION),
)
self.boto3_client_mock = MagicMock()
self.add_service_mock()
self.add_caller_identity()
self.client_patch = patch("boto3.session.Session.client", new=self.boto3_client_mock)
self.client_patch.start()
def tearDown(self) -> None:
self.client_patch.stop()
def add_caller_identity(self):
self.service_mocks["sts"] = MagicMock(**{"get_caller_identity.return_value": {"Account": "123456789012"}})
def add_service_mock(self):
self.service_mocks = {}
for service_name in self.wanderer.cloud_interface.boto3_services.available_services:
self.service_mocks[service_name] = MagicMock()
for method_path, mock_return in self.mock.get(service_name, {}).items():
path_parts = method_path.split(".")
method_name = path_parts[0]
return_type = path_parts[1]
logger.info(f"Adding mock for {service_name} {method_name}")
setattr(self.service_mocks[service_name], method_name, MagicMock(**{return_type: mock_return}))
self.service_mocks[service_name].get_paginator.side_effect = paginator_side_effect(
self.mock.get(service_name, {})
)
self.service_mocks[service_name].meta = boto3.client(service_name).meta
self.boto3_client_mock.side_effect = lambda service_name, **kwargs: self.service_mocks[service_name]
def test_write_resource(self):
for i, scenario in enumerate(self.single_resource_scenarios):
logging.info("Testing fetching %s", scenario.urn)
if isinstance(scenario.expected_results, type):
with self.assertRaises(scenario.expected_results):
self.wanderer.write_resource(urn=scenario.urn)
continue
self.wanderer.write_resource(urn=scenario.urn)
self.assert_dictionary_overlap(
self.storage_connector.read_all(),
scenario.expected_results,
)
if scenario.expected_call is None:
continue
mock_method = getattr(self.service_mocks[scenario.expected_call.service], scenario.expected_call.method)
mock_method.assert_called_with(*scenario.expected_call.args, **scenario.expected_call.kwargs)
def test_write_resources(self):
for scenario in self.multiple_resource_scenarios:
self.wanderer.write_resources(**scenario.arguments)
self.assert_dictionary_overlap(
self.storage_connector.read_all(),
scenario.expected_results,
)
def cloudwanderer_aws(aws_interface):
return CloudWanderer(cloud_interface=aws_interface,
storage_connectors=[MemoryStorageConnector()])
def cloudwanderer(storage_connector):
storage_connector.init()
return CloudWanderer(storage_connectors=[storage_connector])