class EC2RouteTableResourceSpec(EC2ResourceSpec):
"""Resource for Route Tables"""
# type_name = "route-table"
type_name = "route_table"
schema = Schema(
ScalarField("RouteTableId"),
ResourceLinkField("VpcId", VPCResourceSpec),
ScalarField("OwnerId", optional=True),
AnonymousListField("PropagatingVgws",
ScalarField("GatewayId"),
optional=True),
ListField(
"Routes",
EmbeddedDictField(
ScalarField("DestinationCidrBlock", optional=True),
ScalarField("DestinationIpv6CidrBlock", optional=True),
ScalarField("DestinationPrefixListId", optional=True),
ScalarField("EgressOnlyInternetGatewayId", optional=True),
ScalarField("GatewayId", optional=True),
ScalarField("InstanceId", optional=True),
ScalarField("InstanceOwnerId", optional=True),
ScalarField("NatGatewayId", optional=True),
ScalarField("TransitGatewayId", optional=True),
ScalarField("NetworkInterfaceId", optional=True),
ScalarField("Origin", optional=True),
ScalarField("State", optional=True),
ScalarField("VpcPeeringConnectionId", optional=True),
),
optional=True,
alti_key="route",
),
ListField(
"Associations",
EmbeddedDictField(
ScalarField("Main", optional=True),
ScalarField("RouteTableAssociationId", optional=True),
ScalarField("RouteTableId", optional=True),
ScalarField("SubnetId", optional=True),
),
optional=True,
alti_key="association",
),
)
@classmethod
def list_from_aws(cls: Type["EC2RouteTableResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
paginator = client.get_paginator("describe_route_tables")
route_tables = {}
for resp in paginator.paginate():
for attachment in resp.get("RouteTables", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=attachment["RouteTableId"])
route_tables[resource_arn] = attachment
return ListFromAWSResult(resources=route_tables)
def test_valid_strings_input(self):
input_str = '{"Animals": ["cow", "pig", "human"]}'
field = ListField("Animals", EmbeddedScalarField())
expected_output_data = [
{
"pred": "animals",
"obj": "cow",
"type": "simple"
},
{
"pred": "animals",
"obj": "pig",
"type": "simple"
},
{
"pred": "animals",
"obj": "human",
"type": "simple"
},
]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class TargetGroupResourceSpec(ElasticLoadBalancingResourceSpec):
"""Resource for target group"""
type_name = "targetgroup"
schema = Schema(
ScalarField("TargetGroupName"),
ScalarField("Protocol", optional=True),
ScalarField("Port", optional=True),
TransientResourceLinkField("VpcId", VPCResourceSpec, optional=True),
ScalarField("HealthCheckProtocol", optional=True),
ScalarField("HealthCheckPort", optional=True),
ScalarField("HealthCheckEnabled"),
ListField(
"LoadBalancerArns",
EmbeddedResourceLinkField(LoadBalancerResourceSpec,
value_is_id=True),
),
ScalarField("TargetType"),
ListField(
"TargetHealthDescriptions",
EmbeddedDictField(
AnonymousDictField(
"Target",
ScalarField("Id", alti_key="target_id"),
ScalarField("Port", alti_key="target_port", optional=True),
ScalarField("AvailabilityZone",
alti_key="target_az",
optional=True),
),
ScalarField("HealthCheckPort", optional=True),
AnonymousDictField(
"TargetHealth",
ScalarField("State"),
ScalarField("Reason", optional=True),
ScalarField("Description", optional=True),
optional=True,
),
),
),
)
@classmethod
def list_from_aws(cls: Type["TargetGroupResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'target_group_1_arn': {target_group_1_dict},
'target_group_2_arn': {target_group_2_dict},
...}
Where the dicts represent results from describe_target_groups."""
paginator = client.get_paginator("describe_target_groups")
resources = {}
for resp in paginator.paginate():
for resource in resp.get("TargetGroups", []):
resource_arn = resource["TargetGroupArn"]
resource["TargetHealthDescriptions"] = get_target_group_health(
client, resource_arn)
resources[resource_arn] = resource
return ListFromAWSResult(resources=resources)
def test_valid_dicts_input_with_alti_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = ListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person"
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Bob"),
SimpleLink(pred="age", obj=49),
),
),
),
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Sue"),
SimpleLink(pred="age", obj=42),
),
),
),
),
)
self.assertEqual(link_collection, expected_link_collection)
def test_invalid_input_missing_source_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = ListField(
"Stuff", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person"
)
input_data = json.loads(input_str)
with self.assertRaises(ListFieldSourceKeyNotFoundException):
field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
def test_optional(self):
input_str = "{}"
field = ListField("People", EmbeddedScalarField(), alti_key="person", optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
self.assertEqual(link_collection, LinkCollection())
def test_invalid_input_not_list(self):
input_str = '{"People": "foo"}'
field = ListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person"
)
input_data = json.loads(input_str)
with self.assertRaises(ListFieldValueNotAListException):
field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
class LoadBalancerResourceSpec(ElasticLoadBalancingResourceSpec):
"""Resource for load balancer"""
type_name = "loadbalancer"
schema = Schema(
ScalarField("DNSName"),
ScalarField("CreatedTime"),
ScalarField("LoadBalancerName"),
ScalarField("Scheme"),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
AnonymousDictField("State",
ScalarField("Code",
alti_key="load_balancer_state")),
ScalarField("Type"),
ListField(
"AvailabilityZones",
EmbeddedDictField(
ScalarField("ZoneName"),
ResourceLinkField("SubnetId",
SubnetResourceSpec,
optional=True),
ListField(
"LoadBalancerAddresses",
EmbeddedDictField(
ScalarField("IpAddress", optional=True),
ScalarField("AllocationId", optional=True),
),
optional=True,
),
),
),
ListField("SecurityGroups",
EmbeddedResourceLinkField(SecurityGroupResourceSpec),
optional=True),
ScalarField("IpAddressType"),
)
@classmethod
def list_from_aws(cls: Type["LoadBalancerResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'lb_1_arn': {lb_1_dict},
'lb_2_arn': {lb_2_dict},
...}
Where the dicts represent results from describe_load_balancers."""
paginator = client.get_paginator("describe_load_balancers")
load_balancers = {}
for resp in paginator.paginate():
for lb in resp.get("LoadBalancers", []):
resource_arn = lb["LoadBalancerArn"]
load_balancers[resource_arn] = lb
return ListFromAWSResult(resources=load_balancers)
def test_allow_scalar(self):
input_str = '{"People": "bob"}'
field = ListField("People", EmbeddedScalarField(), alti_key="person", allow_scalar=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
simple_links=(SimpleLink(pred="person", obj="bob"),),
)
self.assertEqual(link_collection, expected_link_collection)
def test_optional(self):
input_str = "{}"
field = ListField("People",
EmbeddedScalarField(),
alti_key="person",
optional=True)
expected_output_data = ()
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
def test_valid_dicts_input_with_alti_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = ListField("People",
EmbeddedDictField(ScalarField("Name"),
ScalarField("Age")),
alti_key="person")
expected_output_data = [
{
"pred":
"person",
"obj": [
{
"pred": "name",
"obj": "Bob",
"type": "simple"
},
{
"pred": "age",
"obj": 49,
"type": "simple"
},
],
"type":
"multi",
},
{
"pred":
"person",
"obj": [
{
"pred": "name",
"obj": "Sue",
"type": "simple"
},
{
"pred": "age",
"obj": 42,
"type": "simple"
},
],
"type":
"multi",
},
]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
def test_valid_strings_input(self):
input_str = '{"Animals": ["cow", "pig", "human"]}'
field = ListField("Animals", EmbeddedScalarField())
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
simple_links=(
SimpleLink(pred="animals", obj="cow"),
SimpleLink(pred="animals", obj="pig"),
SimpleLink(pred="animals", obj="human"),
),
)
self.assertEqual(link_collection, expected_link_collection)
def test_allow_scalar(self):
input_str = '{"People": "bob"}'
field = ListField("People",
EmbeddedScalarField(),
alti_key="person",
allow_scalar=True)
expected_output_data = ({
"pred": "person",
"obj": "bob",
"type": "simple"
}, )
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class TransitGatewayVpcAttachmentResourceSpec(EC2ResourceSpec):
"""Resource for Transit Gateway VPC Attachments"""
type_name = "transit-gateway-vpc-attachment"
schema = Schema(
ScalarField("TransitGatewayAttachmentId"),
ScalarField("TransitGatewayId"),
ScalarField("VpcId"),
ScalarField("VpcOwnerId"),
ScalarField("State"),
ScalarField("CreationTime"),
ListField("SubnetIds", EmbeddedScalarField(), alti_key="subnet_id"),
AnonymousDictField("Options", ScalarField("DnsSupport"),
ScalarField("Ipv6Support")),
)
@classmethod
def list_from_aws(
cls: Type["TransitGatewayVpcAttachmentResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
paginator = client.get_paginator(
"describe_transit_gateway_vpc_attachments")
attachments = {}
for resp in paginator.paginate():
for attachment in resp.get("TransitGatewayVpcAttachments", []):
resource_arn = cls.generate_arn(
account_id, region,
attachment["TransitGatewayAttachmentId"])
attachments[resource_arn] = attachment
return ListFromAWSResult(resources=attachments)
class IAMOIDCProviderResourceSpec(IAMResourceSpec):
"""Resource for IAM OIDC Providers"""
type_name = "oidc-provider"
schema = Schema(
ScalarField("Url"),
ScalarField("CreateDate"),
ListField("ClientIDList", EmbeddedScalarField(), alti_key="client_id"),
ListField("ThumbprintList",
EmbeddedScalarField(),
alti_key="thumbprint"),
)
@classmethod
def list_from_aws(cls: Type["IAMOIDCProviderResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'oidc_provider_1_arn': {oidc_provider_1_dict},
'oidc_provider_2_arn': {oidc_provider_2_dict},
...}
Where the dicts represent results from list_oidc_providers and additional info per
oidc_provider list_oidc_providers. An additional 'Name' key is added."""
oidc_providers = {}
resp = client.list_open_id_connect_providers()
for oidc_provider in resp.get("OpenIDConnectProviderList", []):
resource_arn = oidc_provider["Arn"]
try:
oidc_provider_details = cls.get_oidc_provider_details(
client=client, arn=resource_arn)
oidc_provider.update(oidc_provider_details)
oidc_providers[resource_arn] = oidc_provider
except ClientError as c_e:
error_code = getattr(c_e, "response",
{}).get("Error", {}).get("Code", {})
if error_code != "NoSuchEntity":
raise c_e
return ListFromAWSResult(resources=oidc_providers)
@classmethod
def get_oidc_provider_details(cls: Type["IAMOIDCProviderResourceSpec"],
client: BaseClient, arn: str) -> str:
oidc_provider_resp = client.get_open_id_connect_provider(
OpenIDConnectProviderArn=arn)
return oidc_provider_resp
class TransitGatewayResourceSpec(EC2ResourceSpec):
"""Resource for TransitGateways"""
# type_name = "transit-gateway"
type_name = "transit_gateway"
schema = Schema(
ScalarField("OwnerId"),
ScalarField("State"),
ListField(
"VPCAttachments",
EmbeddedDictField(
ScalarField("CreationTime"),
ScalarField("State"),
ResourceLinkField("ResourceId", VPCResourceSpec),
),
optional=True,
alti_key="vpc_attachment",
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["TransitGatewayResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'tgw_1_arn': {tgw_1_dict},
'tgw_2_arn': {tgw_2_dict},
...}
Where the dicts represent results from describe_transit_gateways."""
tgws = {}
paginator = client.get_paginator("describe_transit_gateways")
tgw_filters = [{"Name": "owner-id", "Values": [account_id]}]
for resp in paginator.paginate(Filters=tgw_filters):
for tgw in resp["TransitGateways"]:
resource_arn = tgw["TransitGatewayArn"]
vpc_attachments: List[Dict[str, Any]] = []
vpc_attachments_paginator = client.get_paginator(
"describe_transit_gateway_attachments"
)
vpc_filters = [
{"Name": "transit-gateway-id", "Values": [tgw["TransitGatewayId"]]},
{"Name": "resource-type", "Values": ["vpc"]},
]
for vpc_attachments_resp in vpc_attachments_paginator.paginate(Filters=vpc_filters):
vpc_attachments += vpc_attachments_resp["TransitGatewayAttachments"]
tgw["VPCAttachments"] = vpc_attachments
tgws[resource_arn] = tgw
return ListFromAWSResult(resources=tgws)
class EBSVolumeResourceSpec(EC2ResourceSpec):
"""Resource for EBSVolumes"""
type_name = "volume"
schema = Schema(
ScalarField("AvailabilityZone"),
ScalarField("CreateTime"),
ScalarField("Size"),
ScalarField("State"),
ScalarField("VolumeType"),
ScalarField("Encrypted"),
ListField(
"Attachments",
EmbeddedDictField(
ScalarField("AttachTime"),
ScalarField("State"),
ScalarField("DeleteOnTermination"),
ResourceLinkField("InstanceId", EC2InstanceResourceSpec),
),
optional=True,
alti_key="attachment",
),
TransientResourceLinkField("KmsKeyId",
KMSKeyResourceSpec,
optional=True,
value_is_id=True),
TagsField(),
)
@classmethod
def list_from_aws(cls: Type["EBSVolumeResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'volume_1_arn': {volume_1_dict},
'volume_2_arn': {volume_2_dict},
...}
Where the dicts represent results from describe_volumes."""
volumes = {}
paginator = client.get_paginator("describe_volumes")
for resp in paginator.paginate():
for volume in resp.get("Volumes", []):
resource_arn = cls.generate_arn(account_id=account_id,
region=region,
resource_id=volume["VolumeId"])
volumes[resource_arn] = volume
return ListFromAWSResult(resources=volumes)
class EventsRuleResourceSpec(EventsResourceSpec):
"""Resource for CloudWatchEvents Rules"""
type_name = "rule"
schema = Schema(
ScalarField("Name"),
ScalarField("State"),
ScalarField("EventPattern", optional=True),
ScalarField("ScheduleExpression", optional=True),
ListField(
"Targets",
EmbeddedDictField(ScalarField("Id", "name"), ScalarField("Arn"),
ScalarField("RoleArn", optional=True)),
alti_key="target",
),
)
@classmethod
def list_from_aws(cls: Type["EventsRuleResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'rule_1_arn': {rule_1_dict},
'rule_2_arn': {rule_2_dict},
...}
Where the dicts represent results from list_rules and additional info per rule from
list_targets_by_rule."""
rules = {}
paginator = client.get_paginator("list_rules")
for resp in paginator.paginate():
for rule in resp.get("Rules", []):
resource_arn = rule["Arn"]
try:
rule["Targets"] = list_targets_by_rule(
client=client, rule_name=rule["Name"])
rules[resource_arn] = rule
except ClientError as c_e:
error_code = getattr(c_e, "response",
{}).get("Error", {}).get("Code", {})
if error_code != "ResourceNotFoundException":
raise c_e
return ListFromAWSResult(resources=rules)
class EC2ImageResourceSpec(EC2ResourceSpec):
"""Resource for EC2Images (AMIs)"""
type_name = "image"
schema = Schema(
ScalarField("Name"),
ScalarField("Description", optional=True),
ScalarField("Public"),
ListField(
"LaunchPermissions",
EmbeddedDictField(
ResourceLinkField("UserId",
AccountResourceSpec,
optional=True,
alti_key="account")),
alti_key="launch_permission",
),
)
@classmethod
def list_from_aws(cls: Type["EC2ImageResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'image_1_arn': {image_1_dict},
'image_2_arn': {image_2_dict},
...}
Where the dicts represent results from describe_images."""
images = {}
resp = client.describe_images(Owners=["self"])
for image in resp["Images"]:
image_id = image["ImageId"]
time.sleep(
0.25) # seems necessary to avoid frequent RequestLimitExceeded
perms_resp = client.describe_image_attribute(
Attribute="launchPermission", ImageId=image_id)
launch_permissions = perms_resp["LaunchPermissions"]
image["LaunchPermissions"] = launch_permissions
resource_arn = cls.generate_arn(account_id, region, image_id)
images[resource_arn] = image
return ListFromAWSResult(resources=images)
class EventsRuleResourceSpec(EventsResourceSpec):
"""Resource for CloudWatchEvents Rules"""
type_name = "rule"
schema = Schema(
ScalarField("Name"),
ScalarField("State"),
ScalarField("EventPattern", optional=True),
ScalarField("ScheduleExpression", optional=True),
ListField(
"Targets",
EmbeddedDictField(ScalarField("Id", "name"),
ScalarField("Arn", "arn")),
alti_key="target",
),
)
@classmethod
def list_from_aws(cls: Type["EventsRuleResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'rule_1_arn': {rule_1_dict},
'rule_2_arn': {rule_2_dict},
...}
Where the dicts represent results from list_rules and additional info per rule from
list_targets_by_rule."""
rules = {}
paginator = client.get_paginator("list_rules")
for resp in paginator.paginate():
for rule in resp.get("Rules", []):
resource_arn = rule["Arn"]
rules[resource_arn] = rule
targets_paginator = client.get_paginator(
"list_targets_by_rule")
rule["Targets"] = []
for targets_resp in targets_paginator.paginate(
Rule=rule["Name"]):
rule["Targets"] += targets_resp.get("Targets", [])
return ListFromAWSResult(resources=rules)
class HostedZoneResourceSpec(Route53ResourceSpec):
"""Resource for S3 Buckets"""
type_name = "hostedzone"
schema = Schema(
ScalarField("Name"),
ListField(
"ResourceRecordSets",
EmbeddedDictField(
ScalarField("Name"), ScalarField("Type"), ScalarField("TTL", optional=True)
),
optional=True,
alti_key="resource_record_set",
),
)
@classmethod
def list_from_aws(
cls: Type["HostedZoneResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'hosted_zone_1_arn': {hosted_zone_1_dict},
'hosted_zone_2_arn': {hosted_zone_2_dict},
...}
Where the dicts represent results from list_hosted_zones."""
hosted_zones = {}
paginator = client.get_paginator("list_hosted_zones")
for resp in paginator.paginate():
for hosted_zone in resp.get("HostedZones", []):
hosted_zone_id = hosted_zone["Id"].split("/")[-1]
resource_arn = cls.generate_arn(resource_id=hosted_zone_id, account_id=account_id)
record_sets_paginator = client.get_paginator("list_resource_record_sets")
zone_resource_record_sets = []
for record_sets_resp in record_sets_paginator.paginate(HostedZoneId=hosted_zone_id):
zone_resource_record_sets += record_sets_resp.get("ResourceRecordSets", [])
hosted_zone["ResourceRecordSets"] = zone_resource_record_sets
hosted_zones[resource_arn] = hosted_zone
return ListFromAWSResult(resources=hosted_zones)
class InternetGatewayResourceSpec(EC2ResourceSpec):
"""Resource for InternetGateways"""
# type_name = "internet-gateway"
type_name = "internet_gateway"
schema = Schema(
ScalarField("OwnerId"),
ListField(
"Attachments",
EmbeddedDictField(
ScalarField("State"),
ResourceLinkField("VpcId", VPCResourceSpec),
),
optional=True,
alti_key="attachment",
),
TagsField(),
)
@classmethod
def list_from_aws(cls: Type["InternetGatewayResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'igw_1_arn': {igw_1_dict},
'igw_2_arn': {igw_2_dict},
...}
Where the dicts represent results from describe_internet_gateways."""
igws = {}
paginator = client.get_paginator("describe_internet_gateways")
for resp in paginator.paginate():
for igw in resp["InternetGateways"]:
resource_arn = cls.generate_arn(
resource_id=igw["InternetGatewayId"],
account_id=account_id,
region=region)
igws[resource_arn] = igw
return ListFromAWSResult(resources=igws)
class VpcEndpointServiceResourceSpec(EC2ResourceSpec):
"""Resource for VPC Endpoint Services"""
# type_name = "vpc-endpoint-service"
type_name = "vpc_endpoint_service"
schema = Schema(
AnonymousListField("ServiceType", ScalarField("ServiceType")),
ScalarField("ServiceName"),
ScalarField("ServiceState"),
ScalarField("AcceptanceRequired"),
ListField("AvailabilityZones", EmbeddedScalarField()),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["VpcEndpointServiceResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'vpc_endpoint_svc_1_arn': {vpc_endpoint_svc_1_dict},
'vpc_endpoint_svc_2_arn': {vpc_endpoint_svc_2_dict},
...}
Where the dicts represent results from describe_vpc_endpoint_service_configurations."""
services = {}
paginator = client.get_paginator(
"describe_vpc_endpoint_service_configurations")
for resp in paginator.paginate():
for service in resp.get("ServiceConfigurations", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=service["ServiceId"])
services[resource_arn] = service
return ListFromAWSResult(resources=services)
class RDSInstanceResourceSpec(RDSResourceSpec):
"""Resource for RDS"""
type_name = "db"
schema = Schema(
TagsField(),
ScalarField("DBInstanceIdentifier"),
ScalarField("DBInstanceClass"),
ScalarField("Engine"),
ScalarField("DBInstanceStatus"),
ScalarField("DBName", optional=True),
AnonymousDictField(
"Endpoint",
ScalarField("Address", alti_key="endpoint_address", optional=True),
ScalarField("Port", alti_key="endpoint_port"),
ScalarField("HostedZoneId",
alti_key="endpoint_hosted_zone",
optional=True),
optional=True,
),
AnonymousDictField(
"ListenerEndpoint",
ScalarField("Address", alti_key="listener_address"),
ScalarField("Port", alti_key="listener_port"),
ScalarField("HostedZoneId",
alti_key="listener_hosted_zone",
optional=True),
optional=True,
),
ScalarField("InstanceCreateTime", optional=True),
ScalarField("BackupRetentionPeriod"),
AnonymousListField(
"VpcSecurityGroups",
AnonymousEmbeddedDictField(
TransientResourceLinkField("VpcSecurityGroupId",
SecurityGroupResourceSpec,
optional=True)),
),
ScalarField("AvailabilityZone", optional=True),
AnonymousDictField("DBSubnetGroup",
TransientResourceLinkField("VpcId",
VPCResourceSpec),
optional=True),
ScalarField("MultiAZ"),
ScalarField("PubliclyAccessible"),
ListField("StatusInfos",
EmbeddedDictField(ScalarField("Status")),
optional=True),
ScalarField("StorageType"),
ScalarField("StorageEncrypted"),
TransientResourceLinkField("KmsKeyId",
KMSKeyResourceSpec,
optional=True,
value_is_id=True),
ScalarField("DbiResourceId"),
ScalarField("Timezone", optional=True),
ScalarField("IAMDatabaseAuthenticationEnabled"),
ScalarField("PerformanceInsightsEnabled", optional=True),
ScalarField("PerformanceInsightsRetentionPeriod", optional=True),
ScalarField("DeletionProtection"),
ListField(
"Backup",
EmbeddedDictField(
AnonymousDictField(
"RestoreWindow",
ScalarField("EarliestTime",
alti_key="earliest_restore_time",
optional=True),
optional=True,
),
AnonymousDictField(
"RestoreWindow",
ScalarField("LatestTime",
alti_key="latest_restore_time",
optional=True),
optional=True,
),
ScalarField("AllocatedStorage"),
ScalarField("Status"),
ScalarField("AvailabilityZone", optional=True),
ScalarField("Engine"),
ScalarField("EngineVersion"),
ScalarField("Encrypted"),
ScalarField("StorageType"),
TransientResourceLinkField("KmsKeyId",
KMSKeyResourceSpec,
optional=True,
value_is_id=True),
),
optional=True,
),
)
@classmethod
def list_from_aws(cls: Type["RDSInstanceResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
logger = Logger()
dbinstances = {}
paginator = client.get_paginator("describe_db_instances")
for resp in paginator.paginate():
for db in resp.get("DBInstances", []):
resource_arn = db["DBInstanceArn"]
db["Tags"] = client.list_tags_for_resource(
ResourceName=resource_arn).get("TagList", [])
db["Backup"] = []
dbinstances[resource_arn] = db
backup_paginator = client.get_paginator(
"describe_db_instance_automated_backups")
for resp in backup_paginator.paginate():
for backup in resp.get("DBInstanceAutomatedBackups", []):
if backup["DBInstanceArn"] in dbinstances:
dbinstances[backup["DBInstanceArn"]]["Backup"].append(
backup)
else:
logger.info(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=
(f'Unable to find matching DB Instance {backup["DBInstanceArn"]} '
"(Possible Deletion)"),
)
return ListFromAWSResult(resources=dbinstances)
class IAMRoleResourceSpec(IAMResourceSpec):
"""Resource for IAM Roles"""
type_name = "role"
schema = Schema(
ScalarField("RoleName", "name"),
ScalarField("MaxSessionDuration"),
AnonymousListField(
"PolicyAttachments",
AnonymousEmbeddedDictField(
ResourceLinkField(
"PolicyArn",
IAMPolicyResourceSpec,
optional=True,
value_is_id=True,
alti_key="attached_policy",
)),
),
DictField(
"AssumeRolePolicyDocument",
ScalarField("Version"),
ListField(
"Statement",
EmbeddedDictField(
ScalarField("Effect"),
ScalarField("Action"),
DictField(
"Principal",
ListField("AWS",
EmbeddedScalarField(),
optional=True,
allow_scalar=True),
ListField("Federated",
EmbeddedScalarField(),
optional=True,
allow_scalar=True),
),
),
),
),
ScalarField("AssumeRolePolicyDocumentText"),
)
@classmethod
def list_from_aws(cls: Type["IAMRoleResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'role_1_arn': {role_1_dict},
'role_2_arn': {role_2_dict},
...}
Where the dicts represent results from list_roles and additional info per role from
list_targets_by_role."""
roles = {}
paginator = client.get_paginator("list_roles")
for resp in paginator.paginate():
for role in resp.get("Roles", []):
role_name = role["RoleName"]
assume_role_policy_document = copy.deepcopy(
role["AssumeRolePolicyDocument"])
assume_role_policy_document_text = policy_doc_dict_to_sorted_str(
assume_role_policy_document)
role[
"AssumeRolePolicyDocumentText"] = assume_role_policy_document_text
for statement in assume_role_policy_document.get(
"Statement", []):
for obj in statement.get("Condition", {}).values():
for obj_key in obj.keys():
if obj_key.lower() == "sts:externalid":
obj[obj_key] = "REMOVED"
policies_result = get_attached_role_policies(client, role_name)
policies = policies_result
role["PolicyAttachments"] = policies
resource_arn = role["Arn"]
roles[resource_arn] = role
return ListFromAWSResult(resources=roles)
class S3BucketResourceSpec(S3ResourceSpec):
"""Resource for S3 Buckets"""
type_name = "bucket"
scan_granularity = ScanGranularity.ACCOUNT
schema = Schema(
ScalarField("Name"),
ScalarField("CreationDate"),
AnonymousDictField(
"ServerSideEncryption",
ListField(
"Rules",
EmbeddedDictField(
AnonymousDictField(
"ApplyServerSideEncryptionByDefault",
ScalarField("SSEAlgorithm", "algorithm"),
),
AnonymousDictField(
"ApplyServerSideEncryptionByDefault",
ResourceLinkField("KMSMasterKeyID", KMSKeyResourceSpec, optional=True),
),
),
alti_key="server_side_default_encryption_rule",
),
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["S3BucketResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'bucket_1_arn': {bucket_1_dict},
'bucket_2_arn': {bucket_2_dict},
...}
Where the dicts represent results from list_buckets."""
logger = Logger()
buckets = {}
buckets_resp = client.list_buckets()
for bucket in buckets_resp.get("Buckets", []):
bucket_name = bucket["Name"]
try:
try:
bucket_region = get_s3_bucket_region(client, bucket_name)
except S3BucketAccessDeniedException as s3ade:
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"Unable to determine region for {bucket_name}: {s3ade}",
)
continue
try:
bucket["Tags"] = get_s3_bucket_tags(client, bucket_name)
except S3BucketAccessDeniedException as s3ade:
bucket["Tags"] = []
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"Unable to determine tags for {bucket_name}: {s3ade}",
)
try:
bucket["ServerSideEncryption"] = get_s3_bucket_encryption(client, bucket_name)
except S3BucketAccessDeniedException as s3ade:
bucket["ServerSideEncryption"] = {"Rules": []}
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"Unable to determine encryption status for {bucket_name}: {s3ade}",
)
resource_arn = cls.generate_arn(
account_id=account_id, region=bucket_region, resource_id=bucket_name
)
buckets[resource_arn] = bucket
except S3BucketDoesNotExistException as s3bdnee:
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"{bucket_name}: No longer exists: {s3bdnee}",
)
return ListFromAWSResult(resources=buckets)
class IAMUserResourceSpec(IAMResourceSpec):
"""Resource for IAM Users"""
type_name = "user"
schema = Schema(
ScalarField("UserName", "name"),
ScalarField("UserId"),
ScalarField("CreateDate"),
ScalarField("PasswordLastUsed", optional=True),
ListField(
"AccessKeys",
EmbeddedDictField(
ScalarField("AccessKeyId"),
ScalarField("Status"),
ScalarField("CreateDate"),
AnonymousDictField("AccessKeyLastUsed",
ScalarField("LastUsedDate", optional=True)),
),
optional=True,
alti_key="access_key",
),
DictField(
"LoginProfile",
ScalarField("CreateDate"),
ScalarField("PasswordResetRequired"),
optional=True,
),
ListField(
"MfaDevices",
EmbeddedDictField(ScalarField("SerialNumber"),
ScalarField("EnableDate")),
optional=True,
),
)
@classmethod
def list_from_aws(cls: Type["IAMUserResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'user_1_arn': {user_1_dict},
'user_2_arn': {user_2_dict},
...}
Where the dicts represent results from list_users and additional info per user from
list_targets_by_user."""
users = {}
paginator = client.get_paginator("list_users")
for resp in paginator.paginate():
for user in resp.get("Users", []):
resource_arn = user["Arn"]
user_name = user["UserName"]
access_keys_paginator = client.get_paginator(
"list_access_keys")
access_keys: List[Dict[str, Any]] = []
for access_keys_resp in access_keys_paginator.paginate(
UserName=user_name):
for resp_access_key in access_keys_resp[
"AccessKeyMetadata"]:
access_key = copy.deepcopy(resp_access_key)
access_key_id = access_key["AccessKeyId"]
last_used_resp = client.get_access_key_last_used(
AccessKeyId=access_key_id)
access_key["AccessKeyLastUsed"] = last_used_resp[
"AccessKeyLastUsed"]
access_keys.append(access_key)
user["AccessKeys"] = access_keys
mfa_devices_paginator = client.get_paginator(
"list_mfa_devices")
mfa_devices: List[Dict[str, Any]] = []
for mfa_devices_resp in mfa_devices_paginator.paginate(
UserName=user_name):
mfa_devices += mfa_devices_resp["MFADevices"]
user["MfaDevices"] = mfa_devices
try:
login_profile_resp = client.get_login_profile(
UserName=user_name)
user["LoginProfile"] = login_profile_resp["LoginProfile"]
except ClientError as c_e:
if "NoSuchEntity" not in str(c_e):
raise c_e
users[resource_arn] = user
return ListFromAWSResult(resources=users)
class ClassicLoadBalancerResourceSpec(ELBV1ResourceSpec):
"""Resource for classic load balancer"""
type_name = "loadbalancer"
schema = Schema(
ScalarField("DNSName"),
ScalarField("CreatedTime"),
ScalarField("LoadBalancerName"),
ScalarField("Scheme"),
ResourceLinkField("VPCId", VPCResourceSpec, optional=True),
ListField("Subnets",
EmbeddedResourceLinkField(SubnetResourceSpec),
optional=True),
ListField("SecurityGroups",
EmbeddedResourceLinkField(SecurityGroupResourceSpec),
optional=True),
ScalarField("Type"),
ScalarField("AccessLogsEnabled"),
TransientResourceLinkField(
"AccessLogsS3Bucket",
S3BucketResourceSpec,
alti_key="access_logs_s3_bucket",
optional=True,
),
ScalarField("AccessLogsS3Prefix", optional=True),
)
@classmethod
def list_from_aws(
cls: Type["ClassicLoadBalancerResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'lb_1_arn': {lb_1_dict},
'lb_2_arn': {lb_2_dict},
...}
Where the dicts represent results from describe_load_balancers."""
paginator = client.get_paginator("describe_load_balancers")
load_balancers = {}
for resp in paginator.paginate():
for lb in resp["LoadBalancerDescriptions"]:
lb_name = lb["LoadBalancerName"]
resource_arn = cls.generate_arn(account_id=account_id,
region=region,
resource_id=lb_name)
try:
lb_attrs = cls.get_lb_attrs(client, lb_name)
lb.update(lb_attrs)
lb["Type"] = "classic"
load_balancers[resource_arn] = lb
except ClientError as c_e:
if (getattr(c_e, "response", {}).get("Error", {}).get(
"Code", {}) != "LoadBalancerNotFound"):
raise c_e
return ListFromAWSResult(resources=load_balancers)
@classmethod
def get_lb_attrs(
cls: Type["ClassicLoadBalancerResourceSpec"],
client: BaseClient,
lb_name: str,
) -> Dict[str, str]:
"""Get lb attributes that Altimeter graphs."""
lb_attrs = {}
resp = client.describe_load_balancer_attributes(
LoadBalancerName=lb_name)
access_log_attrs = resp["LoadBalancerAttributes"]["AccessLog"]
lb_attrs["AccessLogsEnabled"] = access_log_attrs["Enabled"]
if "S3BucketName" in access_log_attrs:
lb_attrs["AccessLogsS3Bucket"] = access_log_attrs["S3BucketName"]
if "S3BucketPrefix" in access_log_attrs:
lb_attrs["AccessLogsS3Prefix"] = access_log_attrs["S3BucketPrefix"]
return lb_attrs
class DetectorResourceSpec(GuardDutyResourceSpec):
"""Resource for GuardDuty Detectors"""
type_name = "detector"
schema = Schema(
ScalarField("CreatedAt"),
ScalarField("FindingPublishingFrequency"),
ScalarField("ServiceRole"),
ScalarField("Status"),
ScalarField("UpdatedAt"),
ListField(
"Members",
EmbeddedDictField(
TransientResourceLinkField("DetectorArn",
"DetectorResourceSpec",
value_is_id=True),
ScalarField("Email"),
ScalarField("RelationshipStatus"),
ScalarField("InvitedAt"),
ScalarField("UpdatedAt"),
),
alti_key="member",
),
AnonymousDictField(
"Master",
ScalarField("AccountId", alti_key="master_account_id"),
ScalarField("RelationshipStatus",
alti_key="master_relationship_status"),
# ScalarField("InvitedAt", alti_key="master_invited_at"),
ScalarField("InvitedAt",
alti_key="master_invited_at",
optional=True),
optional=True,
),
)
@classmethod
def list_from_aws(cls: Type["DetectorResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'detector_1_arn': {detector_1_dict},
'detector_2_arn': {detector_2_dict},
...}
Where the dicts represent results from list_detectors and list_members, get_detector for
each listed detector."""
list_detectors_paginator = client.get_paginator("list_detectors")
detectors: Dict[str, Dict[str, Any]] = {}
for list_detectors_resp in list_detectors_paginator.paginate():
detector_ids = list_detectors_resp["DetectorIds"]
for detector_id in detector_ids:
resource_arn = cls.generate_arn(account_id=account_id,
region=region,
resource_id=detector_id)
try:
detectors[resource_arn] = cls.get_detector(
client, detector_id, region)
except ClientError as c_e:
error_code = getattr(c_e, "response",
{}).get("Error", {}).get("Code", {})
if error_code != "BadRequestException":
raise c_e
return ListFromAWSResult(resources=detectors)
@classmethod
def get_detector(cls: Type["DetectorResourceSpec"], client: BaseClient,
detector_id: str, region: str) -> Dict[str, Any]:
detector_resp = client.get_detector(DetectorId=detector_id)
detector = {
key: detector_resp[key]
for key in (
"CreatedAt",
"FindingPublishingFrequency",
"ServiceRole",
"Status",
"UpdatedAt",
)
}
detector["Members"] = cls.get_detector_members(client, detector_id,
region)
master_account_resp = client.get_master_account(DetectorId=detector_id)
master_account_dict = master_account_resp.get("Master")
if master_account_dict:
detector["Master"] = {
# key: master_account_dict[key]
key: master_account_dict.get(key)
for key in (
"AccountId",
"RelationshipStatus",
"InvitedAt",
)
}
return detector
@classmethod
def get_detector_members(cls: Type["DetectorResourceSpec"],
client: BaseClient, detector_id: str,
region: str) -> List[Dict[str, Any]]:
member_resps: List[Dict[str, Any]] = []
list_members_paginator = client.get_paginator("list_members")
for list_members_resp in list_members_paginator.paginate(
DetectorId=detector_id):
member_resps += list_members_resp.get("Members", [])
members = []
if member_resps:
for member_resp in member_resps:
member_account_id = member_resp["AccountId"]
member_detector_id = member_resp["DetectorId"]
# member_email = member_resp["Email"]
member_email = member_resp.get("Email", "[NONE]")
member_relationship_status = member_resp["RelationshipStatus"]
# member_invited_at = member_resp["InvitedAt"]
member_invited_at = member_resp.get("InvitedAt", "[NONE]")
member_updated_at = member_resp["UpdatedAt"]
member_detector_arn = cls.generate_arn(
account_id=member_account_id,
region=region,
resource_id=member_detector_id,
)
member = {
"DetectorArn": member_detector_arn,
"Email": member_email,
"RelationshipStatus": member_relationship_status,
"InvitedAt": member_invited_at,
"UpdatedAt": member_updated_at,
}
members.append(member)
return members
class SecurityGroupResourceSpec(EC2ResourceSpec):
"""Resource for SecurityGroups"""
type_name = "security-group"
schema = Schema(
ScalarField("GroupName", "name"),
ListField(
"IpPermissions",
EmbeddedDictField(
ScalarField("IpProtocol"),
ScalarField("FromPort", default_value=0),
ScalarField("ToPort", default_value=65535),
ListField(
"IpRanges",
EmbeddedDictField(
ScalarField("CidrIp"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ip_range",
optional=True,
),
ListField(
"Ipv6Ranges",
EmbeddedDictField(
ScalarField("CidrIpv6"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ipv6_range",
optional=True,
),
ListField(
"PrefixListIds", EmbeddedDictField(ScalarField("PrefixListId")), optional=True
),
ListField(
"UserIdGroupPairs",
EmbeddedDictField(
ResourceLinkField("GroupId", "SecurityGroupResourceSpec"),
ScalarField("UserId", alti_key="account_id", optional=True),
ScalarField("PeeringStatus", optional=True),
ScalarField("VpcId", optional=True),
ScalarField("VpcPeeringConnectionId", optional=True),
),
alti_key="user_id_group_pairs",
),
),
alti_key="ingress_rule",
),
ListField(
"IpPermissionsEgress",
EmbeddedDictField(
ScalarField("IpProtocol"),
ScalarField("FromPort", default_value=0),
ScalarField("ToPort", default_value=65535),
ListField(
"IpRanges",
EmbeddedDictField(
ScalarField("CidrIp"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ip_range",
optional=True,
),
ListField(
"Ipv6Ranges",
EmbeddedDictField(
ScalarField("CidrIpv6"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ipv6_range",
optional=True,
),
ListField(
"PrefixListIds", EmbeddedDictField(ScalarField("PrefixListId")), optional=True
),
ListField(
"UserIdGroupPairs",
EmbeddedDictField(
ResourceLinkField("GroupId", "SecurityGroupResourceSpec"),
ScalarField("UserId", alti_key="account_id", optional=True),
ScalarField("PeeringStatus", optional=True),
ScalarField("VpcId", optional=True),
ScalarField("VpcPeeringConnectionId", optional=True),
),
alti_key="user_id_group_pairs",
),
),
alti_key="egress_rule",
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["SecurityGroupResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'security_group_1_arn': {security_group_1_dict},
'security_group_2_arn': {security_group_2_dict},
...}
Where the dicts represent results from describe_subnets."""
security_groups = {}
paginator = client.get_paginator("describe_security_groups")
for resp in paginator.paginate():
for security_group in resp.get("SecurityGroups", []):
resource_arn = cls.generate_arn(
account_id=account_id, region=region, resource_id=security_group["GroupId"]
)
for ingress_rule in security_group.get("IpPermissions", []):
for ip_range in ingress_rule.get("IpRanges", []):
cidr = ip_range["CidrIp"]
ipv4_network = ipaddress.IPv4Network(cidr, strict=False)
first_ip, last_ip = int(ipv4_network[0]), int(ipv4_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
for ip_range in ingress_rule.get("Ipv6Ranges", []):
cidr = ip_range["CidrIpv6"]
ipv6_network = ipaddress.IPv6Network(cidr, strict=False)
first_ip, last_ip = int(ipv6_network[0]), int(ipv6_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
for egress_rule in security_group.get("IpPermissionsEgress", []):
for ip_range in egress_rule.get("IpRanges", []):
cidr = ip_range["CidrIp"]
ipv4_network = ipaddress.IPv4Network(cidr, strict=False)
first_ip, last_ip = int(ipv4_network[0]), int(ipv4_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
for ip_range in egress_rule.get("Ipv6Ranges", []):
cidr = ip_range["CidrIpv6"]
ipv6_network = ipaddress.IPv6Network(cidr, strict=False)
first_ip, last_ip = int(ipv6_network[0]), int(ipv6_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
security_groups[resource_arn] = security_group
return ListFromAWSResult(resources=security_groups)