def test_value_not_a_string(self):
input_str = '{"FieldName": [1, 2, 3]}'
field = ResourceLinkField("FieldName", TestResourceSpec)
input_data = json.loads(input_str)
with self.assertRaises(ResourceLinkFieldValueNotAStringException):
field.parse(data=input_data, context={})
def test_key_absent_without_optional(self):
input_str = "{}"
field = ResourceLinkField("FieldName", TestResourceSpec)
input_data = json.loads(input_str)
with self.assertRaises(ResourceLinkFieldSourceKeyNotFoundException):
field.parse(data=input_data, context={})
class EC2InstanceResourceSpec(EC2ResourceSpec):
"""Resource for EC2Instances"""
type_name = "instance"
schema = Schema(
ScalarField("Name", optional=True),
TransientResourceLinkField("ImageId", EC2ImageResourceSpec),
ScalarField("KeyName", optional=True),
AnonymousDictField("Placement", ScalarField("AvailabilityZone"), ScalarField("Tenancy")),
ScalarField("InstanceType"),
ScalarField("LaunchTime"),
AnonymousDictField("State", ScalarField("Name", "state")),
ScalarField("Platform", optional=True),
ScalarField("PrivateIpAddress", optional=True),
ScalarField("PrivateDnsName", optional=True),
ScalarField("PublicIpAddress", optional=True),
ScalarField("PublicDnsName", optional=True),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
ResourceLinkField("SubnetId", SubnetResourceSpec, optional=True),
AnonymousListField(
"SecurityGroups",
AnonymousEmbeddedDictField(ResourceLinkField("GroupId", SecurityGroupResourceSpec)),
),
AnonymousDictField(
"IamInstanceProfile",
TransientResourceLinkField(
"Arn", InstanceProfileResourceSpec, alti_key="instance_profile", value_is_id=True
),
optional=True,
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["EC2InstanceResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'instance_1_arn': {instance_1_dict},
'instance_2_arn': {instance_2_dict},
...}
Where the dicts represent results from describe_instances."""
paginator = client.get_paginator("describe_instances")
instances = {}
for resp in paginator.paginate():
for reservation in resp.get("Reservations", []):
for instance in reservation.get("Instances", []):
resource_arn = cls.generate_arn(
account_id=account_id, region=region, resource_id=instance["InstanceId"]
)
instances[resource_arn] = instance
for tag in instance.get("Tags", []):
if tag["Key"].lower() == "name":
instance["Name"] = tag["Value"]
break
return ListFromAWSResult(resources=instances)
def test_key_absent_with_optional(self):
input_str = "{}"
field = ResourceLinkField("FieldName", TestResourceSpec, optional=True)
expected_output_data = []
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class LoadBalancerResourceSpec(ElasticLoadBalancingResourceSpec):
"""Resource for load balancer"""
type_name = "loadbalancer"
schema = Schema(
ScalarField("DNSName"),
ScalarField("CreatedTime"),
ScalarField("LoadBalancerName"),
ScalarField("Scheme"),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
AnonymousDictField("State",
ScalarField("Code",
alti_key="load_balancer_state")),
ScalarField("Type"),
ListField(
"AvailabilityZones",
EmbeddedDictField(
ScalarField("ZoneName"),
ResourceLinkField("SubnetId",
SubnetResourceSpec,
optional=True),
ListField(
"LoadBalancerAddresses",
EmbeddedDictField(
ScalarField("IpAddress", optional=True),
ScalarField("AllocationId", optional=True),
),
optional=True,
),
),
),
ListField("SecurityGroups",
EmbeddedResourceLinkField(SecurityGroupResourceSpec),
optional=True),
ScalarField("IpAddressType"),
)
@classmethod
def list_from_aws(cls: Type["LoadBalancerResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'lb_1_arn': {lb_1_dict},
'lb_2_arn': {lb_2_dict},
...}
Where the dicts represent results from describe_load_balancers."""
paginator = client.get_paginator("describe_load_balancers")
load_balancers = {}
for resp in paginator.paginate():
for lb in resp.get("LoadBalancers", []):
resource_arn = lb["LoadBalancerArn"]
load_balancers[resource_arn] = lb
return ListFromAWSResult(resources=load_balancers)
class EC2NetworkInterfaceResourceSpec(EC2ResourceSpec):
"""Resource for EC2NetworkInterfaces"""
# type_name = "network-interface"
type_name = "network_interface"
schema = Schema(
AnonymousDictField(
"Association",
ScalarField("PublicDnsName", optional=True),
ScalarField("PublicIp", optional=True),
optional=True,
),
ScalarField("Description"),
ScalarField("InterfaceType"),
ScalarField("MacAddress"),
ScalarField("PrivateDnsName", optional=True),
ScalarField("PrivateIpAddress", optional=True),
ScalarField("Status"),
ResourceLinkField("SubnetId", SubnetResourceSpec, optional=True),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
AnonymousListField(
"Groups",
AnonymousEmbeddedDictField(
ResourceLinkField("GroupId", SecurityGroupResourceSpec)),
),
)
@classmethod
def list_from_aws(
cls: Type["EC2NetworkInterfaceResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'network_interface_1_arn': {network_interface_1_dict},
'network_interface_2_arn': {network_interface_2_dict},
...}
Where the dicts represent results from describe_network_interfaces."""
paginator = client.get_paginator("describe_network_interfaces")
interfaces = {}
for resp in paginator.paginate():
for interface in resp.get("NetworkInterfaces", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=interface["NetworkInterfaceId"],
)
interfaces[resource_arn] = interface
return ListFromAWSResult(resources=interfaces)
def test_valid_input_str_classname(self):
input_str = '{"FieldName": "Value"}'
field = ResourceLinkField("FieldName", "TestResourceSpec")
expected_output_data = [{
"pred": "test_type_name",
"obj": "test_type_name:Value",
"type": "resource_link"
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
def test_key_present_with_optional(self):
input_str = '{"FieldName": "Value"}'
field = ResourceLinkField("FieldName", TestResourceSpec, optional=True)
expected_output_data = [{
"pred": "test_type_name",
"obj": "test_type_name:Value",
"type": "resource_link"
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class SubnetResourceSpec(EC2ResourceSpec):
"""Resource for Subnets"""
type_name = "subnet"
schema = Schema(
ScalarField("CidrBlock"),
ScalarField("FirstIp"),
ScalarField("LastIp"),
ScalarField("State"),
TagsField(),
ResourceLinkField("VpcId", VPCResourceSpec),
)
@classmethod
def list_from_aws(
cls: Type["SubnetResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
subnets = {}
resp = client.describe_subnets()
for subnet in resp.get("Subnets", []):
resource_arn = cls.generate_arn(
account_id=account_id, region=region, resource_id=subnet["SubnetId"]
)
cidr = subnet["CidrBlock"]
ipv4_network = ipaddress.IPv4Network(cidr, strict=False)
first_ip, last_ip = int(ipv4_network[0]), int(ipv4_network[-1])
subnet["FirstIp"] = first_ip
subnet["LastIp"] = last_ip
subnets[resource_arn] = subnet
return ListFromAWSResult(resources=subnets)
class EBSSnapshotResourceSpec(EC2ResourceSpec):
"""Resource for EBSSnapshots"""
type_name = "snapshot"
schema = Schema(
ScalarField("VolumeSize"),
ScalarField("Encrypted"),
ResourceLinkField("KmsKeyId", KMSKeyResourceSpec, optional=True, value_is_id=True),
TransientResourceLinkField("VolumeId", EBSVolumeResourceSpec),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["EBSSnapshotResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'snapshot_1_arn': {snapshot_1_dict},
'snapshot_2_arn': {snapshot_2_dict},
...}
Where the dicts represent results from describe_snapshots."""
snapshots = {}
paginator = client.get_paginator("describe_snapshots")
for resp in paginator.paginate(OwnerIds=["self"]):
for snapshot in resp.get("Snapshots", []):
resource_arn = cls.generate_arn(account_id, region, snapshot["SnapshotId"])
snapshots[resource_arn] = snapshot
return ListFromAWSResult(resources=snapshots)
class EC2RouteTableResourceSpec(EC2ResourceSpec):
"""Resource for Route Tables"""
# type_name = "route-table"
type_name = "route_table"
schema = Schema(
ScalarField("RouteTableId"),
ResourceLinkField("VpcId", VPCResourceSpec),
ScalarField("OwnerId", optional=True),
AnonymousListField("PropagatingVgws",
ScalarField("GatewayId"),
optional=True),
ListField(
"Routes",
EmbeddedDictField(
ScalarField("DestinationCidrBlock", optional=True),
ScalarField("DestinationIpv6CidrBlock", optional=True),
ScalarField("DestinationPrefixListId", optional=True),
ScalarField("EgressOnlyInternetGatewayId", optional=True),
ScalarField("GatewayId", optional=True),
ScalarField("InstanceId", optional=True),
ScalarField("InstanceOwnerId", optional=True),
ScalarField("NatGatewayId", optional=True),
ScalarField("TransitGatewayId", optional=True),
ScalarField("NetworkInterfaceId", optional=True),
ScalarField("Origin", optional=True),
ScalarField("State", optional=True),
ScalarField("VpcPeeringConnectionId", optional=True),
),
optional=True,
alti_key="route",
),
ListField(
"Associations",
EmbeddedDictField(
ScalarField("Main", optional=True),
ScalarField("RouteTableAssociationId", optional=True),
ScalarField("RouteTableId", optional=True),
ScalarField("SubnetId", optional=True),
),
optional=True,
alti_key="association",
),
)
@classmethod
def list_from_aws(cls: Type["EC2RouteTableResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
paginator = client.get_paginator("describe_route_tables")
route_tables = {}
for resp in paginator.paginate():
for attachment in resp.get("RouteTables", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=attachment["RouteTableId"])
route_tables[resource_arn] = attachment
return ListFromAWSResult(resources=route_tables)
class VpcEndpointResourceSpec(EC2ResourceSpec):
"""Resource for VPC Endpoints"""
type_name = "vpc-endpoint"
schema = Schema(
ScalarField("VpcEndpointType"),
ScalarField("ServiceName"),
ScalarField("State"),
ResourceLinkField("VpcId", VPCResourceSpec),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["VpcEndpointResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'vpc_endpoint_1_arn': {vpc_endpoint_1_dict},
'vpc_endpoint_1_arn': {vpc_endpoint_2_dict},
...}
Where the dicts represent results from describe_vpc_endpoints."""
endpoints = {}
paginator = client.get_paginator("describe_vpc_endpoints")
for resp in paginator.paginate():
for endpoint in resp.get("VpcEndpoints", []):
resource_arn = cls.generate_arn(account_id, region, endpoint["VpcEndpointId"])
endpoints[resource_arn] = endpoint
return ListFromAWSResult(resources=endpoints)
class InstanceProfileResourceSpec(IAMResourceSpec):
"""Resource for Instance Profiles"""
# type_name = "instance-profile"
type_name = "instance_profile"
schema = Schema(
ScalarField("InstanceProfileName", alti_key="name"),
AnonymousListField(
"Roles",
AnonymousEmbeddedDictField(
ResourceLinkField("Arn",
IAMRoleResourceSpec,
value_is_id=True,
alti_key="attached_role")),
),
)
@classmethod
def list_from_aws(cls: Type["InstanceProfileResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'instance_profile_1_arn': {instance_profile_1_dict},
'instance_profile_2_arn': {instance_profile_2_dict},
...}
Where the dicts represent results from list_instance_profiles."""
paginator = client.get_paginator("list_instance_profiles")
instance_profiles = {}
for resp in paginator.paginate():
for instance_profile in resp.get("InstanceProfiles", []):
resource_arn = instance_profile["Arn"]
instance_profiles[resource_arn] = instance_profile
return ListFromAWSResult(resources=instance_profiles)
class OUResourceSpec(OrganizationsResourceSpec):
"""Resource representing an AWS OU."""
type_name = "ou"
schema = Schema(
ScalarField("Path"),
ResourceLinkField("OrganizationArn", OrgResourceSpec,
value_is_id=True))
@classmethod
def get_full_type_name(cls: Type["OUResourceSpec"]) -> str:
return f"{cls.provider_name}:{cls.type_name}"
@classmethod
def list_from_aws(cls: Type["OUResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'ou_1_arn': {ou_1_dict},
'ou_2_arn': {ou_2_dict},
...}
Where the dicts represent results from list_organizational_units_for_parent
with some additional info 'Path') tagged on."""
org_resp = client.describe_organization()
org_arn = org_resp["Organization"]["Arn"]
ous = {}
paginator = client.get_paginator("list_roots")
for resp in paginator.paginate():
for root in resp["Roots"]:
root_id, root_arn = root["Id"], root["Arn"]
root_path = f"/{root['Name']}"
ous[root_arn] = root
ous[root_arn]["OrganizationArn"] = org_arn
ous[root_arn]["Path"] = root_path
ou_details = cls._recursively_get_ou_details_for_parent(
client=client, parent_id=root_id, parent_path=root_path)
for ou_detail in ou_details:
arn = ou_detail["Arn"]
ou_detail["OrganizationArn"] = org_arn
ous[arn] = ou_detail
return ListFromAWSResult(resources=ous)
@classmethod
def _recursively_get_ou_details_for_parent(
cls: Type["OUResourceSpec"], client: BaseClient, parent_id: str,
parent_path: str) -> List[Dict[str, Any]]:
ous = []
paginator = client.get_paginator(
"list_organizational_units_for_parent")
for resp in paginator.paginate(ParentId=parent_id):
for ou in resp["OrganizationalUnits"]:
ou_id = ou["Id"]
path = f"{parent_path}/{ou['Name']}"
ou["Path"] = path
ous.append(ou)
ous += cls._recursively_get_ou_details_for_parent(
client=client, parent_id=ou_id, parent_path=path)
return ous
class OrgsAccountResourceSpec(OrganizationsResourceSpec):
"""Resource representing an AWS Account as viewed in Orgs."""
type_name = "account"
schema = Schema(
ScalarField("Name"),
ScalarField("Status"),
ScalarField("JoinedTimestamp"),
ScalarField("Email"),
ScalarField("Id", alti_key="account_id"),
ResourceLinkField("OrganizationArn", OrgResourceSpec, value_is_id=True),
ResourceLinkField("OUArn", OUResourceSpec, value_is_id=True),
)
allow_clobber = [UnscannedAccountResourceSpec]
@classmethod
def get_full_type_name(cls: Type["OrgsAccountResourceSpec"]) -> str:
return f"{cls.provider_name}:{cls.type_name}"
@classmethod
def list_from_aws(
cls: Type["OrgsAccountResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'ou_1_arn': {org_1_dict},
'ou_2_arn': {org_2_dict},
...}
Where the dicts represent results from list_accounts_for_parent."""
org_resp = client.describe_organization()
org_arn = org_resp["Organization"]["Arn"]
# get all ou ids and arns as a dict
ou_ids_arns = get_ou_ids_arns(client)
# now look up accounts for each ou
orgs_accounts = {}
accounts_paginator = client.get_paginator("list_accounts_for_parent")
for parent_id, parent_arn in ou_ids_arns.items():
for accounts_resp in accounts_paginator.paginate(ParentId=parent_id):
for account in accounts_resp["Accounts"]:
account_id = account["Id"]
account_arn = f"arn:aws::::account/{account_id}"
account["OrganizationArn"] = org_arn
account["OUArn"] = parent_arn
orgs_accounts[account_arn] = account
return ListFromAWSResult(resources=orgs_accounts)
class EBSVolumeResourceSpec(EC2ResourceSpec):
"""Resource for EBSVolumes"""
type_name = "volume"
schema = Schema(
ScalarField("AvailabilityZone"),
ScalarField("CreateTime"),
ScalarField("Size"),
ScalarField("State"),
ScalarField("VolumeType"),
ScalarField("Encrypted"),
ListField(
"Attachments",
EmbeddedDictField(
ScalarField("AttachTime"),
ScalarField("State"),
ScalarField("DeleteOnTermination"),
ResourceLinkField("InstanceId", EC2InstanceResourceSpec),
),
optional=True,
alti_key="attachment",
),
ResourceLinkField("KmsKeyId", KMSKeyResourceSpec, optional=True, value_is_id=True),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["EBSVolumeResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'volume_1_arn': {volume_1_dict},
'volume_2_arn': {volume_2_dict},
...}
Where the dicts represent results from describe_volumes."""
volumes = {}
paginator = client.get_paginator("describe_volumes")
for resp in paginator.paginate():
for volume in resp.get("Volumes", []):
resource_arn = cls.generate_arn(account_id, region, volume["VolumeId"])
volumes[resource_arn] = volume
return ListFromAWSResult(resources=volumes)
class RDSSnapshotResourceSpec(RDSResourceSpec):
"""Resource for RDS Snapshot"""
type_name = "snapshot"
schema = Schema(
ScalarField("DBSnapshotIdentifier"),
ScalarField("SnapshotCreateTime", optional=True),
ScalarField("Engine"),
ScalarField("AllocatedStorage"),
ScalarField("Status"),
TransientResourceLinkField("VpcId", VPCResourceSpec, optional=True),
ScalarField("InstanceCreateTime"),
ScalarField("SnapshotType"),
ScalarField("PercentProgress"),
ScalarField("Region", optional=True),
ScalarField("Encrypted"),
ResourceLinkField("KmsKeyID", KMSKeyResourceSpec, optional=True),
ScalarField("Timezone", optional=True),
ScalarField("IAMDatabaseAuthenticationEnabled"),
TransientResourceLinkField("DBInstanceArn",
RDSInstanceResourceSpec,
value_is_id=True),
)
@classmethod
def generate_instance_arn(cls: Type["RDSSnapshotResourceSpec"],
account_id: str, region: str,
resource_id: str) -> str:
"""Generate an ARN for this resource, e.g. arn:aws:rds:<region>:<account>:db:<name>
"""
return ":".join((
"arn",
cls.provider_name,
cls.service_name,
region,
account_id,
cls.type_name,
resource_id,
))
@classmethod
def list_from_aws(cls: Type["RDSSnapshotResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
dbinstances = {}
paginator = client.get_paginator("describe_db_snapshots")
for resp in paginator.paginate():
for db in resp.get("DBSnapshots", []):
resource_arn = db["DBSnapshotArn"]
db["DBInstanceArn"] = cls.generate_instance_arn(
account_id, region, db["DBInstanceIdentifier"])
dbinstances[resource_arn] = db
return ListFromAWSResult(resources=dbinstances)
class TransitGatewayResourceSpec(EC2ResourceSpec):
"""Resource for TransitGateways"""
# type_name = "transit-gateway"
type_name = "transit_gateway"
schema = Schema(
ScalarField("OwnerId"),
ScalarField("State"),
ListField(
"VPCAttachments",
EmbeddedDictField(
ScalarField("CreationTime"),
ScalarField("State"),
ResourceLinkField("ResourceId", VPCResourceSpec),
),
optional=True,
alti_key="vpc_attachment",
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["TransitGatewayResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'tgw_1_arn': {tgw_1_dict},
'tgw_2_arn': {tgw_2_dict},
...}
Where the dicts represent results from describe_transit_gateways."""
tgws = {}
paginator = client.get_paginator("describe_transit_gateways")
tgw_filters = [{"Name": "owner-id", "Values": [account_id]}]
for resp in paginator.paginate(Filters=tgw_filters):
for tgw in resp["TransitGateways"]:
resource_arn = tgw["TransitGatewayArn"]
vpc_attachments: List[Dict[str, Any]] = []
vpc_attachments_paginator = client.get_paginator(
"describe_transit_gateway_attachments"
)
vpc_filters = [
{"Name": "transit-gateway-id", "Values": [tgw["TransitGatewayId"]]},
{"Name": "resource-type", "Values": ["vpc"]},
]
for vpc_attachments_resp in vpc_attachments_paginator.paginate(Filters=vpc_filters):
vpc_attachments += vpc_attachments_resp["TransitGatewayAttachments"]
tgw["VPCAttachments"] = vpc_attachments
tgws[resource_arn] = tgw
return ListFromAWSResult(resources=tgws)
class IAMGroupResourceSpec(IAMResourceSpec):
"""Resource for IAM Groups"""
type_name = "group"
schema = Schema(
ScalarField("GroupName", "name"),
ScalarField("GroupId"),
ScalarField("CreateDate"),
AnonymousListField(
"Users",
AnonymousEmbeddedDictField(
ResourceLinkField("Arn",
IAMUserResourceSpec,
value_is_id=True,
alti_key="user")),
),
)
@classmethod
def list_from_aws(cls: Type["IAMGroupResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'group_1_arn': {group_1_dict},
'group_2_arn': {group_2_dict},
...}
Where the dicts represent results from list_groups."""
groups = {}
paginator = client.get_paginator("list_groups")
for resp in paginator.paginate():
for group in resp.get("Groups", []):
resource_arn = group["Arn"]
try:
group["Users"] = cls.get_group_users(
client=client, group_name=group["GroupName"])
groups[resource_arn] = group
except ClientError as c_e:
error_code = getattr(c_e, "response",
{}).get("Error", {}).get("Code", {})
if error_code != "NoSuchEntity":
raise c_e
return ListFromAWSResult(resources=groups)
@classmethod
def get_group_users(cls: Type["IAMGroupResourceSpec"], client: BaseClient,
group_name: str) -> List[Dict[str, Any]]:
group_resp = client.get_group(GroupName=group_name)
return group_resp["Users"]
class EC2ImageResourceSpec(EC2ResourceSpec):
"""Resource for EC2Images (AMIs)"""
type_name = "image"
schema = Schema(
ScalarField("Name"),
ScalarField("Description", optional=True),
ScalarField("Public"),
ListField(
"LaunchPermissions",
EmbeddedDictField(
ResourceLinkField("UserId",
AccountResourceSpec,
optional=True,
alti_key="account")),
alti_key="launch_permission",
),
)
@classmethod
def list_from_aws(cls: Type["EC2ImageResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'image_1_arn': {image_1_dict},
'image_2_arn': {image_2_dict},
...}
Where the dicts represent results from describe_images."""
images = {}
resp = client.describe_images(Owners=["self"])
for image in resp["Images"]:
image_id = image["ImageId"]
time.sleep(
0.25) # seems necessary to avoid frequent RequestLimitExceeded
perms_resp = client.describe_image_attribute(
Attribute="launchPermission", ImageId=image_id)
launch_permissions = perms_resp["LaunchPermissions"]
image["LaunchPermissions"] = launch_permissions
resource_arn = cls.generate_arn(account_id, region, image_id)
images[resource_arn] = image
return ListFromAWSResult(resources=images)
class InternetGatewayResourceSpec(EC2ResourceSpec):
"""Resource for InternetGateways"""
# type_name = "internet-gateway"
type_name = "internet_gateway"
schema = Schema(
ScalarField("OwnerId"),
ListField(
"Attachments",
EmbeddedDictField(
ScalarField("State"),
ResourceLinkField("VpcId", VPCResourceSpec),
),
optional=True,
alti_key="attachment",
),
TagsField(),
)
@classmethod
def list_from_aws(cls: Type["InternetGatewayResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'igw_1_arn': {igw_1_dict},
'igw_2_arn': {igw_2_dict},
...}
Where the dicts represent results from describe_internet_gateways."""
igws = {}
paginator = client.get_paginator("describe_internet_gateways")
for resp in paginator.paginate():
for igw in resp["InternetGateways"]:
resource_arn = cls.generate_arn(
resource_id=igw["InternetGatewayId"],
account_id=account_id,
region=region)
igws[resource_arn] = igw
return ListFromAWSResult(resources=igws)
class IAMGroupResourceSpec(IAMResourceSpec):
"""Resource for IAM Groups"""
type_name = "group"
schema = Schema(
ScalarField("GroupName", "name"),
ScalarField("GroupId"),
ScalarField("CreateDate"),
AnonymousListField(
"Users",
AnonymousEmbeddedDictField(
ResourceLinkField("Arn",
IAMUserResourceSpec,
value_is_id=True,
alti_key="user")),
),
)
@classmethod
def list_from_aws(cls: Type["IAMGroupResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'group_1_arn': {group_1_dict},
'group_2_arn': {group_2_dict},
...}
Where the dicts represent results from list_groups."""
groups = {}
paginator = client.get_paginator("list_groups")
for resp in paginator.paginate():
for group in resp.get("Groups", []):
resource_arn = group["Arn"]
group_resp = client.get_group(GroupName=group["GroupName"])
group["Users"] = group_resp["Users"]
groups[resource_arn] = group
return ListFromAWSResult(resources=groups)
class IAMRoleResourceSpec(IAMResourceSpec):
"""Resource for IAM Roles"""
type_name = "role"
schema = Schema(
ScalarField("RoleName", "name"),
ScalarField("MaxSessionDuration"),
AnonymousListField(
"PolicyAttachments",
AnonymousEmbeddedDictField(
ResourceLinkField(
"PolicyArn",
IAMPolicyResourceSpec,
optional=True,
value_is_id=True,
alti_key="attached_policy",
)),
),
DictField(
"AssumeRolePolicyDocument",
ScalarField("Version"),
ListField(
"Statement",
EmbeddedDictField(
ScalarField("Effect"),
ScalarField("Action"),
DictField(
"Principal",
ListField("AWS",
EmbeddedScalarField(),
optional=True,
allow_scalar=True),
ListField("Federated",
EmbeddedScalarField(),
optional=True,
allow_scalar=True),
),
),
),
),
ScalarField("AssumeRolePolicyDocumentText"),
)
@classmethod
def list_from_aws(cls: Type["IAMRoleResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'role_1_arn': {role_1_dict},
'role_2_arn': {role_2_dict},
...}
Where the dicts represent results from list_roles and additional info per role from
list_targets_by_role."""
roles = {}
paginator = client.get_paginator("list_roles")
for resp in paginator.paginate():
for role in resp.get("Roles", []):
role_name = role["RoleName"]
assume_role_policy_document = copy.deepcopy(
role["AssumeRolePolicyDocument"])
assume_role_policy_document_text = policy_doc_dict_to_sorted_str(
assume_role_policy_document)
role[
"AssumeRolePolicyDocumentText"] = assume_role_policy_document_text
for statement in assume_role_policy_document.get(
"Statement", []):
for obj in statement.get("Condition", {}).values():
for obj_key in obj.keys():
if obj_key.lower() == "sts:externalid":
obj[obj_key] = "REMOVED"
policies_result = get_attached_role_policies(client, role_name)
policies = policies_result
role["PolicyAttachments"] = policies
resource_arn = role["Arn"]
roles[resource_arn] = role
return ListFromAWSResult(resources=roles)
class LoadBalancerResourceSpec(ELBV2ResourceSpec):
"""Resource for load balancer"""
type_name = "loadbalancer"
schema = Schema(
ScalarField("DNSName"),
ScalarField("CreatedTime"),
ScalarField("LoadBalancerName"),
ScalarField("Scheme"),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
AnonymousDictField("State",
ScalarField("Code",
alti_key="load_balancer_state")),
ScalarField("Type"),
ListField(
"AvailabilityZones",
EmbeddedDictField(
ScalarField("ZoneName"),
ResourceLinkField("SubnetId",
SubnetResourceSpec,
optional=True),
ListField(
"LoadBalancerAddresses",
EmbeddedDictField(
ScalarField("IpAddress", optional=True),
ScalarField("AllocationId", optional=True),
),
optional=True,
),
),
),
ListField("SecurityGroups",
EmbeddedResourceLinkField(SecurityGroupResourceSpec),
optional=True),
ScalarField("IpAddressType"),
ScalarField("AccessLogsEnabled"),
TransientResourceLinkField(
"AccessLogsS3Bucket",
S3BucketResourceSpec,
alti_key="access_logs_s3_bucket",
optional=True,
),
ScalarField("AccessLogsS3Prefix", optional=True),
)
@classmethod
def list_from_aws(cls: Type["LoadBalancerResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'lb_1_arn': {lb_1_dict},
'lb_2_arn': {lb_2_dict},
...}
Where the dicts represent results from describe_load_balancers."""
paginator = client.get_paginator("describe_load_balancers")
load_balancers = {}
for resp in paginator.paginate():
for lb in resp.get("LoadBalancers", []):
resource_arn = lb["LoadBalancerArn"]
try:
lb_attrs = cls.get_lb_attrs(client, resource_arn)
lb.update(lb_attrs)
load_balancers[resource_arn] = lb
except ClientError as c_e:
if (getattr(c_e, "response", {}).get("Error", {}).get(
"Code", {}) != "LoadBalancerNotFound"):
raise c_e
return ListFromAWSResult(resources=load_balancers)
@classmethod
def get_lb_attrs(
cls: Type["LoadBalancerResourceSpec"],
client: BaseClient,
lb_arn: str,
) -> Dict[str, str]:
"""Get lb attributes that Altimeter graphs."""
lb_attrs = {}
resp = client.describe_load_balancer_attributes(LoadBalancerArn=lb_arn)
for attr in resp["Attributes"]:
if attr["Key"] == "access_logs.s3.enabled":
lb_attrs["AccessLogsEnabled"] = attr["Value"]
elif attr["Key"] == "access_logs.s3.bucket":
if attr["Value"]:
lb_attrs["AccessLogsS3Bucket"] = attr["Value"]
elif attr["Key"] == "access_logs.s3.prefix":
if attr["Value"]:
lb_attrs["AccessLogsS3Prefix"] = attr["Value"]
return lb_attrs
class S3BucketResourceSpec(S3ResourceSpec):
"""Resource for S3 Buckets"""
type_name = "bucket"
scan_granularity = ScanGranularity.ACCOUNT
schema = Schema(
ScalarField("Name"),
ScalarField("CreationDate"),
AnonymousDictField(
"ServerSideEncryption",
ListField(
"Rules",
EmbeddedDictField(
AnonymousDictField(
"ApplyServerSideEncryptionByDefault",
ScalarField("SSEAlgorithm", "algorithm"),
),
AnonymousDictField(
"ApplyServerSideEncryptionByDefault",
ResourceLinkField("KMSMasterKeyID", KMSKeyResourceSpec, optional=True),
),
),
alti_key="server_side_default_encryption_rule",
),
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["S3BucketResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'bucket_1_arn': {bucket_1_dict},
'bucket_2_arn': {bucket_2_dict},
...}
Where the dicts represent results from list_buckets."""
logger = Logger()
buckets = {}
buckets_resp = client.list_buckets()
for bucket in buckets_resp.get("Buckets", []):
bucket_name = bucket["Name"]
try:
try:
bucket_region = get_s3_bucket_region(client, bucket_name)
except S3BucketAccessDeniedException as s3ade:
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"Unable to determine region for {bucket_name}: {s3ade}",
)
continue
try:
bucket["Tags"] = get_s3_bucket_tags(client, bucket_name)
except S3BucketAccessDeniedException as s3ade:
bucket["Tags"] = []
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"Unable to determine tags for {bucket_name}: {s3ade}",
)
try:
bucket["ServerSideEncryption"] = get_s3_bucket_encryption(client, bucket_name)
except S3BucketAccessDeniedException as s3ade:
bucket["ServerSideEncryption"] = {"Rules": []}
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"Unable to determine encryption status for {bucket_name}: {s3ade}",
)
resource_arn = cls.generate_arn(
account_id=account_id, region=bucket_region, resource_id=bucket_name
)
buckets[resource_arn] = bucket
except S3BucketDoesNotExistException as s3bdnee:
logger.warn(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=f"{bucket_name}: No longer exists: {s3bdnee}",
)
return ListFromAWSResult(resources=buckets)
class OrgsAccountResourceSpec(OrganizationsResourceSpec):
"""Resource representing an AWS Account as viewed in Orgs."""
type_name = "account"
schema = Schema(
ScalarField("Name"),
ScalarField("Status"),
ScalarField("JoinedTimestamp"),
ScalarField("Email"),
ScalarField("Id", alti_key="account_id"),
ResourceLinkField("OrganizationArn", OrgResourceSpec,
value_is_id=True),
ResourceLinkField("OUArn", OUResourceSpec, value_is_id=True),
)
allow_clobber = [UnscannedAccountResourceSpec]
@classmethod
def get_full_type_name(cls: Type["OrgsAccountResourceSpec"]) -> str:
return f"{cls.provider_name}:{cls.type_name}"
@classmethod
def list_from_aws(cls: Type["OrgsAccountResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'ou_1_arn': {org_1_dict},
'ou_2_arn': {org_2_dict},
...}
Where the dicts represent results from list_accounts_for_parent."""
org_resp = client.describe_organization()
org_arn = org_resp["Organization"]["Arn"]
# get all parents. parents include roots and ous.
parent_ids_arns = {}
paginator = client.get_paginator("list_roots")
for resp in paginator.paginate():
for root in resp["Roots"]:
root_id, root_arn = root["Id"], root["Arn"]
parent_ids_arns[root_id] = root_arn
root_path = f"/{root['Name']}"
ou_details = cls._recursively_get_ou_details_for_parent(
client=client, parent_id=root_id, parent_path=root_path)
for ou_detail in ou_details:
ou_id, ou_arn = ou_detail["Id"], ou_detail["Arn"]
parent_ids_arns[ou_id] = ou_arn
# now look up accounts for each ou
orgs_accounts = {}
accounts_paginator = client.get_paginator("list_accounts_for_parent")
for parent_id, parent_arn in parent_ids_arns.items():
for accounts_resp in accounts_paginator.paginate(
ParentId=parent_id):
for account in accounts_resp["Accounts"]:
account_id = account["Id"]
account_arn = f"arn:aws::::account/{account_id}"
account["OrganizationArn"] = org_arn
account["OUArn"] = parent_arn
orgs_accounts[account_arn] = account
return ListFromAWSResult(resources=orgs_accounts)
@classmethod
def _recursively_get_ou_details_for_parent(
cls: Type["OrgsAccountResourceSpec"], client: BaseClient,
parent_id: str, parent_path: str) -> List[Dict[str, Any]]:
ous = []
paginator = client.get_paginator(
"list_organizational_units_for_parent")
for resp in paginator.paginate(ParentId=parent_id):
for ou in resp["OrganizationalUnits"]:
ou_id = ou["Id"]
path = f"{parent_path}/{ou['Name']}"
ou["Path"] = path
ous.append(ou)
ous += cls._recursively_get_ou_details_for_parent(
client=client, parent_id=ou_id, parent_path=path)
return ous
class ClassicLoadBalancerResourceSpec(ELBV1ResourceSpec):
"""Resource for classic load balancer"""
type_name = "loadbalancer"
schema = Schema(
ScalarField("DNSName"),
ScalarField("CreatedTime"),
ScalarField("LoadBalancerName"),
ScalarField("Scheme"),
ResourceLinkField("VPCId", VPCResourceSpec, optional=True),
ListField("Subnets",
EmbeddedResourceLinkField(SubnetResourceSpec),
optional=True),
ListField("SecurityGroups",
EmbeddedResourceLinkField(SecurityGroupResourceSpec),
optional=True),
ScalarField("Type"),
ScalarField("AccessLogsEnabled"),
TransientResourceLinkField(
"AccessLogsS3Bucket",
S3BucketResourceSpec,
alti_key="access_logs_s3_bucket",
optional=True,
),
ScalarField("AccessLogsS3Prefix", optional=True),
)
@classmethod
def list_from_aws(
cls: Type["ClassicLoadBalancerResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'lb_1_arn': {lb_1_dict},
'lb_2_arn': {lb_2_dict},
...}
Where the dicts represent results from describe_load_balancers."""
paginator = client.get_paginator("describe_load_balancers")
load_balancers = {}
for resp in paginator.paginate():
for lb in resp["LoadBalancerDescriptions"]:
lb_name = lb["LoadBalancerName"]
resource_arn = cls.generate_arn(account_id=account_id,
region=region,
resource_id=lb_name)
try:
lb_attrs = cls.get_lb_attrs(client, lb_name)
lb.update(lb_attrs)
lb["Type"] = "classic"
load_balancers[resource_arn] = lb
except ClientError as c_e:
if (getattr(c_e, "response", {}).get("Error", {}).get(
"Code", {}) != "LoadBalancerNotFound"):
raise c_e
return ListFromAWSResult(resources=load_balancers)
@classmethod
def get_lb_attrs(
cls: Type["ClassicLoadBalancerResourceSpec"],
client: BaseClient,
lb_name: str,
) -> Dict[str, str]:
"""Get lb attributes that Altimeter graphs."""
lb_attrs = {}
resp = client.describe_load_balancer_attributes(
LoadBalancerName=lb_name)
access_log_attrs = resp["LoadBalancerAttributes"]["AccessLog"]
lb_attrs["AccessLogsEnabled"] = access_log_attrs["Enabled"]
if "S3BucketName" in access_log_attrs:
lb_attrs["AccessLogsS3Bucket"] = access_log_attrs["S3BucketName"]
if "S3BucketPrefix" in access_log_attrs:
lb_attrs["AccessLogsS3Prefix"] = access_log_attrs["S3BucketPrefix"]
return lb_attrs
class SecurityGroupResourceSpec(EC2ResourceSpec):
"""Resource for SecurityGroups"""
type_name = "security-group"
schema = Schema(
ScalarField("GroupName", "name"),
ListField(
"IpPermissions",
EmbeddedDictField(
ScalarField("IpProtocol"),
ScalarField("FromPort", default_value=0),
ScalarField("ToPort", default_value=65535),
ListField(
"IpRanges",
EmbeddedDictField(
ScalarField("CidrIp"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ip_range",
optional=True,
),
ListField(
"Ipv6Ranges",
EmbeddedDictField(
ScalarField("CidrIpv6"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ipv6_range",
optional=True,
),
ListField(
"PrefixListIds", EmbeddedDictField(ScalarField("PrefixListId")), optional=True
),
ListField(
"UserIdGroupPairs",
EmbeddedDictField(
ResourceLinkField("GroupId", "SecurityGroupResourceSpec"),
ScalarField("UserId", alti_key="account_id", optional=True),
ScalarField("PeeringStatus", optional=True),
ScalarField("VpcId", optional=True),
ScalarField("VpcPeeringConnectionId", optional=True),
),
alti_key="user_id_group_pairs",
),
),
alti_key="ingress_rule",
),
ListField(
"IpPermissionsEgress",
EmbeddedDictField(
ScalarField("IpProtocol"),
ScalarField("FromPort", default_value=0),
ScalarField("ToPort", default_value=65535),
ListField(
"IpRanges",
EmbeddedDictField(
ScalarField("CidrIp"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ip_range",
optional=True,
),
ListField(
"Ipv6Ranges",
EmbeddedDictField(
ScalarField("CidrIpv6"), ScalarField("FirstIp"), ScalarField("LastIp")
),
alti_key="ipv6_range",
optional=True,
),
ListField(
"PrefixListIds", EmbeddedDictField(ScalarField("PrefixListId")), optional=True
),
ListField(
"UserIdGroupPairs",
EmbeddedDictField(
ResourceLinkField("GroupId", "SecurityGroupResourceSpec"),
ScalarField("UserId", alti_key="account_id", optional=True),
ScalarField("PeeringStatus", optional=True),
ScalarField("VpcId", optional=True),
ScalarField("VpcPeeringConnectionId", optional=True),
),
alti_key="user_id_group_pairs",
),
),
alti_key="egress_rule",
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["SecurityGroupResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'security_group_1_arn': {security_group_1_dict},
'security_group_2_arn': {security_group_2_dict},
...}
Where the dicts represent results from describe_subnets."""
security_groups = {}
paginator = client.get_paginator("describe_security_groups")
for resp in paginator.paginate():
for security_group in resp.get("SecurityGroups", []):
resource_arn = cls.generate_arn(
account_id=account_id, region=region, resource_id=security_group["GroupId"]
)
for ingress_rule in security_group.get("IpPermissions", []):
for ip_range in ingress_rule.get("IpRanges", []):
cidr = ip_range["CidrIp"]
ipv4_network = ipaddress.IPv4Network(cidr, strict=False)
first_ip, last_ip = int(ipv4_network[0]), int(ipv4_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
for ip_range in ingress_rule.get("Ipv6Ranges", []):
cidr = ip_range["CidrIpv6"]
ipv6_network = ipaddress.IPv6Network(cidr, strict=False)
first_ip, last_ip = int(ipv6_network[0]), int(ipv6_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
for egress_rule in security_group.get("IpPermissionsEgress", []):
for ip_range in egress_rule.get("IpRanges", []):
cidr = ip_range["CidrIp"]
ipv4_network = ipaddress.IPv4Network(cidr, strict=False)
first_ip, last_ip = int(ipv4_network[0]), int(ipv4_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
for ip_range in egress_rule.get("Ipv6Ranges", []):
cidr = ip_range["CidrIpv6"]
ipv6_network = ipaddress.IPv6Network(cidr, strict=False)
first_ip, last_ip = int(ipv6_network[0]), int(ipv6_network[-1])
ip_range["FirstIp"] = first_ip
ip_range["LastIp"] = last_ip
security_groups[resource_arn] = security_group
return ListFromAWSResult(resources=security_groups)