def test_invalid_input_not_list(self):
input_str = '{"People": "foo"}'
field = AnonymousListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))
)
input_data = json.loads(input_str)
with self.assertRaises(ListFieldValueNotAListException):
field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
def test_invalid_input_missing_source_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = AnonymousListField(
"Stuff", EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))
)
input_data = json.loads(input_str)
with self.assertRaises(ListFieldSourceKeyNotFoundException):
field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
def test_valid_strings_input(self):
input_str = '{"Biota": {"Animals": ["cow", "pig", "human"], "Plants": ["tree", "fern"]}}'
field = DictField("Biota",
AnonymousListField("Animals", EmbeddedScalarField()))
expected_output_data = [{
"pred":
"biota",
"obj": [
{
"pred": "biota",
"obj": "cow",
"type": "simple"
},
{
"pred": "biota",
"obj": "pig",
"type": "simple"
},
{
"pred": "biota",
"obj": "human",
"type": "simple"
},
],
"type":
"multi",
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class InstanceProfileResourceSpec(IAMResourceSpec):
"""Resource for Instance Profiles"""
# type_name = "instance-profile"
type_name = "instance_profile"
schema = Schema(
ScalarField("InstanceProfileName", alti_key="name"),
AnonymousListField(
"Roles",
AnonymousEmbeddedDictField(
ResourceLinkField("Arn",
IAMRoleResourceSpec,
value_is_id=True,
alti_key="attached_role")),
),
)
@classmethod
def list_from_aws(cls: Type["InstanceProfileResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'instance_profile_1_arn': {instance_profile_1_dict},
'instance_profile_2_arn': {instance_profile_2_dict},
...}
Where the dicts represent results from list_instance_profiles."""
paginator = client.get_paginator("list_instance_profiles")
instance_profiles = {}
for resp in paginator.paginate():
for instance_profile in resp.get("InstanceProfiles", []):
resource_arn = instance_profile["Arn"]
instance_profiles[resource_arn] = instance_profile
return ListFromAWSResult(resources=instance_profiles)
class EC2RouteTableResourceSpec(EC2ResourceSpec):
"""Resource for Route Tables"""
# type_name = "route-table"
type_name = "route_table"
schema = Schema(
ScalarField("RouteTableId"),
ResourceLinkField("VpcId", VPCResourceSpec),
ScalarField("OwnerId", optional=True),
AnonymousListField("PropagatingVgws",
ScalarField("GatewayId"),
optional=True),
ListField(
"Routes",
EmbeddedDictField(
ScalarField("DestinationCidrBlock", optional=True),
ScalarField("DestinationIpv6CidrBlock", optional=True),
ScalarField("DestinationPrefixListId", optional=True),
ScalarField("EgressOnlyInternetGatewayId", optional=True),
ScalarField("GatewayId", optional=True),
ScalarField("InstanceId", optional=True),
ScalarField("InstanceOwnerId", optional=True),
ScalarField("NatGatewayId", optional=True),
ScalarField("TransitGatewayId", optional=True),
ScalarField("NetworkInterfaceId", optional=True),
ScalarField("Origin", optional=True),
ScalarField("State", optional=True),
ScalarField("VpcPeeringConnectionId", optional=True),
),
optional=True,
alti_key="route",
),
ListField(
"Associations",
EmbeddedDictField(
ScalarField("Main", optional=True),
ScalarField("RouteTableAssociationId", optional=True),
ScalarField("RouteTableId", optional=True),
ScalarField("SubnetId", optional=True),
),
optional=True,
alti_key="association",
),
)
@classmethod
def list_from_aws(cls: Type["EC2RouteTableResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
paginator = client.get_paginator("describe_route_tables")
route_tables = {}
for resp in paginator.paginate():
for attachment in resp.get("RouteTables", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=attachment["RouteTableId"])
route_tables[resource_arn] = attachment
return ListFromAWSResult(resources=route_tables)
def test_valid_dicts_input(self):
input_str = (
'{"Biota": {"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}}'
)
field = DictField(
"Biota",
AnonymousListField(
"People",
EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))),
)
expected_output_data = [{
"pred":
"biota",
"obj": [
{
"pred":
"biota",
"obj": [
{
"pred": "name",
"obj": "Bob",
"type": "simple"
},
{
"pred": "age",
"obj": 49,
"type": "simple"
},
],
"type":
"multi",
},
{
"pred":
"biota",
"obj": [
{
"pred": "name",
"obj": "Sue",
"type": "simple"
},
{
"pred": "age",
"obj": 42,
"type": "simple"
},
],
"type":
"multi",
},
],
"type":
"multi",
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class EC2InstanceResourceSpec(EC2ResourceSpec):
"""Resource for EC2Instances"""
type_name = "instance"
schema = Schema(
ScalarField("Name", optional=True),
TransientResourceLinkField("ImageId", EC2ImageResourceSpec),
ScalarField("KeyName", optional=True),
AnonymousDictField("Placement", ScalarField("AvailabilityZone"), ScalarField("Tenancy")),
ScalarField("InstanceType"),
ScalarField("LaunchTime"),
AnonymousDictField("State", ScalarField("Name", "state")),
ScalarField("Platform", optional=True),
ScalarField("PrivateIpAddress", optional=True),
ScalarField("PrivateDnsName", optional=True),
ScalarField("PublicIpAddress", optional=True),
ScalarField("PublicDnsName", optional=True),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
ResourceLinkField("SubnetId", SubnetResourceSpec, optional=True),
AnonymousListField(
"SecurityGroups",
AnonymousEmbeddedDictField(ResourceLinkField("GroupId", SecurityGroupResourceSpec)),
),
AnonymousDictField(
"IamInstanceProfile",
TransientResourceLinkField(
"Arn", InstanceProfileResourceSpec, alti_key="instance_profile", value_is_id=True
),
optional=True,
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["EC2InstanceResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'instance_1_arn': {instance_1_dict},
'instance_2_arn': {instance_2_dict},
...}
Where the dicts represent results from describe_instances."""
paginator = client.get_paginator("describe_instances")
instances = {}
for resp in paginator.paginate():
for reservation in resp.get("Reservations", []):
for instance in reservation.get("Instances", []):
resource_arn = cls.generate_arn(
account_id=account_id, region=region, resource_id=instance["InstanceId"]
)
instances[resource_arn] = instance
for tag in instance.get("Tags", []):
if tag["Key"].lower() == "name":
instance["Name"] = tag["Value"]
break
return ListFromAWSResult(resources=instances)
def test_optional(self):
input_str = '{"Biota": {"Plants": ["tree", "fern"]}}'
field = DictField(
"Biota",
AnonymousListField("Animals", EmbeddedScalarField(),
optional=True))
expected_output_data = [{"pred": "biota", "obj": [], "type": "multi"}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class EC2NetworkInterfaceResourceSpec(EC2ResourceSpec):
"""Resource for EC2NetworkInterfaces"""
# type_name = "network-interface"
type_name = "network_interface"
schema = Schema(
AnonymousDictField(
"Association",
ScalarField("PublicDnsName", optional=True),
ScalarField("PublicIp", optional=True),
optional=True,
),
ScalarField("Description"),
ScalarField("InterfaceType"),
ScalarField("MacAddress"),
ScalarField("PrivateDnsName", optional=True),
ScalarField("PrivateIpAddress", optional=True),
ScalarField("Status"),
ResourceLinkField("SubnetId", SubnetResourceSpec, optional=True),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
AnonymousListField(
"Groups",
AnonymousEmbeddedDictField(
ResourceLinkField("GroupId", SecurityGroupResourceSpec)),
),
)
@classmethod
def list_from_aws(
cls: Type["EC2NetworkInterfaceResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'network_interface_1_arn': {network_interface_1_dict},
'network_interface_2_arn': {network_interface_2_dict},
...}
Where the dicts represent results from describe_network_interfaces."""
paginator = client.get_paginator("describe_network_interfaces")
interfaces = {}
for resp in paginator.paginate():
for interface in resp.get("NetworkInterfaces", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=interface["NetworkInterfaceId"],
)
interfaces[resource_arn] = interface
return ListFromAWSResult(resources=interfaces)
def test_optional(self):
input_str = '{"Biota": {"Plants": ["tree", "fern"]}}'
field = DictField(
"Biota", AnonymousListField("Animals", EmbeddedScalarField(), optional=True)
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(MultiLink(pred="biota", obj=LinkCollection()),),
)
self.assertEqual(link_collection, expected_link_collection)
class IAMGroupResourceSpec(IAMResourceSpec):
"""Resource for IAM Groups"""
type_name = "group"
schema = Schema(
ScalarField("GroupName", "name"),
ScalarField("GroupId"),
ScalarField("CreateDate"),
AnonymousListField(
"Users",
AnonymousEmbeddedDictField(
ResourceLinkField("Arn",
IAMUserResourceSpec,
value_is_id=True,
alti_key="user")),
),
)
@classmethod
def list_from_aws(cls: Type["IAMGroupResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'group_1_arn': {group_1_dict},
'group_2_arn': {group_2_dict},
...}
Where the dicts represent results from list_groups."""
groups = {}
paginator = client.get_paginator("list_groups")
for resp in paginator.paginate():
for group in resp.get("Groups", []):
resource_arn = group["Arn"]
try:
group["Users"] = cls.get_group_users(
client=client, group_name=group["GroupName"])
groups[resource_arn] = group
except ClientError as c_e:
error_code = getattr(c_e, "response",
{}).get("Error", {}).get("Code", {})
if error_code != "NoSuchEntity":
raise c_e
return ListFromAWSResult(resources=groups)
@classmethod
def get_group_users(cls: Type["IAMGroupResourceSpec"], client: BaseClient,
group_name: str) -> List[Dict[str, Any]]:
group_resp = client.get_group(GroupName=group_name)
return group_resp["Users"]
def test_allow_scalar(self):
input_str = '{"Biota": {"Plants": "tree"}}'
field = DictField(
"Biota", AnonymousListField("Plants", EmbeddedScalarField(), allow_scalar=True)
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(simple_links=(SimpleLink(pred="biota", obj="tree"),),),
),
)
)
self.assertEqual(expected_link_collection, link_collection)
def test_valid_dicts_input(self):
input_str = (
'{"Biota": {"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}}'
)
field = DictField(
"Biota",
AnonymousListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))
),
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Bob"),
SimpleLink(pred="age", obj=49),
),
),
),
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Sue"),
SimpleLink(pred="age", obj=42),
),
),
),
),
),
),
)
)
self.assertEqual(link_collection, expected_link_collection)
def test_valid_strings_input(self):
input_str = '{"Biota": {"Animals": ["cow", "pig", "human"], "Plants": ["tree", "fern"]}}'
field = DictField("Biota", AnonymousListField("Animals", EmbeddedScalarField()))
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="biota", obj="cow"),
SimpleLink(pred="biota", obj="pig"),
SimpleLink(pred="biota", obj="human"),
)
),
),
)
)
self.assertEqual(link_collection, expected_link_collection)
class VpcEndpointServiceResourceSpec(EC2ResourceSpec):
"""Resource for VPC Endpoint Services"""
# type_name = "vpc-endpoint-service"
type_name = "vpc_endpoint_service"
schema = Schema(
AnonymousListField("ServiceType", ScalarField("ServiceType")),
ScalarField("ServiceName"),
ScalarField("ServiceState"),
ScalarField("AcceptanceRequired"),
ListField("AvailabilityZones", EmbeddedScalarField()),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["VpcEndpointServiceResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'vpc_endpoint_svc_1_arn': {vpc_endpoint_svc_1_dict},
'vpc_endpoint_svc_2_arn': {vpc_endpoint_svc_2_dict},
...}
Where the dicts represent results from describe_vpc_endpoint_service_configurations."""
services = {}
paginator = client.get_paginator(
"describe_vpc_endpoint_service_configurations")
for resp in paginator.paginate():
for service in resp.get("ServiceConfigurations", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=service["ServiceId"])
services[resource_arn] = service
return ListFromAWSResult(resources=services)
def test_allow_scalar(self):
input_str = '{"Biota": {"Plants": "tree"}}'
field = DictField(
"Biota",
AnonymousListField("Plants",
EmbeddedScalarField(),
allow_scalar=True))
expected_output_data = [{
"pred":
"biota",
"obj": [{
"pred": "biota",
"obj": "tree",
"type": "simple"
}],
"type":
"multi",
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class IAMGroupResourceSpec(IAMResourceSpec):
"""Resource for IAM Groups"""
type_name = "group"
schema = Schema(
ScalarField("GroupName", "name"),
ScalarField("GroupId"),
ScalarField("CreateDate"),
AnonymousListField(
"Users",
AnonymousEmbeddedDictField(
ResourceLinkField("Arn",
IAMUserResourceSpec,
value_is_id=True,
alti_key="user")),
),
)
@classmethod
def list_from_aws(cls: Type["IAMGroupResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'group_1_arn': {group_1_dict},
'group_2_arn': {group_2_dict},
...}
Where the dicts represent results from list_groups."""
groups = {}
paginator = client.get_paginator("list_groups")
for resp in paginator.paginate():
for group in resp.get("Groups", []):
resource_arn = group["Arn"]
group_resp = client.get_group(GroupName=group["GroupName"])
group["Users"] = group_resp["Users"]
groups[resource_arn] = group
return ListFromAWSResult(resources=groups)
class RDSInstanceResourceSpec(RDSResourceSpec):
"""Resource for RDS"""
type_name = "db"
schema = Schema(
TagsField(),
ScalarField("DBInstanceIdentifier"),
ScalarField("DBInstanceClass"),
ScalarField("Engine"),
ScalarField("DBInstanceStatus"),
ScalarField("DBName", optional=True),
AnonymousDictField(
"Endpoint",
ScalarField("Address", alti_key="endpoint_address", optional=True),
ScalarField("Port", alti_key="endpoint_port"),
ScalarField("HostedZoneId",
alti_key="endpoint_hosted_zone",
optional=True),
optional=True,
),
AnonymousDictField(
"ListenerEndpoint",
ScalarField("Address", alti_key="listener_address"),
ScalarField("Port", alti_key="listener_port"),
ScalarField("HostedZoneId",
alti_key="listener_hosted_zone",
optional=True),
optional=True,
),
ScalarField("InstanceCreateTime", optional=True),
ScalarField("BackupRetentionPeriod"),
AnonymousListField(
"VpcSecurityGroups",
AnonymousEmbeddedDictField(
TransientResourceLinkField("VpcSecurityGroupId",
SecurityGroupResourceSpec,
optional=True)),
),
ScalarField("AvailabilityZone", optional=True),
AnonymousDictField("DBSubnetGroup",
TransientResourceLinkField("VpcId",
VPCResourceSpec),
optional=True),
ScalarField("MultiAZ"),
ScalarField("PubliclyAccessible"),
ListField("StatusInfos",
EmbeddedDictField(ScalarField("Status")),
optional=True),
ScalarField("StorageType"),
ScalarField("StorageEncrypted"),
TransientResourceLinkField("KmsKeyId",
KMSKeyResourceSpec,
optional=True,
value_is_id=True),
ScalarField("DbiResourceId"),
ScalarField("Timezone", optional=True),
ScalarField("IAMDatabaseAuthenticationEnabled"),
ScalarField("PerformanceInsightsEnabled", optional=True),
ScalarField("PerformanceInsightsRetentionPeriod", optional=True),
ScalarField("DeletionProtection"),
ListField(
"Backup",
EmbeddedDictField(
AnonymousDictField(
"RestoreWindow",
ScalarField("EarliestTime",
alti_key="earliest_restore_time",
optional=True),
optional=True,
),
AnonymousDictField(
"RestoreWindow",
ScalarField("LatestTime",
alti_key="latest_restore_time",
optional=True),
optional=True,
),
ScalarField("AllocatedStorage"),
ScalarField("Status"),
ScalarField("AvailabilityZone", optional=True),
ScalarField("Engine"),
ScalarField("EngineVersion"),
ScalarField("Encrypted"),
ScalarField("StorageType"),
TransientResourceLinkField("KmsKeyId",
KMSKeyResourceSpec,
optional=True,
value_is_id=True),
),
optional=True,
),
)
@classmethod
def list_from_aws(cls: Type["RDSInstanceResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
logger = Logger()
dbinstances = {}
paginator = client.get_paginator("describe_db_instances")
for resp in paginator.paginate():
for db in resp.get("DBInstances", []):
resource_arn = db["DBInstanceArn"]
db["Tags"] = client.list_tags_for_resource(
ResourceName=resource_arn).get("TagList", [])
db["Backup"] = []
dbinstances[resource_arn] = db
backup_paginator = client.get_paginator(
"describe_db_instance_automated_backups")
for resp in backup_paginator.paginate():
for backup in resp.get("DBInstanceAutomatedBackups", []):
if backup["DBInstanceArn"] in dbinstances:
dbinstances[backup["DBInstanceArn"]]["Backup"].append(
backup)
else:
logger.info(
event=AWSLogEvents.ScanAWSResourcesNonFatalError,
msg=
(f'Unable to find matching DB Instance {backup["DBInstanceArn"]} '
"(Possible Deletion)"),
)
return ListFromAWSResult(resources=dbinstances)
class IAMRoleResourceSpec(IAMResourceSpec):
"""Resource for IAM Roles"""
type_name = "role"
schema = Schema(
ScalarField("RoleName", "name"),
ScalarField("MaxSessionDuration"),
AnonymousListField(
"PolicyAttachments",
AnonymousEmbeddedDictField(
ResourceLinkField(
"PolicyArn",
IAMPolicyResourceSpec,
optional=True,
value_is_id=True,
alti_key="attached_policy",
)),
),
DictField(
"AssumeRolePolicyDocument",
ScalarField("Version"),
ListField(
"Statement",
EmbeddedDictField(
ScalarField("Effect"),
ScalarField("Action"),
DictField(
"Principal",
ListField("AWS",
EmbeddedScalarField(),
optional=True,
allow_scalar=True),
ListField("Federated",
EmbeddedScalarField(),
optional=True,
allow_scalar=True),
),
),
),
),
ScalarField("AssumeRolePolicyDocumentText"),
)
@classmethod
def list_from_aws(cls: Type["IAMRoleResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'role_1_arn': {role_1_dict},
'role_2_arn': {role_2_dict},
...}
Where the dicts represent results from list_roles and additional info per role from
list_targets_by_role."""
roles = {}
paginator = client.get_paginator("list_roles")
for resp in paginator.paginate():
for role in resp.get("Roles", []):
role_name = role["RoleName"]
assume_role_policy_document = copy.deepcopy(
role["AssumeRolePolicyDocument"])
assume_role_policy_document_text = policy_doc_dict_to_sorted_str(
assume_role_policy_document)
role[
"AssumeRolePolicyDocumentText"] = assume_role_policy_document_text
for statement in assume_role_policy_document.get(
"Statement", []):
for obj in statement.get("Condition", {}).values():
for obj_key in obj.keys():
if obj_key.lower() == "sts:externalid":
obj[obj_key] = "REMOVED"
policies_result = get_attached_role_policies(client, role_name)
policies = policies_result
role["PolicyAttachments"] = policies
resource_arn = role["Arn"]
roles[resource_arn] = role
return ListFromAWSResult(resources=roles)