def test_key_absent_without_optional(self):
input_str = "{}"
field = ScalarField("FieldName")
input_data = json.loads(input_str)
with self.assertRaises(ScalarFieldSourceKeyNotFoundException):
field.parse(data=input_data, context={})
class LambdaFunctionResourceSpec(LambdaResourceSpec):
"""Resource for Lambda Functions"""
type_name = "function"
schema = Schema(
ScalarField("FunctionName"),
ScalarField("Runtime", optional=True),
AnonymousDictField(
"VpcConfig",
TransientResourceLinkField("VpcId", VPCResourceSpec,
optional=True),
optional=True,
),
)
@classmethod
def list_from_aws(cls: Type["LambdaFunctionResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'function_1_arn': {function_1_dict},
'function_2_arn': {function_2_dict},
...}
Where the dicts represent results from list_functions."""
functions = {}
paginator = client.get_paginator("list_functions")
for resp in paginator.paginate():
for function in resp.get("Functions", []):
resource_arn = function["FunctionArn"]
functions[resource_arn] = function
return ListFromAWSResult(resources=functions)
class SubnetResourceSpec(EC2ResourceSpec):
"""Resource for Subnets"""
type_name = "subnet"
schema = Schema(
ScalarField("CidrBlock"),
ScalarField("FirstIp"),
ScalarField("LastIp"),
ScalarField("State"),
TagsField(),
ResourceLinkField("VpcId", VPCResourceSpec),
)
@classmethod
def list_from_aws(
cls: Type["SubnetResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
subnets = {}
resp = client.describe_subnets()
for subnet in resp.get("Subnets", []):
resource_arn = cls.generate_arn(
account_id=account_id, region=region, resource_id=subnet["SubnetId"]
)
cidr = subnet["CidrBlock"]
ipv4_network = ipaddress.IPv4Network(cidr, strict=False)
first_ip, last_ip = int(ipv4_network[0]), int(ipv4_network[-1])
subnet["FirstIp"] = first_ip
subnet["LastIp"] = last_ip
subnets[resource_arn] = subnet
return ListFromAWSResult(resources=subnets)
class EC2ImageResourceSpec(EC2ResourceSpec):
"""Resource for EC2Images (AMIs)"""
type_name = "image"
schema = Schema(
ScalarField("Name", optional=True),
ScalarField("Description", optional=True),
ScalarField("Public"),
TagsField(),
)
@classmethod
def list_from_aws(cls: Type["EC2ImageResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'image_1_arn': {image_1_dict},
'image_2_arn': {image_2_dict},
...}
Where the dicts represent results from describe_images."""
images = {}
resp = client.describe_images(Owners=["self"])
for image in resp["Images"]:
image_id = image["ImageId"]
resource_arn = cls.generate_arn(account_id=account_id,
region=region,
resource_id=image_id)
images[resource_arn] = image
return ListFromAWSResult(resources=images)
def test_value_not_scalar(self):
input_str = '{"FieldName": [1, 2, 3]}'
field = ScalarField("FieldName")
input_data = json.loads(input_str)
with self.assertRaises(ScalarFieldValueNotAScalarException):
field.parse(data=input_data, context={})
class VPCResourceSpec(EC2ResourceSpec):
"""Resource for VPCs"""
type_name = "vpc"
schema = Schema(ScalarField("IsDefault"), ScalarField("CidrBlock"),
ScalarField("State"), TagsField())
@classmethod
def list_from_aws(cls: Type["VPCResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'vpc_1_arn': {vpc_1_dict},
'vpc_2_arn': {vpc_2_dict},
...}
Where the dicts represent results from describe_vpcs."""
vpcs = {}
paginator = client.get_paginator("describe_vpcs")
for resp in paginator.paginate():
for vpc in resp.get("Vpcs", []):
resource_arn = cls.generate_arn(account_id=account_id,
region=region,
resource_id=vpc["VpcId"])
vpcs[resource_arn] = vpc
return ListFromAWSResult(resources=vpcs)
class IAMSAMLProviderResourceSpec(IAMResourceSpec):
"""Resource for IAM SAML Providers"""
type_name = "saml-provider"
schema = Schema(
ScalarField("Name"),
ScalarField("ValidUntil"),
ScalarField("CreateDate"),
ScalarField("MetadataDocumentChecksum"),
)
@classmethod
def list_from_aws(cls: Type["IAMSAMLProviderResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'saml_provider_1_arn': {saml_provider_1_dict},
'saml_provider_2_arn': {saml_provider_2_dict},
...}
Where the dicts represent results from list_saml_providers and additional info per
saml_provider list_saml_providers. An additional 'Name' key is added."""
saml_providers = {}
resp = client.list_saml_providers()
for saml_provider in resp.get("SAMLProviderList", []):
resource_arn = saml_provider["Arn"]
saml_provider["Name"] = "/".join(resource_arn.split("/")[1:])
saml_provider_resp = client.get_saml_provider(
SAMLProviderArn=resource_arn)
saml_metadata_document = saml_provider_resp["SAMLMetadataDocument"]
hash_object = hashlib.sha256(saml_metadata_document.encode())
saml_provider["MetadataDocumentChecksum"] = hash_object.hexdigest()
saml_providers[resource_arn] = saml_provider
return ListFromAWSResult(resources=saml_providers)
class VpcEndpointResourceSpec(EC2ResourceSpec):
"""Resource for VPC Endpoints"""
type_name = "vpc-endpoint"
schema = Schema(
ScalarField("VpcEndpointType"),
ScalarField("ServiceName"),
ScalarField("State"),
ResourceLinkField("VpcId", VPCResourceSpec),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["VpcEndpointResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'vpc_endpoint_1_arn': {vpc_endpoint_1_dict},
'vpc_endpoint_1_arn': {vpc_endpoint_2_dict},
...}
Where the dicts represent results from describe_vpc_endpoints."""
endpoints = {}
paginator = client.get_paginator("describe_vpc_endpoints")
for resp in paginator.paginate():
for endpoint in resp.get("VpcEndpoints", []):
resource_arn = cls.generate_arn(account_id, region, endpoint["VpcEndpointId"])
endpoints[resource_arn] = endpoint
return ListFromAWSResult(resources=endpoints)
class IAMAWSManagedPolicyResourceSpec(IAMResourceSpec):
"""Resource for AWS-managed IAM Policies"""
type_name = "policy"
schema = Schema(ScalarField("PolicyName", "name"), ScalarField("PolicyId"))
@classmethod
def list_from_aws(
cls: Type["IAMAWSManagedPolicyResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'role_1_arn': {role_1_dict},
'role_2_arn': {role_2_dict},
...}
Where the dicts represent results from list_policies and additional info per role from
list_targets_by_role."""
policies = {}
paginator = client.get_paginator("list_policies")
for resp in paginator.paginate(Scope="AWS", OnlyAttached=True):
for policy in resp.get("Policies", []):
resource_arn = policy["Arn"]
policies[resource_arn] = policy
return ListFromAWSResult(resources=policies)
class RegionResourceSpec(EC2ResourceSpec):
"""Resource representing an AWS Region"""
type_name = "region"
scan_granularity = ScanGranularity.ACCOUNT
schema = Schema(ScalarField("RegionName", "name"),
ScalarField("OptInStatus"))
@classmethod
def get_full_type_name(cls: Type["RegionResourceSpec"]) -> str:
return f"{cls.provider_name}:{cls.type_name}"
@classmethod
def list_from_aws(cls: Type["RegionResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'region_1_arn': {region_1_dict},
'region_2_arn': {region_2_dict},
...}
Where the dicts represent results from describe_regions."""
regions = {}
resp = client.describe_regions(AllRegions=True)
for region_resp in resp["Regions"]:
region_name = region_resp["RegionName"]
region_arn = f"arn:aws:::{account_id}:region/{region_name}"
regions[region_arn] = region_resp
return ListFromAWSResult(resources=regions)
class OrgResourceSpec(OrganizationsResourceSpec):
"""Resource representing an AWS Org."""
type_name = "organization"
schema = Schema(ScalarField("MasterAccountId"),
ScalarField("MasterAccountEmail"))
@classmethod
def get_full_type_name(cls: Type["OrgResourceSpec"]) -> str:
return f"{cls.provider_name}:{cls.type_name}"
@classmethod
def list_from_aws(cls: Type["OrgResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'org_1_arn': {org_1_dict},
'org_2_arn': {org_2_dict},
...}
Where the dicts represent results from describe_organization."""
resp = client.describe_organization()
org = resp["Organization"]
orgs = {org["Arn"]: org}
return ListFromAWSResult(resources=orgs)
class EBSSnapshotResourceSpec(EC2ResourceSpec):
"""Resource for EBSSnapshots"""
type_name = "snapshot"
schema = Schema(
ScalarField("VolumeSize"),
ScalarField("Encrypted"),
TransientResourceLinkField("KmsKeyId",
KMSKeyResourceSpec,
optional=True,
value_is_id=True),
TransientResourceLinkField("VolumeId", EBSVolumeResourceSpec),
TagsField(),
)
@classmethod
def list_from_aws(cls: Type["EBSSnapshotResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'snapshot_1_arn': {snapshot_1_dict},
'snapshot_2_arn': {snapshot_2_dict},
...}
Where the dicts represent results from describe_snapshots."""
snapshots = {}
paginator = client.get_paginator("describe_snapshots")
for resp in paginator.paginate(OwnerIds=["self"]):
for snapshot in resp.get("Snapshots", []):
resource_arn = cls.generate_arn(
account_id=account_id,
region=region,
resource_id=snapshot["SnapshotId"])
snapshots[resource_arn] = snapshot
return ListFromAWSResult(resources=snapshots)
def test_valid_dicts_input_with_alti_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = ListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person"
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Bob"),
SimpleLink(pred="age", obj=49),
),
),
),
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Sue"),
SimpleLink(pred="age", obj=42),
),
),
),
),
)
self.assertEqual(link_collection, expected_link_collection)
class CloudTrailTrailResourceSpec(CloudTrailResourceSpec):
"""Resource representing a CloudTrail Trail"""
type_name = "trail"
schema = Schema(
ScalarField("Name"),
ScalarField("S3BucketName"),
ScalarField("IncludeGlobalServiceEvents"),
ScalarField("IsMultiRegionTrail"),
)
@classmethod
def list_from_aws(
cls: Type["CloudTrailTrailResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'trail_1_arn': {trail_1_dict},
'trail_2_arn': {trail_2_dict},
...}
Where the dicts represent results from describe_trails."""
trails = {}
resp = client.describe_trails(includeShadowTrails=False)
for trail in resp.get("trailList", []):
resource_arn = trail["TrailARN"]
trails[resource_arn] = trail
return ListFromAWSResult(resources=trails)
def test_valid_dicts_input(self):
input_str = (
'{"Biota": {"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}}'
)
field = DictField(
"Biota",
AnonymousListField(
"People",
EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))),
)
expected_output_data = [{
"pred":
"biota",
"obj": [
{
"pred":
"biota",
"obj": [
{
"pred": "name",
"obj": "Bob",
"type": "simple"
},
{
"pred": "age",
"obj": 49,
"type": "simple"
},
],
"type":
"multi",
},
{
"pred":
"biota",
"obj": [
{
"pred": "name",
"obj": "Sue",
"type": "simple"
},
{
"pred": "age",
"obj": 42,
"type": "simple"
},
],
"type":
"multi",
},
],
"type":
"multi",
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
def test_key_absent_with_optional(self):
input_str = "{}"
field = ScalarField("FieldName", optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
self.assertEqual(link_collection, LinkCollection())
def test_invalid_input_missing_source_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = ListField(
"Stuff", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person"
)
input_data = json.loads(input_str)
with self.assertRaises(ListFieldSourceKeyNotFoundException):
field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
def test_invalid_input_not_list(self):
input_str = '{"People": "foo"}'
field = AnonymousListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))
)
input_data = json.loads(input_str)
with self.assertRaises(ListFieldValueNotAListException):
field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
class IAMPolicyResourceSpec(IAMResourceSpec):
"""Resource for user-managed IAM Policies"""
type_name = "policy"
parallel_scan = True
schema = Schema(
ScalarField("PolicyName", "name"),
ScalarField("PolicyId"),
ScalarField("DefaultVersionId"),
ScalarField("DefaultVersionPolicyDocumentText"),
)
@classmethod
def list_from_aws(cls: Type["IAMPolicyResourceSpec"], client: BaseClient,
account_id: str, region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'role_1_arn': {role_1_dict},
'role_2_arn': {role_2_dict},
...}
Where the dicts represent results from list_policies and additional info per role from
list_targets_by_role."""
policies = {}
paginator = client.get_paginator("list_policies")
for resp in paginator.paginate(Scope="Local"):
for policy in resp.get("Policies", []):
resource_arn = policy["Arn"]
default_policy_version = policy["DefaultVersionId"]
try:
default_policy_version_document_text = cls.get_policy_version_document_text(
client=client,
policy_arn=resource_arn,
policy_version=default_policy_version,
)
policy[
"DefaultVersionPolicyDocumentText"] = policy_doc_dict_to_sorted_str(
default_policy_version_document_text)
policies[resource_arn] = policy
except ClientError as c_e:
error_code = getattr(c_e, "response",
{}).get("Error", {}).get("Code", {})
if error_code != "NoSuchEntity":
raise c_e
return ListFromAWSResult(resources=policies)
@classmethod
def get_policy_version_document_text(
cls: Type["IAMPolicyResourceSpec"],
client: BaseClient,
policy_arn: str,
policy_version: str,
) -> Dict[str, Any]:
policy_version_resp = client.get_policy_version(
PolicyArn=policy_arn, VersionId=policy_version)
return policy_version_resp["PolicyVersion"]["Document"]
def test_key_absent_with_optional(self):
input_str = "{}"
field = ScalarField("FieldName", optional=True)
expected_output_data = []
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
def test_parse(self):
schema = Schema(ScalarField("Key1"), ScalarField("Key2"))
data = {"Key1": "Value1", "Key2": "Value2"}
link_collection = schema.parse(data, {})
expected_link_collection = LinkCollection(simple_links=(
SimpleLink(pred="key1", obj="Value1"),
SimpleLink(pred="key2", obj="Value2"),
))
self.assertEqual(link_collection, expected_link_collection)
def test_valid_input_with_alti_key(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", alti_key="alti_field_name")
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="alti_field_name", obj="Value"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_key_present_with_optional(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="field_name", obj="Value"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_key_absent_with_default(self):
input_str = "{}"
field = ScalarField("FieldName", default_value="DefaultValue")
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="field_name", obj="DefaultValue"), ), )
self.assertEqual(link_collection, expected_link_collection)
class LoadBalancerResourceSpec(ElasticLoadBalancingResourceSpec):
"""Resource for load balancer"""
type_name = "loadbalancer"
schema = Schema(
ScalarField("DNSName"),
ScalarField("CreatedTime"),
ScalarField("LoadBalancerName"),
ScalarField("Scheme"),
ResourceLinkField("VpcId", VPCResourceSpec, optional=True),
AnonymousDictField("State",
ScalarField("Code",
alti_key="load_balancer_state")),
ScalarField("Type"),
ListField(
"AvailabilityZones",
EmbeddedDictField(
ScalarField("ZoneName"),
ResourceLinkField("SubnetId",
SubnetResourceSpec,
optional=True),
ListField(
"LoadBalancerAddresses",
EmbeddedDictField(
ScalarField("IpAddress", optional=True),
ScalarField("AllocationId", optional=True),
),
optional=True,
),
),
),
ListField("SecurityGroups",
EmbeddedResourceLinkField(SecurityGroupResourceSpec),
optional=True),
ScalarField("IpAddressType"),
)
@classmethod
def list_from_aws(cls: Type["LoadBalancerResourceSpec"],
client: BaseClient, account_id: str,
region: str) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'lb_1_arn': {lb_1_dict},
'lb_2_arn': {lb_2_dict},
...}
Where the dicts represent results from describe_load_balancers."""
paginator = client.get_paginator("describe_load_balancers")
load_balancers = {}
for resp in paginator.paginate():
for lb in resp.get("LoadBalancers", []):
resource_arn = lb["LoadBalancerArn"]
load_balancers[resource_arn] = lb
return ListFromAWSResult(resources=load_balancers)
def test_key_present_with_optional(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", optional=True)
expected_output_data = [{
"pred": "field_name",
"obj": "Value",
"type": "simple"
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
def test_key_absent_with_default(self):
input_str = "{}"
field = ScalarField("FieldName", default_value="DefaultValue")
expected_output_data = [{
"pred": "field_name",
"obj": "DefaultValue",
"type": "simple"
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
def test_valid_input_with_alti_key(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", alti_key="alti_field_name")
expected_output_data = [{
"pred": "alti_field_name",
"obj": "Value",
"type": "simple"
}]
input_data = json.loads(input_str)
links = field.parse(data=input_data, context={})
output_data = [link.to_dict() for link in links]
self.assertCountEqual(output_data, expected_output_data)
class TransitGatewayResourceSpec(EC2ResourceSpec):
"""Resource for TransitGateways"""
# type_name = "transit-gateway"
type_name = "transit_gateway"
schema = Schema(
ScalarField("OwnerId"),
ScalarField("State"),
ListField(
"VPCAttachments",
EmbeddedDictField(
ScalarField("CreationTime"),
ScalarField("State"),
ResourceLinkField("ResourceId", VPCResourceSpec),
),
optional=True,
alti_key="vpc_attachment",
),
TagsField(),
)
@classmethod
def list_from_aws(
cls: Type["TransitGatewayResourceSpec"], client: BaseClient, account_id: str, region: str
) -> ListFromAWSResult:
"""Return a dict of dicts of the format:
{'tgw_1_arn': {tgw_1_dict},
'tgw_2_arn': {tgw_2_dict},
...}
Where the dicts represent results from describe_transit_gateways."""
tgws = {}
paginator = client.get_paginator("describe_transit_gateways")
tgw_filters = [{"Name": "owner-id", "Values": [account_id]}]
for resp in paginator.paginate(Filters=tgw_filters):
for tgw in resp["TransitGateways"]:
resource_arn = tgw["TransitGatewayArn"]
vpc_attachments: List[Dict[str, Any]] = []
vpc_attachments_paginator = client.get_paginator(
"describe_transit_gateway_attachments"
)
vpc_filters = [
{"Name": "transit-gateway-id", "Values": [tgw["TransitGatewayId"]]},
{"Name": "resource-type", "Values": ["vpc"]},
]
for vpc_attachments_resp in vpc_attachments_paginator.paginate(Filters=vpc_filters):
vpc_attachments += vpc_attachments_resp["TransitGatewayAttachments"]
tgw["VPCAttachments"] = vpc_attachments
tgws[resource_arn] = tgw
return ListFromAWSResult(resources=tgws)
class TransitGatewayVpcAttachmentResourceSpec(EC2ResourceSpec):
"""Resource for Transit Gateway VPC Attachments"""
type_name = "transit-gateway-vpc-attachment"
schema = Schema(
ScalarField("TransitGatewayAttachmentId"),
ScalarField("TransitGatewayId"),
ScalarField("VpcId"),
ScalarField("VpcOwnerId"),
ScalarField("State"),
ScalarField("CreationTime"),
ListField("SubnetIds", EmbeddedScalarField(), alti_key="subnet_id"),
AnonymousDictField("Options", ScalarField("DnsSupport"),
ScalarField("Ipv6Support")),
)
@classmethod
def list_from_aws(
cls: Type["TransitGatewayVpcAttachmentResourceSpec"],
client: BaseClient,
account_id: str,
region: str,
) -> ListFromAWSResult:
paginator = client.get_paginator(
"describe_transit_gateway_vpc_attachments")
attachments = {}
for resp in paginator.paginate():
for attachment in resp.get("TransitGatewayVpcAttachments", []):
resource_arn = cls.generate_arn(
account_id, region,
attachment["TransitGatewayAttachmentId"])
attachments[resource_arn] = attachment
return ListFromAWSResult(resources=attachments)
