def test_key_absent_without_optional(self): input_str = "{}" field = ScalarField("FieldName") input_data = json.loads(input_str) with self.assertRaises(ScalarFieldSourceKeyNotFoundException): field.parse(data=input_data, context={})
class LambdaFunctionResourceSpec(LambdaResourceSpec): """Resource for Lambda Functions""" type_name = "function" schema = Schema( ScalarField("FunctionName"), ScalarField("Runtime", optional=True), AnonymousDictField( "VpcConfig", TransientResourceLinkField("VpcId", VPCResourceSpec, optional=True), optional=True, ), ) @classmethod def list_from_aws(cls: Type["LambdaFunctionResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'function_1_arn': {function_1_dict}, 'function_2_arn': {function_2_dict}, ...} Where the dicts represent results from list_functions.""" functions = {} paginator = client.get_paginator("list_functions") for resp in paginator.paginate(): for function in resp.get("Functions", []): resource_arn = function["FunctionArn"] functions[resource_arn] = function return ListFromAWSResult(resources=functions)
class SubnetResourceSpec(EC2ResourceSpec): """Resource for Subnets""" type_name = "subnet" schema = Schema( ScalarField("CidrBlock"), ScalarField("FirstIp"), ScalarField("LastIp"), ScalarField("State"), TagsField(), ResourceLinkField("VpcId", VPCResourceSpec), ) @classmethod def list_from_aws( cls: Type["SubnetResourceSpec"], client: BaseClient, account_id: str, region: str ) -> ListFromAWSResult: subnets = {} resp = client.describe_subnets() for subnet in resp.get("Subnets", []): resource_arn = cls.generate_arn( account_id=account_id, region=region, resource_id=subnet["SubnetId"] ) cidr = subnet["CidrBlock"] ipv4_network = ipaddress.IPv4Network(cidr, strict=False) first_ip, last_ip = int(ipv4_network[0]), int(ipv4_network[-1]) subnet["FirstIp"] = first_ip subnet["LastIp"] = last_ip subnets[resource_arn] = subnet return ListFromAWSResult(resources=subnets)
class EC2ImageResourceSpec(EC2ResourceSpec): """Resource for EC2Images (AMIs)""" type_name = "image" schema = Schema( ScalarField("Name", optional=True), ScalarField("Description", optional=True), ScalarField("Public"), TagsField(), ) @classmethod def list_from_aws(cls: Type["EC2ImageResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'image_1_arn': {image_1_dict}, 'image_2_arn': {image_2_dict}, ...} Where the dicts represent results from describe_images.""" images = {} resp = client.describe_images(Owners=["self"]) for image in resp["Images"]: image_id = image["ImageId"] resource_arn = cls.generate_arn(account_id=account_id, region=region, resource_id=image_id) images[resource_arn] = image return ListFromAWSResult(resources=images)
def test_value_not_scalar(self): input_str = '{"FieldName": [1, 2, 3]}' field = ScalarField("FieldName") input_data = json.loads(input_str) with self.assertRaises(ScalarFieldValueNotAScalarException): field.parse(data=input_data, context={})
class VPCResourceSpec(EC2ResourceSpec): """Resource for VPCs""" type_name = "vpc" schema = Schema(ScalarField("IsDefault"), ScalarField("CidrBlock"), ScalarField("State"), TagsField()) @classmethod def list_from_aws(cls: Type["VPCResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'vpc_1_arn': {vpc_1_dict}, 'vpc_2_arn': {vpc_2_dict}, ...} Where the dicts represent results from describe_vpcs.""" vpcs = {} paginator = client.get_paginator("describe_vpcs") for resp in paginator.paginate(): for vpc in resp.get("Vpcs", []): resource_arn = cls.generate_arn(account_id=account_id, region=region, resource_id=vpc["VpcId"]) vpcs[resource_arn] = vpc return ListFromAWSResult(resources=vpcs)
class IAMSAMLProviderResourceSpec(IAMResourceSpec): """Resource for IAM SAML Providers""" type_name = "saml-provider" schema = Schema( ScalarField("Name"), ScalarField("ValidUntil"), ScalarField("CreateDate"), ScalarField("MetadataDocumentChecksum"), ) @classmethod def list_from_aws(cls: Type["IAMSAMLProviderResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'saml_provider_1_arn': {saml_provider_1_dict}, 'saml_provider_2_arn': {saml_provider_2_dict}, ...} Where the dicts represent results from list_saml_providers and additional info per saml_provider list_saml_providers. An additional 'Name' key is added.""" saml_providers = {} resp = client.list_saml_providers() for saml_provider in resp.get("SAMLProviderList", []): resource_arn = saml_provider["Arn"] saml_provider["Name"] = "/".join(resource_arn.split("/")[1:]) saml_provider_resp = client.get_saml_provider( SAMLProviderArn=resource_arn) saml_metadata_document = saml_provider_resp["SAMLMetadataDocument"] hash_object = hashlib.sha256(saml_metadata_document.encode()) saml_provider["MetadataDocumentChecksum"] = hash_object.hexdigest() saml_providers[resource_arn] = saml_provider return ListFromAWSResult(resources=saml_providers)
class VpcEndpointResourceSpec(EC2ResourceSpec): """Resource for VPC Endpoints""" type_name = "vpc-endpoint" schema = Schema( ScalarField("VpcEndpointType"), ScalarField("ServiceName"), ScalarField("State"), ResourceLinkField("VpcId", VPCResourceSpec), TagsField(), ) @classmethod def list_from_aws( cls: Type["VpcEndpointResourceSpec"], client: BaseClient, account_id: str, region: str ) -> ListFromAWSResult: """Return a dict of dicts of the format: {'vpc_endpoint_1_arn': {vpc_endpoint_1_dict}, 'vpc_endpoint_1_arn': {vpc_endpoint_2_dict}, ...} Where the dicts represent results from describe_vpc_endpoints.""" endpoints = {} paginator = client.get_paginator("describe_vpc_endpoints") for resp in paginator.paginate(): for endpoint in resp.get("VpcEndpoints", []): resource_arn = cls.generate_arn(account_id, region, endpoint["VpcEndpointId"]) endpoints[resource_arn] = endpoint return ListFromAWSResult(resources=endpoints)
class IAMAWSManagedPolicyResourceSpec(IAMResourceSpec): """Resource for AWS-managed IAM Policies""" type_name = "policy" schema = Schema(ScalarField("PolicyName", "name"), ScalarField("PolicyId")) @classmethod def list_from_aws( cls: Type["IAMAWSManagedPolicyResourceSpec"], client: BaseClient, account_id: str, region: str, ) -> ListFromAWSResult: """Return a dict of dicts of the format: {'role_1_arn': {role_1_dict}, 'role_2_arn': {role_2_dict}, ...} Where the dicts represent results from list_policies and additional info per role from list_targets_by_role.""" policies = {} paginator = client.get_paginator("list_policies") for resp in paginator.paginate(Scope="AWS", OnlyAttached=True): for policy in resp.get("Policies", []): resource_arn = policy["Arn"] policies[resource_arn] = policy return ListFromAWSResult(resources=policies)
class RegionResourceSpec(EC2ResourceSpec): """Resource representing an AWS Region""" type_name = "region" scan_granularity = ScanGranularity.ACCOUNT schema = Schema(ScalarField("RegionName", "name"), ScalarField("OptInStatus")) @classmethod def get_full_type_name(cls: Type["RegionResourceSpec"]) -> str: return f"{cls.provider_name}:{cls.type_name}" @classmethod def list_from_aws(cls: Type["RegionResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'region_1_arn': {region_1_dict}, 'region_2_arn': {region_2_dict}, ...} Where the dicts represent results from describe_regions.""" regions = {} resp = client.describe_regions(AllRegions=True) for region_resp in resp["Regions"]: region_name = region_resp["RegionName"] region_arn = f"arn:aws:::{account_id}:region/{region_name}" regions[region_arn] = region_resp return ListFromAWSResult(resources=regions)
class OrgResourceSpec(OrganizationsResourceSpec): """Resource representing an AWS Org.""" type_name = "organization" schema = Schema(ScalarField("MasterAccountId"), ScalarField("MasterAccountEmail")) @classmethod def get_full_type_name(cls: Type["OrgResourceSpec"]) -> str: return f"{cls.provider_name}:{cls.type_name}" @classmethod def list_from_aws(cls: Type["OrgResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'org_1_arn': {org_1_dict}, 'org_2_arn': {org_2_dict}, ...} Where the dicts represent results from describe_organization.""" resp = client.describe_organization() org = resp["Organization"] orgs = {org["Arn"]: org} return ListFromAWSResult(resources=orgs)
class EBSSnapshotResourceSpec(EC2ResourceSpec): """Resource for EBSSnapshots""" type_name = "snapshot" schema = Schema( ScalarField("VolumeSize"), ScalarField("Encrypted"), TransientResourceLinkField("KmsKeyId", KMSKeyResourceSpec, optional=True, value_is_id=True), TransientResourceLinkField("VolumeId", EBSVolumeResourceSpec), TagsField(), ) @classmethod def list_from_aws(cls: Type["EBSSnapshotResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'snapshot_1_arn': {snapshot_1_dict}, 'snapshot_2_arn': {snapshot_2_dict}, ...} Where the dicts represent results from describe_snapshots.""" snapshots = {} paginator = client.get_paginator("describe_snapshots") for resp in paginator.paginate(OwnerIds=["self"]): for snapshot in resp.get("Snapshots", []): resource_arn = cls.generate_arn( account_id=account_id, region=region, resource_id=snapshot["SnapshotId"]) snapshots[resource_arn] = snapshot return ListFromAWSResult(resources=snapshots)
def test_valid_dicts_input_with_alti_key(self): input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}' field = ListField( "People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person" ) input_data = json.loads(input_str) link_collection = field.parse(data=input_data, context={}) expected_link_collection = LinkCollection( multi_links=( MultiLink( pred="person", obj=LinkCollection( simple_links=( SimpleLink(pred="name", obj="Bob"), SimpleLink(pred="age", obj=49), ), ), ), MultiLink( pred="person", obj=LinkCollection( simple_links=( SimpleLink(pred="name", obj="Sue"), SimpleLink(pred="age", obj=42), ), ), ), ), ) self.assertEqual(link_collection, expected_link_collection)
class CloudTrailTrailResourceSpec(CloudTrailResourceSpec): """Resource representing a CloudTrail Trail""" type_name = "trail" schema = Schema( ScalarField("Name"), ScalarField("S3BucketName"), ScalarField("IncludeGlobalServiceEvents"), ScalarField("IsMultiRegionTrail"), ) @classmethod def list_from_aws( cls: Type["CloudTrailTrailResourceSpec"], client: BaseClient, account_id: str, region: str, ) -> ListFromAWSResult: """Return a dict of dicts of the format: {'trail_1_arn': {trail_1_dict}, 'trail_2_arn': {trail_2_dict}, ...} Where the dicts represent results from describe_trails.""" trails = {} resp = client.describe_trails(includeShadowTrails=False) for trail in resp.get("trailList", []): resource_arn = trail["TrailARN"] trails[resource_arn] = trail return ListFromAWSResult(resources=trails)
def test_valid_dicts_input(self): input_str = ( '{"Biota": {"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}}' ) field = DictField( "Biota", AnonymousListField( "People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))), ) expected_output_data = [{ "pred": "biota", "obj": [ { "pred": "biota", "obj": [ { "pred": "name", "obj": "Bob", "type": "simple" }, { "pred": "age", "obj": 49, "type": "simple" }, ], "type": "multi", }, { "pred": "biota", "obj": [ { "pred": "name", "obj": "Sue", "type": "simple" }, { "pred": "age", "obj": 42, "type": "simple" }, ], "type": "multi", }, ], "type": "multi", }] input_data = json.loads(input_str) links = field.parse(data=input_data, context={}) output_data = [link.to_dict() for link in links] self.assertCountEqual(output_data, expected_output_data)
def test_key_absent_with_optional(self): input_str = "{}" field = ScalarField("FieldName", optional=True) input_data = json.loads(input_str) link_collection = field.parse(data=input_data, context={}) self.assertEqual(link_collection, LinkCollection())
def test_invalid_input_missing_source_key(self): input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}' field = ListField( "Stuff", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person" ) input_data = json.loads(input_str) with self.assertRaises(ListFieldSourceKeyNotFoundException): field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
def test_invalid_input_not_list(self): input_str = '{"People": "foo"}' field = AnonymousListField( "People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")) ) input_data = json.loads(input_str) with self.assertRaises(ListFieldValueNotAListException): field.parse(data=input_data, context={"parent_alti_key": "test_parent"})
class IAMPolicyResourceSpec(IAMResourceSpec): """Resource for user-managed IAM Policies""" type_name = "policy" parallel_scan = True schema = Schema( ScalarField("PolicyName", "name"), ScalarField("PolicyId"), ScalarField("DefaultVersionId"), ScalarField("DefaultVersionPolicyDocumentText"), ) @classmethod def list_from_aws(cls: Type["IAMPolicyResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'role_1_arn': {role_1_dict}, 'role_2_arn': {role_2_dict}, ...} Where the dicts represent results from list_policies and additional info per role from list_targets_by_role.""" policies = {} paginator = client.get_paginator("list_policies") for resp in paginator.paginate(Scope="Local"): for policy in resp.get("Policies", []): resource_arn = policy["Arn"] default_policy_version = policy["DefaultVersionId"] try: default_policy_version_document_text = cls.get_policy_version_document_text( client=client, policy_arn=resource_arn, policy_version=default_policy_version, ) policy[ "DefaultVersionPolicyDocumentText"] = policy_doc_dict_to_sorted_str( default_policy_version_document_text) policies[resource_arn] = policy except ClientError as c_e: error_code = getattr(c_e, "response", {}).get("Error", {}).get("Code", {}) if error_code != "NoSuchEntity": raise c_e return ListFromAWSResult(resources=policies) @classmethod def get_policy_version_document_text( cls: Type["IAMPolicyResourceSpec"], client: BaseClient, policy_arn: str, policy_version: str, ) -> Dict[str, Any]: policy_version_resp = client.get_policy_version( PolicyArn=policy_arn, VersionId=policy_version) return policy_version_resp["PolicyVersion"]["Document"]
def test_key_absent_with_optional(self): input_str = "{}" field = ScalarField("FieldName", optional=True) expected_output_data = [] input_data = json.loads(input_str) links = field.parse(data=input_data, context={}) output_data = [link.to_dict() for link in links] self.assertCountEqual(output_data, expected_output_data)
def test_parse(self): schema = Schema(ScalarField("Key1"), ScalarField("Key2")) data = {"Key1": "Value1", "Key2": "Value2"} link_collection = schema.parse(data, {}) expected_link_collection = LinkCollection(simple_links=( SimpleLink(pred="key1", obj="Value1"), SimpleLink(pred="key2", obj="Value2"), )) self.assertEqual(link_collection, expected_link_collection)
def test_valid_input_with_alti_key(self): input_str = '{"FieldName": "Value"}' field = ScalarField("FieldName", alti_key="alti_field_name") input_data = json.loads(input_str) link_collection = field.parse(data=input_data, context={}) expected_link_collection = LinkCollection(simple_links=(SimpleLink( pred="alti_field_name", obj="Value"), ), ) self.assertEqual(link_collection, expected_link_collection)
def test_key_present_with_optional(self): input_str = '{"FieldName": "Value"}' field = ScalarField("FieldName", optional=True) input_data = json.loads(input_str) link_collection = field.parse(data=input_data, context={}) expected_link_collection = LinkCollection(simple_links=(SimpleLink( pred="field_name", obj="Value"), ), ) self.assertEqual(link_collection, expected_link_collection)
def test_key_absent_with_default(self): input_str = "{}" field = ScalarField("FieldName", default_value="DefaultValue") input_data = json.loads(input_str) link_collection = field.parse(data=input_data, context={}) expected_link_collection = LinkCollection(simple_links=(SimpleLink( pred="field_name", obj="DefaultValue"), ), ) self.assertEqual(link_collection, expected_link_collection)
class LoadBalancerResourceSpec(ElasticLoadBalancingResourceSpec): """Resource for load balancer""" type_name = "loadbalancer" schema = Schema( ScalarField("DNSName"), ScalarField("CreatedTime"), ScalarField("LoadBalancerName"), ScalarField("Scheme"), ResourceLinkField("VpcId", VPCResourceSpec, optional=True), AnonymousDictField("State", ScalarField("Code", alti_key="load_balancer_state")), ScalarField("Type"), ListField( "AvailabilityZones", EmbeddedDictField( ScalarField("ZoneName"), ResourceLinkField("SubnetId", SubnetResourceSpec, optional=True), ListField( "LoadBalancerAddresses", EmbeddedDictField( ScalarField("IpAddress", optional=True), ScalarField("AllocationId", optional=True), ), optional=True, ), ), ), ListField("SecurityGroups", EmbeddedResourceLinkField(SecurityGroupResourceSpec), optional=True), ScalarField("IpAddressType"), ) @classmethod def list_from_aws(cls: Type["LoadBalancerResourceSpec"], client: BaseClient, account_id: str, region: str) -> ListFromAWSResult: """Return a dict of dicts of the format: {'lb_1_arn': {lb_1_dict}, 'lb_2_arn': {lb_2_dict}, ...} Where the dicts represent results from describe_load_balancers.""" paginator = client.get_paginator("describe_load_balancers") load_balancers = {} for resp in paginator.paginate(): for lb in resp.get("LoadBalancers", []): resource_arn = lb["LoadBalancerArn"] load_balancers[resource_arn] = lb return ListFromAWSResult(resources=load_balancers)
def test_key_present_with_optional(self): input_str = '{"FieldName": "Value"}' field = ScalarField("FieldName", optional=True) expected_output_data = [{ "pred": "field_name", "obj": "Value", "type": "simple" }] input_data = json.loads(input_str) links = field.parse(data=input_data, context={}) output_data = [link.to_dict() for link in links] self.assertCountEqual(output_data, expected_output_data)
def test_key_absent_with_default(self): input_str = "{}" field = ScalarField("FieldName", default_value="DefaultValue") expected_output_data = [{ "pred": "field_name", "obj": "DefaultValue", "type": "simple" }] input_data = json.loads(input_str) links = field.parse(data=input_data, context={}) output_data = [link.to_dict() for link in links] self.assertCountEqual(output_data, expected_output_data)
def test_valid_input_with_alti_key(self): input_str = '{"FieldName": "Value"}' field = ScalarField("FieldName", alti_key="alti_field_name") expected_output_data = [{ "pred": "alti_field_name", "obj": "Value", "type": "simple" }] input_data = json.loads(input_str) links = field.parse(data=input_data, context={}) output_data = [link.to_dict() for link in links] self.assertCountEqual(output_data, expected_output_data)
class TransitGatewayResourceSpec(EC2ResourceSpec): """Resource for TransitGateways""" # type_name = "transit-gateway" type_name = "transit_gateway" schema = Schema( ScalarField("OwnerId"), ScalarField("State"), ListField( "VPCAttachments", EmbeddedDictField( ScalarField("CreationTime"), ScalarField("State"), ResourceLinkField("ResourceId", VPCResourceSpec), ), optional=True, alti_key="vpc_attachment", ), TagsField(), ) @classmethod def list_from_aws( cls: Type["TransitGatewayResourceSpec"], client: BaseClient, account_id: str, region: str ) -> ListFromAWSResult: """Return a dict of dicts of the format: {'tgw_1_arn': {tgw_1_dict}, 'tgw_2_arn': {tgw_2_dict}, ...} Where the dicts represent results from describe_transit_gateways.""" tgws = {} paginator = client.get_paginator("describe_transit_gateways") tgw_filters = [{"Name": "owner-id", "Values": [account_id]}] for resp in paginator.paginate(Filters=tgw_filters): for tgw in resp["TransitGateways"]: resource_arn = tgw["TransitGatewayArn"] vpc_attachments: List[Dict[str, Any]] = [] vpc_attachments_paginator = client.get_paginator( "describe_transit_gateway_attachments" ) vpc_filters = [ {"Name": "transit-gateway-id", "Values": [tgw["TransitGatewayId"]]}, {"Name": "resource-type", "Values": ["vpc"]}, ] for vpc_attachments_resp in vpc_attachments_paginator.paginate(Filters=vpc_filters): vpc_attachments += vpc_attachments_resp["TransitGatewayAttachments"] tgw["VPCAttachments"] = vpc_attachments tgws[resource_arn] = tgw return ListFromAWSResult(resources=tgws)
class TransitGatewayVpcAttachmentResourceSpec(EC2ResourceSpec): """Resource for Transit Gateway VPC Attachments""" type_name = "transit-gateway-vpc-attachment" schema = Schema( ScalarField("TransitGatewayAttachmentId"), ScalarField("TransitGatewayId"), ScalarField("VpcId"), ScalarField("VpcOwnerId"), ScalarField("State"), ScalarField("CreationTime"), ListField("SubnetIds", EmbeddedScalarField(), alti_key="subnet_id"), AnonymousDictField("Options", ScalarField("DnsSupport"), ScalarField("Ipv6Support")), ) @classmethod def list_from_aws( cls: Type["TransitGatewayVpcAttachmentResourceSpec"], client: BaseClient, account_id: str, region: str, ) -> ListFromAWSResult: paginator = client.get_paginator( "describe_transit_gateway_vpc_attachments") attachments = {} for resp in paginator.paginate(): for attachment in resp.get("TransitGatewayVpcAttachments", []): resource_arn = cls.generate_arn( account_id, region, attachment["TransitGatewayAttachmentId"]) attachments[resource_arn] = attachment return ListFromAWSResult(resources=attachments)