def test_scan(self):
scan_accessor = TestScanAccessor()
graph_spec = GraphSpec(
name="test-name",
version="1",
resource_spec_classes=(TestResourceSpecA, TestResourceSpecB),
scan_accessor=scan_accessor,
)
resources = graph_spec.scan()
expected_resources = [
Resource(resource_id="123",
type="a",
link_collection=LinkCollection()),
Resource(resource_id="456",
type="a",
link_collection=LinkCollection()),
Resource(resource_id="abc",
type="b",
link_collection=LinkCollection()),
Resource(resource_id="def",
type="b",
link_collection=LinkCollection()),
]
self.assertEqual(resources, expected_resources)
def scan(cls: Type["TestResourceSpecB"],
scan_accessor: Any) -> List[Resource]:
resources = [
Resource(resource_id="abc", type_name=cls.type_name, links=[]),
Resource(resource_id="def", type_name=cls.type_name, links=[]),
]
return resources
def test_scan(self):
scan_accessor = TestScanAccessor()
graph_spec = GraphSpec(
name="test-name",
version="1",
resource_spec_classes=(TestResourceSpecA, TestResourceSpecB),
scan_accessor=scan_accessor,
)
graph_set = graph_spec.scan()
self.assertEqual(graph_set.name, "test-name")
self.assertEqual(graph_set.version, "1")
self.assertEqual(graph_set.errors, [])
expected_resource_dicts = [
Resource(resource_id="123",
type_name=TestResourceSpecA.type_name).to_dict(),
Resource(resource_id="456",
type_name=TestResourceSpecA.type_name).to_dict(),
Resource(resource_id="abc",
type_name=TestResourceSpecB.type_name).to_dict(),
Resource(resource_id="def",
type_name=TestResourceSpecB.type_name).to_dict(),
]
self.assertCountEqual(
[resource.to_dict() for resource in graph_set.resources],
expected_resource_dicts)
def setUp(self):
resource_a1 = Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]),
)
resource_a2 = Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),)
resource_b1 = Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-a", obj="123")]),
)
resource_b2 = Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="name", obj="sue")]),
)
resources = (resource_a1, resource_a2, resource_b1, resource_b2)
self.validated_graph_set = ValidatedGraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=["test err 1", "test err 2"],
)
def test_graph_content(self):
expected_resources = (
Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-foo", obj="goo"),),
),
),
Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),),
Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-a", obj="123"),),
),
),
Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=(SimpleLink(pred="name", obj="sue"),),),
),
)
expected_errors = ["test err 1", "test err 2"]
self.assertEqual(self.validated_graph_set.resources, expected_resources)
self.assertEqual(self.validated_graph_set.errors, expected_errors)
def scan(cls: Type["TestResourceSpecB"],
scan_accessor: Any) -> ResourceScanResult:
resources = [
Resource(resource_id="abc", type_name=cls.type_name, links=[]),
Resource(resource_id="def", type_name=cls.type_name, links=[]),
]
return ResourceScanResult(resources=resources,
stats=MultilevelCounter(),
errors=[])
def scan(cls: Type["TestResourceSpecB"],
scan_accessor: Any) -> List[Resource]:
resources = [
Resource(resource_id="abc",
type=cls.type_name,
link_collection=LinkCollection()),
Resource(resource_id="def",
type=cls.type_name,
link_collection=LinkCollection()),
]
return resources
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
ec2_client = session.client("ec2", region_name=region_name)
list_resp = ec2_client.describe_vpcs()
present_vpcs = list_resp["Vpcs"]
self.assertEqual(len(present_vpcs), 1)
present_vpc_id = present_vpcs[0]["VpcId"]
present_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{present_vpc_id}"
create_resp = ec2_client.create_vpc(CidrBlock="10.0.0.0/16")
created_vpc_id = create_resp["Vpc"]["VpcId"]
created_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{created_vpc_id}"
scan_accessor = AWSAccessor(session=session, account_id=account_id, region_name=region_name)
resources = VPCResourceSpec.scan(scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id=present_vpc_arn,
type="aws:ec2:vpc",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="is_default", obj=True),
SimpleLink(pred="cidr_block", obj="172.31.0.0/16"),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(pred="account", obj="arn:aws::::account/123456789012"),
ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"),
),
),
),
Resource(
resource_id=created_vpc_arn,
type="aws:ec2:vpc",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="is_default", obj=False),
SimpleLink(pred="cidr_block", obj="10.0.0.0/16"),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(pred="account", obj="arn:aws::::account/123456789012"),
ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"),
),
),
),
]
self.assertEqual(resources, expected_resources)
def test_invalid_resources_dupes_same_class_conflicting_types_no_allow_clobber(self):
resources = [
Resource(resource_id="123", type="test:a", link_collection=LinkCollection()),
Resource(resource_id="123", type="test:b", link_collection=LinkCollection()),
]
with self.assertRaises(UnmergableDuplicateResourceIdsFoundException):
ValidatedGraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=[],
)
def test_unknown_type_name(self):
resources = [
Resource(resource_id="xyz", type="test:a", link_collection=LinkCollection()),
Resource(resource_id="xyz", type="test:c", link_collection=LinkCollection()),
]
with self.assertRaises(ResourceSpecClassNotFoundException):
ValidatedGraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=[],
)
def test_unknown_type_name(self):
resources = [
Resource(resource_id="xyz", type_name="test:a"),
Resource(resource_id="xyz", type_name="test:c"),
]
with self.assertRaises(ResourceSpecClassNotFoundException):
GraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=[],
stats=MultilevelCounter(),
)
def test_invalid_resources_dupes_same_class_conflicting_types_no_allow_clobber(
self):
resources = [
Resource(resource_id="123", type_name="test:a"),
Resource(resource_id="123", type_name="test:b"),
]
with self.assertRaises(UnmergableDuplicateResourceIdsFoundException):
GraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=[],
stats=MultilevelCounter(),
)
def _list_from_aws_result_to_resources(
cls: Type["AWSResourceSpec"],
list_from_aws_result: ListFromAWSResult,
context: Dict[str, str],
) -> List[Resource]:
resources: List[Resource] = []
for arn, resource_dict in list_from_aws_result.resources.items():
try:
links = cls.schema.parse(resource_dict, context)
except Exception as ex:
raise SchemaParseException(
(
f"Error parsing {cls.__name__} : "
f"{arn}:\n\nData: {resource_dict}\n\nError: {ex}"
)
)
resource_account_id = arn.split(":")[4]
if resource_account_id:
if resource_account_id != "aws":
account_link = ResourceLinkLink(
pred="account", obj=f"arn:aws::::account/{resource_account_id}"
)
links.append(account_link)
resource_region_name = arn.split(":")[3]
if resource_region_name:
region_link = ResourceLinkLink(
pred="region",
obj=f"arn:aws:::{resource_account_id}:region/{resource_region_name}",
)
links.append(region_link)
resource = Resource(resource_id=arn, type_name=cls.get_full_type_name(), links=links)
resources.append(resource)
return resources
def from_dict(cls: Type["GraphSet"], data: Dict[str, Any]) -> "GraphSet":
"""Create a GraphSet from a dict.
Args:
data: dict of Resource data
Returns:
GraphSet object
"""
resources: List[Resource] = []
name = data["name"]
start_time = data["start_time"]
end_time = data["end_time"]
version = data["version"]
errors = data["errors"]
stats = MultilevelCounter.from_dict(data["stats"])
for resource_id, resource_data in data["resources"].items():
resource = Resource.from_dict(resource_id, resource_data)
resources.append(resource)
return cls(
name=name,
version=version,
start_time=start_time,
end_time=end_time,
resources=resources,
errors=errors,
stats=stats,
)
def test_schema_parse(self):
resource_arn = "arn:aws:ec2:us-west-2:111122223333:vpc-endpoint-service/com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01"
aws_resource_dict = {
"ServiceType": [{"ServiceType": "Interface"}],
"ServiceId": "vpce-svc-01234abcd5678ef01",
"ServiceName": "com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01",
"ServiceState": "Available",
"AvailabilityZones": ["us-west-2a", "us-west-2b"],
"AcceptanceRequired": True,
"ManagesVpcEndpoints": False,
"NetworkLoadBalancerArns": [
"arn:aws:elasticloadbalancing:us-west-2:111122223333:loadbalancer/net/splunk-hwf-lb/1a7ff9c18eeaaf9b"
],
"BaseEndpointDnsNames": ["vpce-svc-01234abcd5678ef01.us-west-2.vpce.amazonaws.com"],
"PrivateDnsNameConfiguration": {},
"Tags": [{"Key": "Name", "Value": "Splunk HEC"}],
}
link_collection = VpcEndpointServiceResourceSpec.schema.parse(
data=aws_resource_dict, context={"account_id": "111122223333", "region": "us-west-2"}
)
resource = Resource(
resource_id=resource_arn,
type=VpcEndpointServiceResourceSpec.type_name,
link_collection=link_collection,
)
expected_resource = Resource(
resource_id="arn:aws:ec2:us-west-2:111122223333:vpc-endpoint-service/com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01",
type="vpc-endpoint-service",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="service_type", obj="Interface"),
SimpleLink(
pred="service_name",
obj="com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01",
),
SimpleLink(pred="service_state", obj="Available"),
SimpleLink(pred="acceptance_required", obj=True),
SimpleLink(pred="availability_zones", obj="us-west-2a"),
SimpleLink(pred="availability_zones", obj="us-west-2b"),
),
tag_links=(TagLink(pred="Name", obj="Splunk HEC"),),
),
)
self.assertEqual(resource, expected_resource)
def test_schema_parse(self):
resource_arn = "arn:aws:iam:us-west-2:111122223333:account-password-policy/default"
aws_resource_dict = {
"MinimumPasswordLength": 12,
"RequireSymbols": True,
"RequireNumbers": True,
"RequireUppercaseCharacters": True,
"RequireLowercaseCharacters": True,
"AllowUsersToChangePassword": True,
"ExpirePasswords": True,
"MaxPasswordAge": 90,
"PasswordReusePrevention": 5,
"HardExpiry": True,
}
link_collection = IAMAccountPasswordPolicyResourceSpec.schema.parse(
data=aws_resource_dict,
context={
"account_id": "111122223333",
"region": "us-west-2"
})
resource = Resource(
resource_id=resource_arn,
type=IAMAccountPasswordPolicyResourceSpec.type_name,
link_collection=link_collection,
)
expected_resource = Resource(
resource_id=
"arn:aws:iam:us-west-2:111122223333:account-password-policy/default",
type="account-password-policy",
link_collection=LinkCollection(simple_links=(
SimpleLink(pred="minimum_password_length", obj=12),
SimpleLink(pred="require_symbols", obj=True),
SimpleLink(pred="require_numbers", obj=True),
SimpleLink(pred="require_uppercase_characters", obj=True),
SimpleLink(pred="require_lowercase_characters", obj=True),
SimpleLink(pred="allow_users_to_change_password", obj=True),
SimpleLink(pred="expire_passwords", obj=True),
SimpleLink(pred="max_password_age", obj=90),
SimpleLink(pred="password_reuse_prevention", obj=5),
SimpleLink(pred="hard_expiry", obj=True),
)),
)
self.assertEqual(resource, expected_resource)
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
client = session.client("iam")
oidc_url = "https://oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E"
oidc_client_ids = ["sts.amazonaws.com"]
oidc_thumbprints = ["9999999999999999999999999999999999999999"]
_ = client.create_open_id_connect_provider(
Url=oidc_url,
ClientIDList=oidc_client_ids,
ThumbprintList=oidc_thumbprints,
)
scan_accessor = AWSAccessor(session=session,
account_id=account_id,
region_name=region_name)
resources = IAMOIDCProviderResourceSpec.scan(
scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id=
"arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E",
type="aws:iam:oidc-provider",
link_collection=LinkCollection(
simple_links=(
SimpleLink(
pred="url",
obj=
"oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E",
),
SimpleLink(
pred="create_date",
obj=resources[0].link_collection.simple_links[1].
obj,
),
SimpleLink(pred="client_id", obj="sts.amazonaws.com"),
SimpleLink(
pred="thumbprint",
obj="9999999999999999999999999999999999999999"),
),
multi_links=None,
tag_links=None,
resource_links=(ResourceLink(
pred="account",
obj="arn:aws::::account/123456789012"), ),
transient_resource_links=None,
),
)
]
self.assertEqual(resources, expected_resources)
def test_orphaned_ref(self):
resource_a1 = Resource(resource_id="123",
type_name="test:a",
links=[SimpleLink(pred="has-foo", obj="goo")])
resource_b1 = Resource(
resource_id="abc",
type_name="test:b",
links=[ResourceLinkLink(pred="has-a", obj="456")])
resources = [resource_a1, resource_b1]
graph_set = GraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=["test err 1", "test err 2"],
stats=MultilevelCounter(),
)
with self.assertRaises(GraphSetOrphanedReferencesException):
graph_set.validate()
def create_resource(cls: Type["UnscannedAccountResourceSpec"],
account_id: str, errors: List[str]) -> Resource:
links: List[Link] = []
links.append(SimpleLink(pred="account_id", obj=account_id))
for error in errors:
link = SimpleLink(pred="error", obj=error)
links.append(link)
return Resource(
resource_id=cls.generate_arn("", "", account_id),
type_name=cls.get_full_type_name(),
links=links,
)
def test_orphaned_ref(self):
resource_a1 = Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]),
)
resource_b1 = Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(resource_links=[ResourceLink(pred="has-a", obj="456")]),
)
resources = [resource_a1, resource_b1]
graph_set = GraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=["test err 1", "test err 2"],
)
with self.assertRaises(GraphSetOrphanedReferencesException):
ValidatedGraphSet.from_graph_set(graph_set)
def create_resource(cls: Type["UnscannedAccountResourceSpec"],
account_id: str, errors: List[str]) -> Resource:
links: List[Link] = []
links.append(SimpleLink(pred="account_id", obj=account_id))
if errors:
error = "\n".join(errors)
links.append(
SimpleLink(pred="error", obj=f"{error} - {uuid.uuid4()}"))
return Resource(
resource_id=cls.generate_arn(resource_id=account_id),
type_name=cls.get_full_type_name(),
links=links,
)
def setUp(self):
resource_a1 = Resource(resource_id="123",
type_name="test:a",
links=[SimpleLink(pred="has-foo", obj="goo")])
resource_a2 = Resource(resource_id="456", type_name="test:a")
resource_b1 = Resource(
resource_id="abc",
type_name="test:b",
links=[ResourceLinkLink(pred="has-a", obj="123")])
resource_b2 = Resource(resource_id="def",
type_name="test:b",
links=[SimpleLink(pred="name", obj="sue")])
resources = [resource_a1, resource_a2, resource_b1, resource_b2]
self.graph_set = GraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=["test err 1", "test err 2"],
stats=MultilevelCounter(),
)
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
scan_accessor = AWSAccessor(session=session,
account_id=account_id,
region_name=region_name)
resources = AccountResourceSpec.scan(scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id="arn:aws::::account/123456789012",
type="aws:account",
link_collection=LinkCollection(simple_links=(SimpleLink(
pred="account_id", obj="123456789012"), ), ),
)
]
self.assertEqual(resources, expected_resources)
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
ec2_client = session.client("ec2", region_name=region_name)
resp = ec2_client.create_volume(Size=1, AvailabilityZone="us-east-1a")
create_time = resp["CreateTime"]
created_volume_id = resp["VolumeId"]
created_volume_arn = f"arn:aws:ec2:us-east-1:123456789012:volume/{created_volume_id}"
scan_accessor = AWSAccessor(session=session,
account_id=account_id,
region_name=region_name)
resources = EBSVolumeResourceSpec.scan(scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id=created_volume_arn,
type="aws:ec2:volume",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="availability_zone", obj="us-east-1a"),
SimpleLink(pred="create_time", obj=create_time),
SimpleLink(pred="size", obj=True),
SimpleLink(pred="state", obj="available"),
SimpleLink(pred="volume_type", obj="standard"),
SimpleLink(pred="encrypted", obj=False),
),
resource_links=(
ResourceLink(pred="account",
obj="arn:aws::::account/123456789012"),
ResourceLink(
pred="region",
obj="arn:aws:::123456789012:region/us-east-1"),
),
),
)
]
self.assertEqual(resources, expected_resources)
def test_schema_parse(self):
resource_arn = "arn:aws:ec2:us-east-2:111122223333:route-table/rtb-099c7b032f2bbddda"
aws_resource_dict = {
"Associations": [
{
"Main": False,
"RouteTableAssociationId": "rtbassoc-069d59127bf10a728",
"RouteTableId": "rtb-099c7b032f2bbddda",
"SubnetId": "subnet-00f9fe55b9d7ca4fb",
},
{
"Main": False,
"RouteTableAssociationId": "rtbassoc-07bfd170c4ece33c8",
"RouteTableId": "rtb-099c7b032f2bbddda",
"SubnetId": "subnet-0b98092b454c882cf",
},
],
"PropagatingVgws": [],
"RouteTableId":
"rtb-099c7b032f2bbddda",
"Routes": [
{
"DestinationCidrBlock": "172.31.0.0/16",
"GatewayId": "local",
"Origin": "CreateRouteTable",
"State": "active",
},
{
"DestinationCidrBlock": "0.0.0.0/0",
"GatewayId": "igw-092e5ec1685fd0c0b",
"Origin": "CreateRoute",
"State": "active",
},
{
"DestinationPrefixListId": "pl-68a54001",
"GatewayId": "vpce-0678bce2b63b8ad0f",
"Origin": "CreateRoute",
"State": "active",
},
],
"VpcId":
"vpc-03c33051f57d21ff0",
"OwnerId":
"210554966933",
}
links = EC2RouteTableResourceSpec.schema.parse(data=aws_resource_dict,
context={
"account_id":
"111122223333",
"region":
"us-west-2"
})
resource = Resource(resource_id=resource_arn,
type_name=EC2RouteTableResourceSpec.type_name,
links=links)
alti_resource_dict = resource.to_dict()
expected_alti_resource_dict = {
"type":
"route-table",
"links": [
{
"pred": "route_table_id",
"obj": "rtb-099c7b032f2bbddda",
"type": "simple"
},
{
"pred": "vpc",
"obj":
"arn:aws:ec2:us-west-2:111122223333:vpc/vpc-03c33051f57d21ff0",
"type": "resource_link",
},
{
"pred": "owner_id",
"obj": "210554966933",
"type": "simple"
},
{
"pred":
"route",
"obj": [
{
"pred": "destination_cidr_block",
"obj": "172.31.0.0/16",
"type": "simple",
},
{
"pred": "gateway_id",
"obj": "local",
"type": "simple"
},
{
"pred": "origin",
"obj": "CreateRouteTable",
"type": "simple"
},
{
"pred": "state",
"obj": "active",
"type": "simple"
},
],
"type":
"multi",
},
{
"pred":
"route",
"obj": [
{
"pred": "destination_cidr_block",
"obj": "0.0.0.0/0",
"type": "simple"
},
{
"pred": "gateway_id",
"obj": "igw-092e5ec1685fd0c0b",
"type": "simple"
},
{
"pred": "origin",
"obj": "CreateRoute",
"type": "simple"
},
{
"pred": "state",
"obj": "active",
"type": "simple"
},
],
"type":
"multi",
},
{
"pred":
"route",
"obj": [
{
"pred": "destination_prefix_list_id",
"obj": "pl-68a54001",
"type": "simple",
},
{
"pred": "gateway_id",
"obj": "vpce-0678bce2b63b8ad0f",
"type": "simple"
},
{
"pred": "origin",
"obj": "CreateRoute",
"type": "simple"
},
{
"pred": "state",
"obj": "active",
"type": "simple"
},
],
"type":
"multi",
},
{
"pred":
"association",
"obj": [
{
"pred": "main",
"obj": False,
"type": "simple"
},
{
"pred": "route_table_association_id",
"obj": "rtbassoc-069d59127bf10a728",
"type": "simple",
},
{
"pred": "route_table_id",
"obj": "rtb-099c7b032f2bbddda",
"type": "simple",
},
{
"pred": "subnet_id",
"obj": "subnet-00f9fe55b9d7ca4fb",
"type": "simple"
},
],
"type":
"multi",
},
{
"pred":
"association",
"obj": [
{
"pred": "main",
"obj": False,
"type": "simple"
},
{
"pred": "route_table_association_id",
"obj": "rtbassoc-07bfd170c4ece33c8",
"type": "simple",
},
{
"pred": "route_table_id",
"obj": "rtb-099c7b032f2bbddda",
"type": "simple",
},
{
"pred": "subnet_id",
"obj": "subnet-0b98092b454c882cf",
"type": "simple"
},
],
"type":
"multi",
},
],
}
self.assertDictEqual(alti_resource_dict, expected_alti_resource_dict)
def test_valid_merge(self):
resource_a1 = Resource(resource_id="123",
type_name="test:a",
links=[SimpleLink(pred="has-foo", obj="goo")])
resource_a2 = Resource(resource_id="456", type_name="test:a")
resource_b1 = Resource(
resource_id="abc",
type_name="test:b",
links=[ResourceLinkLink(pred="has-a", obj="123")])
resource_b2 = Resource(resource_id="def",
type_name="test:b",
links=[SimpleLink(pred="name", obj="sue")])
graph_set_1 = GraphSet(
name="graph-1",
version="1",
start_time=10,
end_time=20,
resources=[resource_a1, resource_a2],
errors=["errora1", "errora2"],
stats=MultilevelCounter(),
)
graph_set_2 = GraphSet(
name="graph-1",
version="1",
start_time=15,
end_time=25,
resources=[resource_b1, resource_b2],
errors=["errorb1", "errorb2"],
stats=MultilevelCounter(),
)
graph_set_1.merge(graph_set_2)
self.assertEqual(graph_set_1.name, "graph-1")
self.assertEqual(graph_set_1.version, "1")
self.assertEqual(graph_set_1.start_time, 10)
self.assertEqual(graph_set_1.end_time, 25)
self.assertCountEqual(graph_set_1.errors,
["errora1", "errora2", "errorb1", "errorb2"])
expected_resource_dicts = [
{
"type": "test:a",
"links": [{
"pred": "has-foo",
"obj": "goo",
"type": "simple"
}]
},
{
"type": "test:a"
},
{
"type": "test:b",
"links": [{
"pred": "has-a",
"obj": "123",
"type": "resource_link"
}]
},
{
"type": "test:b",
"links": [{
"pred": "name",
"obj": "sue",
"type": "simple"
}]
},
]
resource_dicts = [
resource.to_dict() for resource in graph_set_1.resources
]
self.assertCountEqual(expected_resource_dicts, resource_dicts)
def test_schema_parse(self):
self.maxDiff = None
resource_arn = "arn:aws:ec2:us-east-2:111122223333:transit-gateway-vpc-attachment/tgw-attach-09ece7878ee9ab7a4"
aws_resource_dict = {
"TransitGatewayAttachmentId":
"tgw-attach-09ece7878ee9ab7a4",
"TransitGatewayId":
"tgw-086b599bebfee5d40",
"VpcId":
"vpc-01e8457e8c00c40a7",
"VpcOwnerId":
"123456789012",
"State":
"available",
"SubnetIds":
["subnet-07697f82fe4c6a8d6", "subnet-0396137c18d6c30ef"],
"CreationTime":
datetime.datetime(2019, 8, 23, 15, 59, 46, tzinfo=tzutc()),
"Options": {
"DnsSupport": "enable",
"Ipv6Support": "disable"
},
"Tags": [{
"Key": "Name",
"Value": "customer-dev-tgw-attachment"
}],
}
link_collection = TransitGatewayVpcAttachmentResourceSpec.schema.parse(
data=aws_resource_dict,
context={
"account_id": "111122223333",
"region": "us-west-2"
})
resource = Resource(
resource_id=resource_arn,
type=TransitGatewayVpcAttachmentResourceSpec.type_name,
link_collection=link_collection,
)
expected_resource = Resource(
resource_id=
"arn:aws:ec2:us-east-2:111122223333:transit-gateway-vpc-attachment/tgw-attach-09ece7878ee9ab7a4",
type="transit-gateway-vpc-attachment",
link_collection=LinkCollection(simple_links=(
SimpleLink(pred="transit_gateway_attachment_id",
obj="tgw-attach-09ece7878ee9ab7a4"),
SimpleLink(pred="transit_gateway_id",
obj="tgw-086b599bebfee5d40"),
SimpleLink(pred="vpc_id", obj="vpc-01e8457e8c00c40a7"),
SimpleLink(pred="vpc_owner_id", obj="123456789012"),
SimpleLink(pred="state", obj="available"),
SimpleLink(
pred="creation_time",
obj=datetime.datetime(2019,
8,
23,
15,
59,
46,
tzinfo=tzutc()),
),
SimpleLink(pred="subnet_id", obj="subnet-07697f82fe4c6a8d6"),
SimpleLink(pred="subnet_id", obj="subnet-0396137c18d6c30ef"),
SimpleLink(pred="dns_support", obj="enable"),
SimpleLink(pred="ipv6_support", obj="disable"),
), ),
)
self.assertEqual(resource, expected_resource)
def test(self):
with tempfile.TemporaryDirectory() as temp_dir:
resource_region_name = "us-east-1"
# get moto"s enabled regions
ec2_client = boto3.client("ec2", region_name=resource_region_name)
all_regions = ec2_client.describe_regions(
Filters=[{
"Name": "opt-in-status",
"Values": ["opt-in-not-required", "opted-in"]
}])["Regions"]
account_id = get_account_id()
all_region_names = tuple(region["RegionName"]
for region in all_regions)
enabled_region_names = tuple(
region["RegionName"] for region in all_regions
if region["OptInStatus"] != "not-opted-in")
delete_vpcs(all_region_names)
# add a diverse set of resources which are supported by moto
## dynamodb
# TODO moto is not returning TableId in list/describe
# dynamodb_table_1_arn = create_dynamodb_table(
# name="test_table_1",
# attr_name="test_hash_key_attr_1",
# attr_type="S",
# key_type="HASH",
# region_name=region_name,
# )
## s3
bucket_1_name = "test_bucket"
bucket_1_arn, bucket_1_creation_date = create_bucket(
name=bucket_1_name,
account_id=account_id,
region_name=resource_region_name)
## ec2
vpc_1_cidr = "10.0.0.0/16"
vpc_1_id = create_vpc(cidr_block=vpc_1_cidr,
region_name=resource_region_name)
vpc_1_arn = VPCResourceSpec.generate_arn(
resource_id=vpc_1_id,
account_id=account_id,
region=resource_region_name)
subnet_1_cidr = "10.0.0.0/24"
subnet_1_cidr_network = ipaddress.IPv4Network(subnet_1_cidr,
strict=False)
subnet_1_first_ip, subnet_1_last_ip = (
int(subnet_1_cidr_network[0]),
int(subnet_1_cidr_network[-1]),
)
subnet_1_id = create_subnet(cidr_block=subnet_1_cidr,
vpc_id=vpc_1_id,
region_name=resource_region_name)
subnet_1_arn = SubnetResourceSpec.generate_arn(
resource_id=subnet_1_id,
account_id=account_id,
region=resource_region_name)
fixed_bucket_1_arn = f"arn:aws:s3:::{bucket_1_name}"
flow_log_1_id, flow_log_1_creation_time = create_flow_log(
vpc_id=vpc_1_id,
dest_bucket_arn=fixed_bucket_1_arn,
region_name=resource_region_name,
)
flow_log_1_arn = FlowLogResourceSpec.generate_arn(
resource_id=flow_log_1_id,
account_id=account_id,
region=resource_region_name)
ebs_volume_1_size = 128
ebs_volume_1_az = f"{resource_region_name}a"
ebs_volume_1_arn, ebs_volume_1_create_time = create_volume(
size=ebs_volume_1_size,
az=ebs_volume_1_az,
region_name=resource_region_name)
## iam
policy_1_name = "test_policy_1"
policy_1_arn, policy_1_id = create_iam_policy(
name=policy_1_name,
policy_doc={
"Version":
"2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "logs:CreateLogGroup",
"Resource": "*"
},
],
},
)
role_1_name = "test_role_1"
role_1_assume_role_policy_doc = {
"Version":
"2012-10-17",
"Statement": [{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "lambda.amazonaws.com"
},
"Sid": "",
}],
}
role_1_description = "Test Role 1"
role_1_max_session_duration = 3600
role_1_arn = create_iam_role(
name=role_1_name,
assume_role_policy_doc=role_1_assume_role_policy_doc,
description=role_1_description,
max_session_duration=role_1_max_session_duration,
)
## lambda
lambda_function_1_name = "test_lambda_function_1"
lambda_function_1_runtime = "python3.7"
lambda_function_1_handler = "lambda_function.lambda_handler"
lambda_function_1_description = "Test Lambda Function 1"
lambda_function_1_timeout = 30
lambda_function_1_memory_size = 256
lambda_function_1_arn = create_lambda_function(
name=lambda_function_1_name,
runtime=lambda_function_1_runtime,
role_name=role_1_arn,
handler=lambda_function_1_handler,
description=lambda_function_1_description,
timeout=lambda_function_1_timeout,
memory_size=lambda_function_1_memory_size,
publish=False,
region_name=resource_region_name,
)
# scan
test_scan_id = "test_scan_id"
aws_config = AWSConfig(
artifact_path=temp_dir,
pruner_max_age_min=4320,
graph_name="alti",
concurrency=ConcurrencyConfig(max_account_scan_threads=1,
max_svc_scan_threads=1,
max_account_scan_tries=2),
scan=ScanConfig(
accounts=(),
regions=(),
scan_sub_accounts=False,
preferred_account_scan_regions=(
"us-west-1",
"us-west-2",
"us-east-1",
"us-east-2",
),
),
accessor=Accessor(
credentials_cache=AWSCredentialsCache(cache={}),
multi_hop_accessors=[],
cache_creds=True,
),
write_master_json=True,
)
resource_spec_classes = (
# DynamoDbTableResourceSpec, TODO moto
EBSVolumeResourceSpec,
FlowLogResourceSpec,
IAMPolicyResourceSpec,
IAMRoleResourceSpec,
LambdaFunctionResourceSpec,
S3BucketResourceSpec,
SubnetResourceSpec,
VPCResourceSpec,
)
muxer = LocalAWSScanMuxer(
scan_id=test_scan_id,
config=aws_config,
resource_spec_classes=resource_spec_classes,
)
with unittest.mock.patch(
"altimeter.aws.scan.account_scanner.get_all_enabled_regions"
) as mock_get_all_enabled_regions:
mock_get_all_enabled_regions.return_value = enabled_region_names
aws2n_result = aws2n(
scan_id=test_scan_id,
config=aws_config,
muxer=muxer,
load_neptune=False,
)
graph_set = GraphSet.from_json_file(
Path(aws2n_result.json_path))
self.assertEqual(len(graph_set.errors), 0)
self.assertEqual(graph_set.name, "alti")
self.assertEqual(graph_set.version, "2")
# now check each resource type
self.maxDiff = None
## Accounts
expected_account_resources = [
Resource(
resource_id=f"arn:aws::::account/{account_id}",
type="aws:account",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="account_id",
obj=account_id), ), ),
)
]
account_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:account"
]
self.assertCountEqual(account_resources,
expected_account_resources)
## Regions
expected_region_resources = [
Resource(
resource_id=
f"arn:aws:::{account_id}:region/{region['RegionName']}",
type="aws:region",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="name",
obj=region["RegionName"]),
SimpleLink(pred="opt_in_status",
obj=region["OptInStatus"]),
),
resource_links=(ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"), ),
),
) for region in all_regions
]
region_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:region"
]
self.assertCountEqual(region_resources,
expected_region_resources)
## IAM Policies
expected_iam_policy_resources = [
Resource(
resource_id=policy_1_arn,
type="aws:iam:policy",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj=policy_1_name),
SimpleLink(pred="policy_id", obj=policy_1_id),
SimpleLink(pred="default_version_id",
obj="v1"),
SimpleLink(
pred="default_version_policy_document_text",
obj=
'{"Statement": [{"Action": "logs:CreateLogGroup", "Effect": "Allow", "Resource": "*"}], "Version": "2012-10-17"}',
),
),
resource_links=(ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"), ),
),
)
]
iam_policy_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:iam:policy"
]
self.assertCountEqual(iam_policy_resources,
expected_iam_policy_resources)
## IAM Roles
expected_iam_role_resources = [
Resource(
resource_id=role_1_arn,
type="aws:iam:role",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj=role_1_name),
SimpleLink(pred="max_session_duration",
obj=role_1_max_session_duration),
SimpleLink(pred="description",
obj=role_1_description),
SimpleLink(
pred="assume_role_policy_document_text",
obj=policy_doc_dict_to_sorted_str(
role_1_assume_role_policy_doc),
),
),
multi_links=(MultiLink(
pred="assume_role_policy_document",
obj=LinkCollection(
simple_links=(SimpleLink(
pred="version", obj="2012-10-17"), ),
multi_links=(MultiLink(
pred="statement",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="effect",
obj="Allow"),
SimpleLink(
pred="action",
obj="sts:AssumeRole"),
),
multi_links=(MultiLink(
pred="principal",
obj=LinkCollection(
simple_links=(SimpleLink(
pred="service",
obj=
"lambda.amazonaws.com",
), )),
), ),
),
), ),
),
), ),
resource_links=(ResourceLink(
pred="account",
obj="arn:aws::::account/123456789012"), ),
),
)
]
iam_role_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:iam:role"
]
self.assertCountEqual(iam_role_resources,
expected_iam_role_resources)
## Lambda functions
expected_lambda_function_resources = [
Resource(
resource_id=lambda_function_1_arn,
type="aws:lambda:function",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="function_name",
obj=lambda_function_1_name),
SimpleLink(pred="runtime",
obj=lambda_function_1_runtime),
),
resource_links=(
ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"),
ResourceLink(
pred="region",
obj=
f"arn:aws:::{account_id}:region/{resource_region_name}",
),
),
transient_resource_links=(ResourceLink(
pred="role",
obj="arn:aws:iam::123456789012:role/test_role_1"
), ),
),
),
]
lambda_function_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:lambda:function"
]
self.assertCountEqual(lambda_function_resources,
expected_lambda_function_resources)
## EC2 VPCs
expected_ec2_vpc_resources = [
Resource(
resource_id=vpc_1_arn,
type="aws:ec2:vpc",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="is_default", obj=True),
SimpleLink(pred="cidr_block", obj=vpc_1_cidr),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"),
ResourceLink(
pred="region",
obj=
f"arn:aws:::{account_id}:region/{resource_region_name}",
),
),
),
)
]
ec2_vpc_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:ec2:vpc"
]
self.assertCountEqual(ec2_vpc_resources,
expected_ec2_vpc_resources)
## EC2 VPC Flow Logs
expected_ec2_vpc_flow_log_resources = [
Resource(
resource_id=flow_log_1_arn,
type="aws:ec2:flow-log",
link_collection=LinkCollection(
simple_links=(
SimpleLink(
pred="creation_time",
obj=flow_log_1_creation_time.replace(
tzinfo=datetime.timezone.utc).
isoformat(),
),
SimpleLink(pred="deliver_logs_status",
obj="SUCCESS"),
SimpleLink(pred="flow_log_status",
obj="ACTIVE"),
SimpleLink(pred="traffic_type", obj="ALL"),
SimpleLink(pred="log_destination_type",
obj="s3"),
SimpleLink(pred="log_destination",
obj=fixed_bucket_1_arn),
SimpleLink(
pred="log_format",
obj=
"${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}",
),
),
resource_links=(
ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"),
ResourceLink(
pred="region",
obj=
f"arn:aws:::{account_id}:region/{resource_region_name}",
),
),
transient_resource_links=(TransientResourceLink(
pred="vpc",
obj=vpc_1_arn,
), ),
),
)
]
ec2_vpc_flow_log_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:ec2:flow-log"
]
self.assertCountEqual(ec2_vpc_flow_log_resources,
expected_ec2_vpc_flow_log_resources)
## EC2 Subnets
expected_ec2_subnet_resources = [
Resource(
resource_id=subnet_1_arn,
type="aws:ec2:subnet",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="cidr_block",
obj=subnet_1_cidr),
SimpleLink(pred="first_ip",
obj=subnet_1_first_ip),
SimpleLink(pred="last_ip",
obj=subnet_1_last_ip),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(pred="vpc", obj=vpc_1_arn),
ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"),
ResourceLink(
pred="region",
obj=
f"arn:aws:::{account_id}:region/{resource_region_name}",
),
),
),
)
]
ec2_subnet_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:ec2:subnet"
]
self.assertCountEqual(ec2_subnet_resources,
expected_ec2_subnet_resources)
## EC2 EBS Volumes
expected_ec2_ebs_volume_resources = [
Resource(
resource_id=ebs_volume_1_arn,
type="aws:ec2:volume",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="availability_zone",
obj=ebs_volume_1_az),
SimpleLink(
pred="create_time",
obj=ebs_volume_1_create_time.replace(
tzinfo=datetime.timezone.utc).
isoformat(),
),
SimpleLink(pred="size", obj=ebs_volume_1_size),
SimpleLink(pred="state", obj="available"),
SimpleLink(pred="volume_type", obj="gp2"),
SimpleLink(pred="encrypted", obj=False),
),
resource_links=(
ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"),
ResourceLink(
pred="region",
obj=
f"arn:aws:::{account_id}:region/{resource_region_name}",
),
),
),
)
]
ec2_ebs_volume_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:ec2:volume"
]
self.assertCountEqual(ec2_ebs_volume_resources,
expected_ec2_ebs_volume_resources)
## S3 Buckets
expected_s3_bucket_resources = [
Resource(
resource_id=bucket_1_arn,
type="aws:s3:bucket",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj=bucket_1_name),
SimpleLink(
pred="creation_date",
obj=bucket_1_creation_date.replace(
tzinfo=datetime.timezone.utc).
isoformat(),
),
),
resource_links=(
ResourceLink(
pred="account",
obj=f"arn:aws::::account/{account_id}"),
ResourceLink(
pred="region",
obj=
f"arn:aws:::{account_id}:region/{resource_region_name}",
),
),
),
)
]
s3_bucket_resources = [
resource for resource in graph_set.resources
if resource.type == "aws:s3:bucket"
]
self.assertCountEqual(s3_bucket_resources,
expected_s3_bucket_resources)
expected_num_graph_set_resources = (
0 + len(expected_account_resources) +
len(expected_region_resources) +
len(expected_iam_policy_resources) +
len(expected_iam_role_resources) +
len(expected_lambda_function_resources) +
len(expected_ec2_ebs_volume_resources) +
len(expected_ec2_subnet_resources) +
len(expected_ec2_vpc_resources) +
len(expected_ec2_vpc_flow_log_resources) +
len(expected_s3_bucket_resources))
self.assertEqual(len(graph_set.resources),
expected_num_graph_set_resources)
def test_schema_parse(self):
resource_arn = "arn:aws:ec2:us-west-2:111122223333:vpc-endpoint-service/com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01"
aws_resource_dict = {
"ServiceType": [{
"ServiceType": "Interface"
}],
"ServiceId":
"vpce-svc-01234abcd5678ef01",
"ServiceName":
"com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01",
"ServiceState":
"Available",
"AvailabilityZones": ["us-west-2a", "us-west-2b"],
"AcceptanceRequired":
True,
"ManagesVpcEndpoints":
False,
"NetworkLoadBalancerArns": [
"arn:aws:elasticloadbalancing:us-west-2:111122223333:loadbalancer/net/splunk-hwf-lb/1a7ff9c18eeaaf9b"
],
"BaseEndpointDnsNames":
["vpce-svc-01234abcd5678ef01.us-west-2.vpce.amazonaws.com"],
"PrivateDnsNameConfiguration": {},
"Tags": [{
"Key": "Name",
"Value": "Splunk HEC"
}],
}
links = VpcEndpointServiceResourceSpec.schema.parse(
data=aws_resource_dict,
context={
"account_id": "111122223333",
"region": "us-west-2"
})
resource = Resource(
resource_id=resource_arn,
type_name=VpcEndpointServiceResourceSpec.type_name,
links=links,
)
alti_resource_dict = resource.to_dict()
expected_alti_resource_dict = {
"type":
"vpc-endpoint-service",
"links": [
{
"pred": "service_type",
"obj": "Interface",
"type": "simple"
},
{
"pred": "service_name",
"obj":
"com.amazonaws.vpce.us-west-2.vpce-svc-01234abcd5678ef01",
"type": "simple",
},
{
"pred": "service_state",
"obj": "Available",
"type": "simple"
},
{
"pred": "acceptance_required",
"obj": True,
"type": "simple"
},
{
"pred": "availability_zones",
"obj": "us-west-2a",
"type": "simple"
},
{
"pred": "availability_zones",
"obj": "us-west-2b",
"type": "simple"
},
{
"pred": "Name",
"obj": "Splunk HEC",
"type": "tag"
},
],
}
self.assertDictEqual(alti_resource_dict, expected_alti_resource_dict)
def test_valid_merge(self):
resource_a1 = Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]),
)
resource_a2 = Resource(resource_id="456", type="test:a", link_collection=LinkCollection())
resource_b1 = Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-a", obj="123")]),
)
resource_b2 = Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="name", obj="sue")]),
)
graph_set_1 = GraphSet(
name="graph-1",
version="1",
start_time=10,
end_time=20,
resources=[resource_a1, resource_a2],
errors=["errora1", "errora2"],
)
graph_set_2 = GraphSet(
name="graph-1",
version="1",
start_time=15,
end_time=25,
resources=[resource_b1, resource_b2],
errors=["errorb1", "errorb2"],
)
merged_graph_set = ValidatedGraphSet.from_graph_sets([graph_set_1, graph_set_2])
self.assertEqual(merged_graph_set.name, "graph-1")
self.assertEqual(merged_graph_set.version, "1")
self.assertEqual(merged_graph_set.start_time, 10)
self.assertEqual(merged_graph_set.end_time, 25)
self.assertCountEqual(merged_graph_set.errors, ["errora1", "errora2", "errorb1", "errorb2"])
expected_resources = (
Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-foo", obj="goo"),),
),
),
Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),),
Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-a", obj="123"),),
),
),
Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=(SimpleLink(pred="name", obj="sue"),),),
),
)
expected_errors = ["errora1", "errora2", "errorb1", "errorb2"]
self.assertCountEqual(merged_graph_set.resources, expected_resources)
self.assertCountEqual(merged_graph_set.errors, expected_errors)