def testToJson(self):
pred = "test-pred"
obj = "test-obj"
link = SimpleLink(pred=pred, obj=obj)
expected_link_dict = {"pred": pred, "obj": obj}
link_dict = link.dict()
self.assertDictEqual(expected_link_dict, link_dict)
def testToJson(self):
pred = "test-multi-pred"
obj = LinkCollection(simple_links=(
SimpleLink(pred="test-simple-pred-1", obj="test-simple-obj-1"),
SimpleLink(pred="test-simple-pred-1", obj="test-simple-obj-2"),
SimpleLink(pred="test-simple-pred-2", obj="test-simple-obj-3"),
), )
link = MultiLink(pred=pred, obj=obj)
link_dict = link.dict(exclude_unset=True)
expected_link_dict = {
"pred": "test-multi-pred",
"obj": {
"simple_links": (
{
"pred": "test-simple-pred-1",
"obj": "test-simple-obj-1"
},
{
"pred": "test-simple-pred-1",
"obj": "test-simple-obj-2"
},
{
"pred": "test-simple-pred-2",
"obj": "test-simple-obj-3"
},
),
},
}
self.assertDictEqual(expected_link_dict, link_dict)
def testToRdf(self):
pred = "test-multi-pred"
obj = LinkCollection(simple_links=(
SimpleLink(pred="test-simple-pred-1", obj="test-simple-obj-1"),
SimpleLink(pred="test-simple-pred-2", obj="test-simple-obj-2"),
SimpleLink(pred="test-simple-pred-3", obj="test-simple-obj-3"),
), )
link = MultiLink(pred=pred, obj=obj)
bnode = BNode()
graph = Graph()
namespace = Namespace("test:")
node_cache = NodeCache()
link.to_rdf(subj=bnode,
namespace=namespace,
graph=graph,
node_cache=node_cache)
results = graph.query(
"select ?p ?o where {?s a <test:test-multi-pred> ; ?p ?o} order by ?p ?o"
)
result_tuples = []
for result in results:
self.assertEqual(2, len(result))
result_tuples.append((str(result[0]), str(result[1])))
expected_result_tuples = [
("http://www.w3.org/1999/02/22-rdf-syntax-ns#type",
"test:test-multi-pred"),
("test:test-simple-pred-1", "test-simple-obj-1"),
("test:test-simple-pred-2", "test-simple-obj-2"),
("test:test-simple-pred-3", "test-simple-obj-3"),
]
self.assertEqual(result_tuples, expected_result_tuples)
def test_valid_dicts_input_with_alti_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = ListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person"
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Bob"),
SimpleLink(pred="age", obj=49),
),
),
),
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Sue"),
SimpleLink(pred="age", obj=42),
),
),
),
),
)
self.assertEqual(link_collection, expected_link_collection)
def parse(self, data: Dict[str, Any],
context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a LinkCollection
Args:
data: dictionary of data to parse
context: context dict containing data from higher level parsing code.
Returns:
LinkCollection
"""
value = data.get(self.source_key)
if value is None:
if self.default_value is not None:
return LinkCollection(simple_links=[
SimpleLink(pred=self.alti_key, obj=self.default_value)
], )
if self.optional:
return LinkCollection()
raise ScalarFieldSourceKeyNotFoundException(
f"Expected key '{self.source_key}' in data, present keys: {', '.join(data.keys())}"
)
if isinstance(value, SCALAR_TYPES):
return LinkCollection(
simple_links=[SimpleLink(pred=self.alti_key, obj=value)], )
raise ScalarFieldValueNotAScalarException(
(f"Expected data for key '{self.source_key}' to be one "
f"of {SCALAR_TYPES}, is {type(value)}: {value}"))
def test_graph_content(self):
expected_resources = (
Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-foo", obj="goo"),),
),
),
Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),),
Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-a", obj="123"),),
),
),
Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=(SimpleLink(pred="name", obj="sue"),),),
),
)
expected_errors = ["test err 1", "test err 2"]
self.assertEqual(self.validated_graph_set.resources, expected_resources)
self.assertEqual(self.validated_graph_set.errors, expected_errors)
def setUp(self):
resource_a1 = Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]),
)
resource_a2 = Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),)
resource_b1 = Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-a", obj="123")]),
)
resource_b2 = Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="name", obj="sue")]),
)
resources = (resource_a1, resource_a2, resource_b1, resource_b2)
self.validated_graph_set = ValidatedGraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=["test err 1", "test err 2"],
)
def test_parse(self):
schema = Schema(ScalarField("Key1"), ScalarField("Key2"))
data = {"Key1": "Value1", "Key2": "Value2"}
link_collection = schema.parse(data, {})
expected_link_collection = LinkCollection(simple_links=(
SimpleLink(pred="key1", obj="Value1"),
SimpleLink(pred="key2", obj="Value2"),
))
self.assertEqual(link_collection, expected_link_collection)
def testToLpgLargeInt(self):
pred = "test-pred"
obj = 9223372036854775808
link = SimpleLink(pred=pred, obj=obj)
expected_link_dict = {"test-pred": "9223372036854775808"}
parent = {}
vertices = []
edges = []
link.to_lpg(parent, vertices, edges, "")
self.assertDictEqual(expected_link_dict, parent)
def testToLpgSmallInt(self):
pred = "test-pred"
obj = 1
link = SimpleLink(pred=pred, obj=obj)
expected_link_dict = {"test-pred": 1}
parent = {}
vertices = []
edges = []
link.to_lpg(parent, vertices, edges, "")
self.assertDictEqual(expected_link_dict, parent)
def testToLpgPrefix(self):
pred = "test-pred"
obj = "test-obj"
link = SimpleLink(pred=pred, obj=obj)
expected_link_dict = {"abctest-pred": "test-obj"}
parent = {}
vertices = []
edges = []
link.to_lpg(parent, vertices, edges, "abc")
self.assertDictEqual(expected_link_dict, parent)
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
client = session.client("iam")
oidc_url = "https://oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E"
oidc_client_ids = ["sts.amazonaws.com"]
oidc_thumbprints = ["9999999999999999999999999999999999999999"]
_ = client.create_open_id_connect_provider(
Url=oidc_url,
ClientIDList=oidc_client_ids,
ThumbprintList=oidc_thumbprints,
)
scan_accessor = AWSAccessor(session=session,
account_id=account_id,
region_name=region_name)
resources = IAMOIDCProviderResourceSpec.scan(
scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id=
"arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E",
type="aws:iam:oidc-provider",
link_collection=LinkCollection(
simple_links=(
SimpleLink(
pred="url",
obj=
"oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E",
),
SimpleLink(
pred="create_date",
obj=resources[0].link_collection.simple_links[1].
obj,
),
SimpleLink(pred="client_id", obj="sts.amazonaws.com"),
SimpleLink(
pred="thumbprint",
obj="9999999999999999999999999999999999999999"),
),
multi_links=None,
tag_links=None,
resource_links=(ResourceLink(
pred="account",
obj="arn:aws::::account/123456789012"), ),
transient_resource_links=None,
),
)
]
self.assertEqual(resources, expected_resources)
def create_resource(
cls: Type["UnscannedAccountResourceSpec"], account_id: str, errors: List[str]
) -> Resource:
simple_links: List[SimpleLink] = []
simple_links.append(SimpleLink(pred="account_id", obj=account_id))
if errors:
error = "\n".join(errors)
simple_links.append(SimpleLink(pred="error", obj=f"{error} - {uuid.uuid4()}"))
return Resource(
resource_id=cls.generate_arn(resource_id=account_id),
type=cls.get_full_type_name(),
link_collection=LinkCollection(simple_links=simple_links),
)
def test_valid_strings_input(self):
input_str = '{"Animals": ["cow", "pig", "human"]}'
field = ListField("Animals", EmbeddedScalarField())
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
simple_links=(
SimpleLink(pred="animals", obj="cow"),
SimpleLink(pred="animals", obj="pig"),
SimpleLink(pred="animals", obj="human"),
),
)
self.assertEqual(link_collection, expected_link_collection)
def test_schema_parse(self):
resource_arn = "arn:aws:iam:us-west-2:111122223333:account-password-policy/default"
aws_resource_dict = {
"MinimumPasswordLength": 12,
"RequireSymbols": True,
"RequireNumbers": True,
"RequireUppercaseCharacters": True,
"RequireLowercaseCharacters": True,
"AllowUsersToChangePassword": True,
"ExpirePasswords": True,
"MaxPasswordAge": 90,
"PasswordReusePrevention": 5,
"HardExpiry": True,
}
link_collection = IAMAccountPasswordPolicyResourceSpec.schema.parse(
data=aws_resource_dict,
context={
"account_id": "111122223333",
"region": "us-west-2"
})
resource = Resource(
resource_id=resource_arn,
type=IAMAccountPasswordPolicyResourceSpec.type_name,
link_collection=link_collection,
)
expected_resource = Resource(
resource_id=
"arn:aws:iam:us-west-2:111122223333:account-password-policy/default",
type="account-password-policy",
link_collection=LinkCollection(simple_links=(
SimpleLink(pred="minimum_password_length", obj=12),
SimpleLink(pred="require_symbols", obj=True),
SimpleLink(pred="require_numbers", obj=True),
SimpleLink(pred="require_uppercase_characters", obj=True),
SimpleLink(pred="require_lowercase_characters", obj=True),
SimpleLink(pred="allow_users_to_change_password", obj=True),
SimpleLink(pred="expire_passwords", obj=True),
SimpleLink(pred="max_password_age", obj=90),
SimpleLink(pred="password_reuse_prevention", obj=5),
SimpleLink(pred="hard_expiry", obj=True),
)),
)
self.assertEqual(resource, expected_resource)
def testToRdfSmallInt(self):
pred = "test-pred"
obj = 20
link = SimpleLink(pred=pred, obj=obj)
bnode = BNode()
graph = Graph()
namespace = Namespace("test:")
node_cache = NodeCache()
link.to_rdf(subj=bnode,
namespace=namespace,
graph=graph,
node_cache=node_cache)
results = graph.query("select ?s ?p ?o where {?s ?p ?o}")
self.assertEqual(1, len(results))
for result in results:
_, _, o = result
self.assertEqual(
o.datatype, URIRef("http://www.w3.org/2001/XMLSchema#integer"))
def test_valid_dicts_input(self):
input_str = (
'{"Biota": {"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}}'
)
field = DictField(
"Biota",
AnonymousListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))
),
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Bob"),
SimpleLink(pred="age", obj=49),
),
),
),
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Sue"),
SimpleLink(pred="age", obj=42),
),
),
),
),
),
),
)
)
self.assertEqual(link_collection, expected_link_collection)
def testToRdf(self):
pred = "test-pred"
obj = "test-obj"
link = SimpleLink(pred=pred, obj=obj)
bnode = BNode()
graph = Graph()
namespace = Namespace("test:")
node_cache = NodeCache()
link.to_rdf(subj=bnode,
namespace=namespace,
graph=graph,
node_cache=node_cache)
results = graph.query("select ?s ?p ?o where {?s ?p ?o}")
self.assertEqual(1, len(results))
for result in results:
s, p, o = result
self.assertEqual(s, bnode)
self.assertEqual(str(p), "test:test-pred")
self.assertEqual(str(o), "test-obj")
def test_key_absent_with_default(self):
input_str = "{}"
field = ScalarField("FieldName", default_value="DefaultValue")
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="field_name", obj="DefaultValue"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_key_present_with_optional(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="field_name", obj="Value"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_valid_input_with_alti_key(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", alti_key="alti_field_name")
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="alti_field_name", obj="Value"), ), )
self.assertEqual(link_collection, expected_link_collection)
def testToLpg(self):
pred = "test-multi-pred"
obj = LinkCollection(simple_links=(
SimpleLink(pred="test-simple-pred-1", obj="test-simple-obj-1"),
SimpleLink(pred="test-simple-pred-2", obj="test-simple-obj-2"),
SimpleLink(pred="test-simple-pred-3", obj="test-simple-obj-3"),
), )
link = MultiLink(pred=pred, obj=obj)
parent = {"~id": "parent_id"}
vertices = []
edges = []
link.to_lpg(parent, vertices, edges, "")
expected_vertex = {
"~label": "test-multi-pred",
"test-simple-pred-1": "test-simple-obj-1",
"test-simple-pred-2": "test-simple-obj-2",
"test-simple-pred-3": "test-simple-obj-3",
}
self.assertEqual(len(vertices), 1)
self.assertIsInstance(vertices[0]["~id"], uuid.UUID)
vertex_id = vertices[0]["~id"]
del vertices[0]["~id"]
self.assertDictEqual(expected_vertex, vertices[0])
expected_edge = {
"~label": "test-multi-pred",
"~from": "parent_id",
"~to": vertex_id,
}
self.assertEqual(len(edges), 1)
self.assertIsInstance(edges[0]["~id"], uuid.UUID)
del edges[0]["~id"]
self.assertDictEqual(expected_edge, edges[0])
vertices = []
edges = []
link.to_lpg(parent, vertices, edges, "test_prefix")
self.assertEqual(len(edges), 1)
self.assertEqual("test_prefix", edges[0]["~label"])
def test_allow_scalar(self):
input_str = '{"People": "bob"}'
field = ListField("People", EmbeddedScalarField(), alti_key="person", allow_scalar=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
simple_links=(SimpleLink(pred="person", obj="bob"),),
)
self.assertEqual(link_collection, expected_link_collection)
def test_valid_input(self):
input_data = "foo"
parent_alti_key = "parent_alti_key"
field = EmbeddedScalarField()
link_collection = field.parse(
data=input_data, context={"parent_alti_key": parent_alti_key})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="parent_alti_key", obj="foo"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_valid_strings_input(self):
input_str = '{"Biota": {"Animals": ["cow", "pig", "human"], "Plants": ["tree", "fern"]}}'
field = DictField("Biota", AnonymousListField("Animals", EmbeddedScalarField()))
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="biota", obj="cow"),
SimpleLink(pred="biota", obj="pig"),
SimpleLink(pred="biota", obj="human"),
)
),
),
)
)
self.assertEqual(link_collection, expected_link_collection)
def test_allow_scalar(self):
input_str = '{"Biota": {"Plants": "tree"}}'
field = DictField(
"Biota", AnonymousListField("Plants", EmbeddedScalarField(), allow_scalar=True)
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(simple_links=(SimpleLink(pred="biota", obj="tree"),),),
),
)
)
self.assertEqual(expected_link_collection, link_collection)
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
ec2_client = session.client("ec2", region_name=region_name)
list_resp = ec2_client.describe_vpcs()
present_vpcs = list_resp["Vpcs"]
self.assertEqual(len(present_vpcs), 1)
present_vpc_id = present_vpcs[0]["VpcId"]
present_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{present_vpc_id}"
create_resp = ec2_client.create_vpc(CidrBlock="10.0.0.0/16")
created_vpc_id = create_resp["Vpc"]["VpcId"]
created_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{created_vpc_id}"
scan_accessor = AWSAccessor(session=session, account_id=account_id, region_name=region_name)
resources = VPCResourceSpec.scan(scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id=present_vpc_arn,
type="aws:ec2:vpc",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="is_default", obj=True),
SimpleLink(pred="cidr_block", obj="172.31.0.0/16"),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(pred="account", obj="arn:aws::::account/123456789012"),
ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"),
),
),
),
Resource(
resource_id=created_vpc_arn,
type="aws:ec2:vpc",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="is_default", obj=False),
SimpleLink(pred="cidr_block", obj="10.0.0.0/16"),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(pred="account", obj="arn:aws::::account/123456789012"),
ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"),
),
),
),
]
self.assertEqual(resources, expected_resources)
def parse(self, data: Union[str, bool, int, float],
context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a LinkCollection.
Args:
data: scalar value
context: context dict containing data from higher level parsing code.
Returns:
LinkCollection
"""
parent_alti_key = self.get_parent_alti_key(data, context)
if isinstance(data, SCALAR_TYPES):
link = SimpleLink(pred=parent_alti_key, obj=data)
return LinkCollection(simple_links=[link], )
raise ScalarFieldValueNotAScalarException(
(f"Expected data to be one of {SCALAR_TYPES}, is "
f"{type(data)}: {data}"))
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
scan_accessor = AWSAccessor(session=session,
account_id=account_id,
region_name=region_name)
resources = AccountResourceSpec.scan(scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id="arn:aws::::account/123456789012",
type="aws:account",
link_collection=LinkCollection(simple_links=(SimpleLink(
pred="account_id", obj="123456789012"), ), ),
)
]
self.assertEqual(resources, expected_resources)
def test(self):
account_id = "012345678901"
errors = ["foo", "boo"]
unscanned_account_resource = UnscannedAccountResourceSpec.create_resource(
account_id=account_id, errors=errors)
resource = ResourceSpec.merge_resources("foo",
[unscanned_account_resource])
self.assertEqual(resource.resource_id, "foo")
self.assertEqual(resource.type, "aws:unscanned-account")
self.assertEqual(len(resource.link_collection.simple_links), 2)
self.assertEqual(
resource.link_collection.simple_links[0],
SimpleLink(pred="account_id", obj="012345678901"),
)
self.assertEqual(resource.link_collection.simple_links[1].pred,
"error")
self.assertTrue(
resource.link_collection.simple_links[1].obj.startswith(
"foo\nboo - "))
