def parse(self, data: Dict[str, Any],
context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a LinkCollection
Args:
data: dictionary of data to parse
context: context dict containing data from higher level parsing code.
Returns:
LinkCollection
"""
value = data.get(self.source_key)
if value is None:
if self.default_value is not None:
return LinkCollection(simple_links=[
SimpleLink(pred=self.alti_key, obj=self.default_value)
], )
if self.optional:
return LinkCollection()
raise ScalarFieldSourceKeyNotFoundException(
f"Expected key '{self.source_key}' in data, present keys: {', '.join(data.keys())}"
)
if isinstance(value, SCALAR_TYPES):
return LinkCollection(
simple_links=[SimpleLink(pred=self.alti_key, obj=value)], )
raise ScalarFieldValueNotAScalarException(
(f"Expected data for key '{self.source_key}' to be one "
f"of {SCALAR_TYPES}, is {type(value)}: {value}"))
def parse(self, data: Dict[str, Any],
context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a LinkCollection.
Args:
data: data to parse
context: contains data from higher level parsing code.
Returns:
LinkCollection
"""
if isinstance(self._resource_spec_class, str):
resource_spec_class: Type[
ResourceSpec] = ResourceSpec.get_by_class_name(
self._resource_spec_class)
else:
resource_spec_class = self._resource_spec_class
if not self.alti_key:
self.alti_key = resource_spec_class.type_name
short_resource_id = data.get(self.source_key)
if not short_resource_id:
if self.optional:
return LinkCollection()
raise ResourceLinkFieldSourceKeyNotFoundException(
f"Expected key '{self.source_key}' with non-empty/zero value in {data}"
)
if self.value_is_id:
resource_id = short_resource_id
else:
resource_id = resource_spec_class.generate_id(
short_resource_id, context)
return LinkCollection(transient_resource_links=[
TransientResourceLink(pred=self.alti_key, obj=resource_id)
], )
def parse(self, data: Dict[str, Any],
context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a LinkCollection
Args:
data: dictionary of data to parse
context: context dict containing data from higher level parsing code.
Returns:
LinkCollection
Raises:
ListFieldSourceKeyNotFoundException if self.source_key is not in data.
ListFieldValueNotAListException if the data does not appear to represent a list.
"""
if self.source_key not in data:
if self.optional:
return LinkCollection()
raise ListFieldSourceKeyNotFoundException(
f"Expected key '{self.source_key}' in data, present keys: {', '.join(data.keys())}"
)
sub_datas = data.get(self.source_key, [])
if not isinstance(sub_datas, list):
if self.allow_scalar and isinstance(sub_datas, SCALAR_TYPES):
sub_datas = [sub_datas]
else:
raise ListFieldValueNotAListException((
f"Key '{self.source_key}' value had unexpected type, value: {sub_datas} "
f"type: {type(sub_datas)}"))
link_collection = LinkCollection()
updated_context = deepcopy(context)
updated_context.update({"parent_alti_key": self.alti_key})
for sub_data in sub_datas:
link_collection += self.sub_field.parse(sub_data, updated_context)
return link_collection
def test_graph_content(self):
expected_resources = (
Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-foo", obj="goo"),),
),
),
Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),),
Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(
simple_links=(SimpleLink(pred="has-a", obj="123"),),
),
),
Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=(SimpleLink(pred="name", obj="sue"),),),
),
)
expected_errors = ["test err 1", "test err 2"]
self.assertEqual(self.validated_graph_set.resources, expected_resources)
self.assertEqual(self.validated_graph_set.errors, expected_errors)
def test_scan(self):
scan_accessor = TestScanAccessor()
graph_spec = GraphSpec(
name="test-name",
version="1",
resource_spec_classes=(TestResourceSpecA, TestResourceSpecB),
scan_accessor=scan_accessor,
)
resources = graph_spec.scan()
expected_resources = [
Resource(resource_id="123",
type="a",
link_collection=LinkCollection()),
Resource(resource_id="456",
type="a",
link_collection=LinkCollection()),
Resource(resource_id="abc",
type="b",
link_collection=LinkCollection()),
Resource(resource_id="def",
type="b",
link_collection=LinkCollection()),
]
self.assertEqual(resources, expected_resources)
def parse(self, data: Dict[str, Any],
context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a LinkCollection.
Args:
data: dictionary of data to parse
context: context dict containing data from higher level parsing code.
Returns:
LinkCollection
Raises:
DictFieldSourceKeyNotFoundException if self.source_key is not in data.
DictFieldValueNotADictException if the data does not appear to represent a dict.
"""
parent_alti_key = self.get_parent_alti_key(data, context)
if not isinstance(data, dict):
raise Exception(f"{type(data)} {data} was expected to be a dict.")
updated_context = deepcopy(context)
updated_context.update({"parent_alti_key": parent_alti_key})
multi_link_object_link_collection = LinkCollection()
for field in self.fields:
multi_link_object_link_collection += field.parse(data, context)
return LinkCollection(multi_links=[
MultiLink(pred=parent_alti_key,
obj=multi_link_object_link_collection)
])
def test_valid_dicts_input_with_alti_key(self):
input_str = '{"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}'
field = ListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age")), alti_key="person"
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Bob"),
SimpleLink(pred="age", obj=49),
),
),
),
MultiLink(
pred="person",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Sue"),
SimpleLink(pred="age", obj=42),
),
),
),
),
)
self.assertEqual(link_collection, expected_link_collection)
def setUp(self):
resource_a1 = Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]),
)
resource_a2 = Resource(resource_id="456", type="test:a", link_collection=LinkCollection(),)
resource_b1 = Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-a", obj="123")]),
)
resource_b2 = Resource(
resource_id="def",
type="test:b",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="name", obj="sue")]),
)
resources = (resource_a1, resource_a2, resource_b1, resource_b2)
self.validated_graph_set = ValidatedGraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=["test err 1", "test err 2"],
)
def scan(cls: Type["TestResourceSpecB"],
scan_accessor: Any) -> List[Resource]:
resources = [
Resource(resource_id="abc",
type=cls.type_name,
link_collection=LinkCollection()),
Resource(resource_id="def",
type=cls.type_name,
link_collection=LinkCollection()),
]
return resources
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
ec2_client = session.client("ec2", region_name=region_name)
list_resp = ec2_client.describe_vpcs()
present_vpcs = list_resp["Vpcs"]
self.assertEqual(len(present_vpcs), 1)
present_vpc_id = present_vpcs[0]["VpcId"]
present_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{present_vpc_id}"
create_resp = ec2_client.create_vpc(CidrBlock="10.0.0.0/16")
created_vpc_id = create_resp["Vpc"]["VpcId"]
created_vpc_arn = f"arn:aws:ec2:us-east-1:123456789012:vpc/{created_vpc_id}"
scan_accessor = AWSAccessor(session=session, account_id=account_id, region_name=region_name)
resources = VPCResourceSpec.scan(scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id=present_vpc_arn,
type="aws:ec2:vpc",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="is_default", obj=True),
SimpleLink(pred="cidr_block", obj="172.31.0.0/16"),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(pred="account", obj="arn:aws::::account/123456789012"),
ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"),
),
),
),
Resource(
resource_id=created_vpc_arn,
type="aws:ec2:vpc",
link_collection=LinkCollection(
simple_links=(
SimpleLink(pred="is_default", obj=False),
SimpleLink(pred="cidr_block", obj="10.0.0.0/16"),
SimpleLink(pred="state", obj="available"),
),
resource_links=(
ResourceLink(pred="account", obj="arn:aws::::account/123456789012"),
ResourceLink(pred="region", obj="arn:aws:::123456789012:region/us-east-1"),
),
),
),
]
self.assertEqual(resources, expected_resources)
def test_optional(self):
input_str = '{"Biota": {"Plants": ["tree", "fern"]}}'
field = DictField(
"Biota", AnonymousListField("Animals", EmbeddedScalarField(), optional=True)
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(MultiLink(pred="biota", obj=LinkCollection()),),
)
self.assertEqual(link_collection, expected_link_collection)
def test_unknown_type_name(self):
resources = [
Resource(resource_id="xyz", type="test:a", link_collection=LinkCollection()),
Resource(resource_id="xyz", type="test:c", link_collection=LinkCollection()),
]
with self.assertRaises(ResourceSpecClassNotFoundException):
ValidatedGraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=[],
)
def test_invalid_resources_dupes_same_class_conflicting_types_no_allow_clobber(self):
resources = [
Resource(resource_id="123", type="test:a", link_collection=LinkCollection()),
Resource(resource_id="123", type="test:b", link_collection=LinkCollection()),
]
with self.assertRaises(UnmergableDuplicateResourceIdsFoundException):
ValidatedGraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=[],
)
def testToJson(self):
pred = "test-multi-pred"
obj = LinkCollection(simple_links=(
SimpleLink(pred="test-simple-pred-1", obj="test-simple-obj-1"),
SimpleLink(pred="test-simple-pred-1", obj="test-simple-obj-2"),
SimpleLink(pred="test-simple-pred-2", obj="test-simple-obj-3"),
), )
link = MultiLink(pred=pred, obj=obj)
link_dict = link.dict(exclude_unset=True)
expected_link_dict = {
"pred": "test-multi-pred",
"obj": {
"simple_links": (
{
"pred": "test-simple-pred-1",
"obj": "test-simple-obj-1"
},
{
"pred": "test-simple-pred-1",
"obj": "test-simple-obj-2"
},
{
"pred": "test-simple-pred-2",
"obj": "test-simple-obj-3"
},
),
},
}
self.assertDictEqual(expected_link_dict, link_dict)
def parse(self, data: str, context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a LinkCollection.
Args:
data: data to parse
context: contains data from higher level parsing code.
Returns:
LinkCollection
"""
if isinstance(self._resource_spec_class, str):
resource_spec_class: Type[
ResourceSpec] = ResourceSpec.get_by_class_name(
self._resource_spec_class)
else:
resource_spec_class = self._resource_spec_class
if not self.alti_key:
self.alti_key = resource_spec_class.type_name
short_resource_id = data
if self.value_is_id:
resource_id = short_resource_id
else:
resource_id = resource_spec_class.generate_id(
short_resource_id, context)
return LinkCollection(
resource_links=[ResourceLink(pred=self.alti_key,
obj=resource_id)], )
def testToRdf(self):
pred = "test-multi-pred"
obj = LinkCollection(simple_links=(
SimpleLink(pred="test-simple-pred-1", obj="test-simple-obj-1"),
SimpleLink(pred="test-simple-pred-2", obj="test-simple-obj-2"),
SimpleLink(pred="test-simple-pred-3", obj="test-simple-obj-3"),
), )
link = MultiLink(pred=pred, obj=obj)
bnode = BNode()
graph = Graph()
namespace = Namespace("test:")
node_cache = NodeCache()
link.to_rdf(subj=bnode,
namespace=namespace,
graph=graph,
node_cache=node_cache)
results = graph.query(
"select ?p ?o where {?s a <test:test-multi-pred> ; ?p ?o} order by ?p ?o"
)
result_tuples = []
for result in results:
self.assertEqual(2, len(result))
result_tuples.append((str(result[0]), str(result[1])))
expected_result_tuples = [
("http://www.w3.org/1999/02/22-rdf-syntax-ns#type",
"test:test-multi-pred"),
("test:test-simple-pred-1", "test-simple-obj-1"),
("test:test-simple-pred-2", "test-simple-obj-2"),
("test:test-simple-pred-3", "test-simple-obj-3"),
]
self.assertEqual(result_tuples, expected_result_tuples)
def test_allow_scalar(self):
input_str = '{"Biota": {"Plants": "tree"}}'
field = DictField(
"Biota", AnonymousListField("Plants", EmbeddedScalarField(), allow_scalar=True)
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(simple_links=(SimpleLink(pred="biota", obj="tree"),),),
),
)
)
self.assertEqual(expected_link_collection, link_collection)
def test_optional(self):
input_str = '{"NoTagsHere": []}'
field = TagsField(optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
self.assertCountEqual(link_collection, LinkCollection())
def test_optional(self):
input_str = "{}"
field = ListField("People", EmbeddedScalarField(), alti_key="person", optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
self.assertEqual(link_collection, LinkCollection())
def test_key_absent_with_optional(self):
input_str = "{}"
field = ScalarField("FieldName", optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
self.assertEqual(link_collection, LinkCollection())
def test_parse(self):
schema = Schema(ScalarField("Key1"), ScalarField("Key2"))
data = {"Key1": "Value1", "Key2": "Value2"}
link_collection = schema.parse(data, {})
expected_link_collection = LinkCollection(simple_links=(
SimpleLink(pred="key1", obj="Value1"),
SimpleLink(pred="key2", obj="Value2"),
))
self.assertEqual(link_collection, expected_link_collection)
def test_valid_dicts_input(self):
input_str = (
'{"Biota": {"People": [{"Name": "Bob", "Age": 49}, {"Name": "Sue", "Age": 42}]}}'
)
field = DictField(
"Biota",
AnonymousListField(
"People", EmbeddedDictField(ScalarField("Name"), ScalarField("Age"))
),
)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
multi_links=(
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Bob"),
SimpleLink(pred="age", obj=49),
),
),
),
MultiLink(
pred="biota",
obj=LinkCollection(
simple_links=(
SimpleLink(pred="name", obj="Sue"),
SimpleLink(pred="age", obj=42),
),
),
),
),
),
),
)
)
self.assertEqual(link_collection, expected_link_collection)
def test_valid_input_with_alti_key(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", alti_key="alti_field_name")
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="alti_field_name", obj="Value"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_key_absent_with_default(self):
input_str = "{}"
field = ScalarField("FieldName", default_value="DefaultValue")
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="field_name", obj="DefaultValue"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_key_present_with_optional(self):
input_str = '{"FieldName": "Value"}'
field = ScalarField("FieldName", optional=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="field_name", obj="Value"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_scan(self):
account_id = "123456789012"
region_name = "us-east-1"
session = boto3.Session()
client = session.client("iam")
oidc_url = "https://oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E"
oidc_client_ids = ["sts.amazonaws.com"]
oidc_thumbprints = ["9999999999999999999999999999999999999999"]
_ = client.create_open_id_connect_provider(
Url=oidc_url,
ClientIDList=oidc_client_ids,
ThumbprintList=oidc_thumbprints,
)
scan_accessor = AWSAccessor(session=session,
account_id=account_id,
region_name=region_name)
resources = IAMOIDCProviderResourceSpec.scan(
scan_accessor=scan_accessor)
expected_resources = [
Resource(
resource_id=
"arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E",
type="aws:iam:oidc-provider",
link_collection=LinkCollection(
simple_links=(
SimpleLink(
pred="url",
obj=
"oidc.eks.us-east-1.amazonaws.com/id/EXAMPLED539D4633E53DE1B716D3041E",
),
SimpleLink(
pred="create_date",
obj=resources[0].link_collection.simple_links[1].
obj,
),
SimpleLink(pred="client_id", obj="sts.amazonaws.com"),
SimpleLink(
pred="thumbprint",
obj="9999999999999999999999999999999999999999"),
),
multi_links=None,
tag_links=None,
resource_links=(ResourceLink(
pred="account",
obj="arn:aws::::account/123456789012"), ),
transient_resource_links=None,
),
)
]
self.assertEqual(resources, expected_resources)
def test_valid_input(self):
input_data = "foo"
parent_alti_key = "parent_alti_key"
field = EmbeddedScalarField()
link_collection = field.parse(
data=input_data, context={"parent_alti_key": parent_alti_key})
expected_link_collection = LinkCollection(simple_links=(SimpleLink(
pred="parent_alti_key", obj="foo"), ), )
self.assertEqual(link_collection, expected_link_collection)
def test_allow_scalar(self):
input_str = '{"People": "bob"}'
field = ListField("People", EmbeddedScalarField(), alti_key="person", allow_scalar=True)
input_data = json.loads(input_str)
link_collection = field.parse(data=input_data, context={})
expected_link_collection = LinkCollection(
simple_links=(SimpleLink(pred="person", obj="bob"),),
)
self.assertEqual(link_collection, expected_link_collection)
def test_orphaned_ref(self):
resource_a1 = Resource(
resource_id="123",
type="test:a",
link_collection=LinkCollection(simple_links=[SimpleLink(pred="has-foo", obj="goo")]),
)
resource_b1 = Resource(
resource_id="abc",
type="test:b",
link_collection=LinkCollection(resource_links=[ResourceLink(pred="has-a", obj="456")]),
)
resources = [resource_a1, resource_b1]
graph_set = GraphSet(
name="test-name",
version="1",
start_time=1234,
end_time=4567,
resources=resources,
errors=["test err 1", "test err 2"],
)
with self.assertRaises(GraphSetOrphanedReferencesException):
ValidatedGraphSet.from_graph_set(graph_set)
def parse(self, data: Dict[str, Any],
context: Dict[str, Any]) -> LinkCollection:
"""Parse this field and return a list of Links.
Args:
data: dictionary of data to parse
context: context dict containing data from higher level parsing code.
Returns:
List of TagLink objects, one for each tag.
"""
links: List[TagLink] = []
tag_dicts = data.get("Tags")
if tag_dicts:
for tag_dict in tag_dicts:
links.append(
TagLink(pred=tag_dict["Key"], obj=tag_dict["Value"]))
return LinkCollection(tag_links=links)
if self.optional:
return LinkCollection()
raise TagsFieldMissingTagsKeyException(
f"Expected key 'Tags' in {data}")