def _load_associations(self):
if len(self.associates) == 0:
return
self._print(
f"[*] Adding {self.__class__.__name__} associative relationships")
edges = Elements()
for resource in self.get("Resource"):
references = {}
label = [l for l in resource.labels() if l != "Resource"][0]
# Find references to other resources in the form of a dictionary (refs)
self._references(resource.properties(), references)
# Create an edge, for all known associations (as defined by self.rels).
for rel in [
r for r in self.associates
if r[0] == label or r[1] == label
]:
i = 1 if label == rel[0] else 0
# Get a list of foreign keys that we must be capable of referencing
# in order to create an association
fk = [
a for a in re.compile("{([A-Za-z]+)}").findall(
RESOURCES.definition(rel[i]))
if a not in ["Account", "Region"]
]
if not all([k in references.keys() for k in fk]):
continue
# TODO: Handle Types that make use of more than one
# variable identifier
if len(fk) != 1:
raise NotImplementedError
fk = fk[0]
for v in list(references[fk]):
# Find the first resource matching the reference
r = next((r
for r in self if re.compile(
RESOURCES.definition(rel[i]).
format(Account=self.account_id,
Region=self.session.region_name,
**{
**{
x: list(y)[0]
for x, y in references.items(
) if len(y) == 1
},
**{
fk: v
}
})).match(r.id()) is not None), None)
if r is None:
# print("Failed to match (%s: %s) against any resources" % (k, v))
# print("Its likely that the resource was missed during ingestion")
continue
# Delete the properties that are responsible for the edge's existence.
properties = self._extract_property_value(
resource.properties(), fk)
# Even though direction is irrelavent when dealing with Associative
# edges, neo4j is directed. We need to ensure the direction is kept
# in order to eliminate duplicate edges.
(source, target) = (resource, r) if i == 1 else (r,
resource)
edge = Associative(properties={"Name": "Attached"},
source=source,
target=target)
opposite_edge = Associative(
properties={"Name": "Attached"},
source=target,
target=source)
if (edge not in self and opposite_edge
not in self) and edge not in edges:
edges.append(edge)
self.extend(edges)
def load_associatives(self):
if len(self.associations) == 0:
return
def set_references(references, item, key=None):
if isinstance(item, list):
[set_references(references, i) for i in item]
elif isinstance(item, dict):
[set_references(references, v, k) for k, v in item.items()]
elif (key is not None
and any([isinstance(item, t) for t in [str, int, bool]])
and len(str(item)) > 0):
if key not in references:
references[key] = set()
references[key].update([item])
for resource in self.console.tasklist(
f"Adding Associative relationships",
self.get("Resource"),
done="Added Associative relationships"):
# Extract reference key-value pairs from this resource's
# properties (if we need to)
prop_refs = {}
# Extract reference key-value pairs from this resource's ARN:
regex = re.compile(RESOURCES[resource.label()])
matches = regex.match(resource.id())
arn_refs = {
k: set([matches.group(k)])
for k in regex.groupindex.keys()
} if matches is not None else {}
# For each of the resource types associated with this resource type
for rt in [[rt for rt in association if rt != resource.label()][0]
for association in self.associations
if resource.label() in association]:
refs = {}
required = list(re.compile(RESOURCES[rt]).groupindex.keys())
# We have all the information we need using just the ARN
if all([k in arn_refs for k in required]):
refs = arn_refs
else:
# Check the resource's properties (once)
if len(prop_refs) == 0:
set_references(prop_refs, resource.properties())
# Use property and ARN refs (ARN values take precedence)
refs = {
**{
k: v
for k, v in prop_refs.items() if k in required
},
**{k: v
for k, v in arn_refs.items() if k in required},
}
# There isn't enough information to create a reference ARN
if not all([k in refs for k in required]):
continue
# Hopefully, this never happens
if not all([len(v) == 1 for v in refs.values()]):
continue
# Construct a reference ARN and get the associated resource
arn = RESOURCES.types[rt].format(
**{k: list(v)[0]
for k, v in refs.items()})
associate = next((r for r in self if r.id() == arn), None)
if associate is None:
# self.console.debug(f"Couldn't create association: resource ({arn}), "
# f"referenced by {resource}, doesn't exist ")
continue
(source, target) = sorted((resource, associate),
key=lambda r: r.id())
self.add(
Associative(properties={"Name": "Attached"},
source=source,
target=target))
def _load_resource_collection(self,
collection,
boto_base_resource=None,
awspx_base_resource=None):
resources = []
label = "AWS::{service}::{type}".format(
service=self.__class__.__name__.capitalize(),
type=getattr(boto_base_resource, collection)._model.resource.type)
# Known cases of misaligned naming conventions (boto and lib.aws.RESOURCES)
if label.endswith("Version"):
label = label[:-7]
elif label == "AWS::Ec2::KeyPairInfo":
label = "AWS::Ec2::KeyPair"
# Skip excluded types or types unknown to us
if label not in RESOURCES.types \
or (not self.run_all and label not in self.run):
return
try:
# Note: Depending on the stage at which the exception is thrown, we
# may miss certain resources.
resources = [
r for r in getattr(boto_base_resource, collection).all()
]
except Exception as e:
print(f"[!] Couldn't load {collection} of {boto_base_resource} "
"-- probably due to a resource based policy or something.")
for resource in resources:
# Get properties
properties = resource.meta.data
if properties is None:
continue
# Get Arn and Name
arn = self._get_resource_arn(resource, boto_base_resource)
name = self._get_resource_name(resource)
# Include or exclude this ARN
if self.only_arns and arn not in self.only_arns:
continue
elif self.except_arns and arn in self.except_arns:
continue
properties["Arn"] = arn
properties["Name"] = name
if not (properties["Arn"] and properties["Name"]):
print(label)
print(json.dumps(properties, indent=2, default=str))
raise ValueError
# Create resource
r = Resource(labels=[label], properties=properties)
if r not in self:
self._print(f"[*] Adding {r}")
self.append(r)
# Add associative relationship with parent
if awspx_base_resource:
assocs = [set(a) for a in self.associates]
if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \
or not self.associates:
e = Associative(properties={"Name": "Attached"},
source=r,
target=awspx_base_resource)
oe = Associative(properties={"Name": "Attached"},
source=awspx_base_resource,
target=r)
if not (e in self or oe in self):
self.append(e)
# Load resources from this one's collections
if self._get_collections(resource):
self._load_resources(resource, r)
# Return when we've seen all explicit resources
if self.only_arns and all(
[r in map(lambda x: x.id(), self) for r in self.only_arns]):
return
def _load_resource_collection(self,
collection,
boto_base_resource=None,
awspx_base_resource=None):
resources = []
try:
# Note: Depending on the stage at which the exception is thrown, we
# may miss certain resources.
resources = [
r for r in getattr(boto_base_resource, collection).all()
]
except Exception as e:
print(f"Couldn't load {collection} of {boto_base_resource} "
"- probably due to a resource based policy or something.")
for resource in resources:
# Get properties
properties = resource.meta.data
if properties is None:
continue
# Get label
resource_type = resource.__class__.__name__.split(".")[-1]
label = self._get_resource_type_label(resource_type)
# Get Arn and Name
arn = self._get_resource_arn(resource, boto_base_resource)
name = self._get_resource_name(resource)
# Include or exclude this ARN
if self.only_arns and arn not in self.only_arns:
continue
elif self.except_arns and arn in self.except_arns:
continue
properties["Arn"] = arn
properties["Name"] = name
if not (properties["Arn"] and properties["Name"]):
print(label)
print(json.dumps(properties, indent=2, default=str))
raise ValueError
# Create resource
r = Resource(labels=[label], properties=properties)
if r not in self:
print(f" \-> Adding {r}")
self.append(r)
# Add associative relationship with parent
if awspx_base_resource:
assocs = [set(a) for a in self.associates]
if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \
or not self.associates:
e = Associative(properties={"Name": "Attached"},
source=r,
target=awspx_base_resource)
oe = Associative(properties={"Name": "Attached"},
source=awspx_base_resource,
target=r)
if not (e in self or oe in self):
self.append(e)
# Load resources from this one's collections
if self._get_collections(resource):
self._load_resources(resource, r)
# Return when we've seen all explicit resources
if self.only_arns and all(
[r in map(lambda x: x.id(), self) for r in self.only_arns]):
return