def _get_resource_arn(self, resource, base_resource):
resource_type = resource.__class__.__name__.split(".")[-1]
properties = resource.meta.data
keys = properties.keys()
label = self._get_resource_type_label(resource_type)
arn = None
if "Arn" in keys:
arn = properties["Arn"]
elif f"{resource_type}Arn" in keys:
arn = properties[f"{resource_type}Arn"]
elif f"{resource_type}Id" in keys and properties[
f"{resource_type}Id"].startswith("arn:aws"):
arn = properties[f"{resource_type}Id"]
elif label in RESOURCES.keys():
parent = base_resource.meta.data if base_resource.meta.data is not None else {}
combined = {**parent, **properties}
arn = RESOURCES.definition(label).format(
Region=self.session.region_name,
Account=self.account_id,
**combined)
if isinstance(arn, str) \
and re.compile("arn:aws:([a-zA-Z0-9]+):([a-z0-9-]*):(\d{12}|aws)?:(.*)"
).match(arn) is not None:
return arn
return None
def load_generics(self, types=None):
for k in self.console.tasklist(f"Adding Generic resources",
self.types,
done=f"Added Generic resources"):
self.add(
Generic(properties={
"Name": f"${k.split(':')[-1]}",
"Arn": RESOURCES.definition(k),
},
labels=[k]))
def _load_generics(self, types=None):
labels = [t for t in types if t in RESOURCES] \
if types is not None else \
[t for t in RESOURCES if t.startswith(
"AWS::%s::" % self.__class__.__name__.capitalize())]
for k in labels:
self.add(Generic(properties={
"Name": "$%s" % k.split(':')[-1],
"Arn": RESOURCES.definition(k)
}, labels=[k]))
def _load_associations(self):
if len(self.associates) == 0:
return
self._print(
f"[*] Adding {self.__class__.__name__} associative relationships")
edges = Elements()
for resource in self.get("Resource"):
references = {}
label = [l for l in resource.labels() if l != "Resource"][0]
# Find references to other resources in the form of a dictionary (refs)
self._references(resource.properties(), references)
# Create an edge, for all known associations (as defined by self.rels).
for rel in [
r for r in self.associates
if r[0] == label or r[1] == label
]:
i = 1 if label == rel[0] else 0
# Get a list of foreign keys that we must be capable of referencing
# in order to create an association
fk = [
a for a in re.compile("{([A-Za-z]+)}").findall(
RESOURCES.definition(rel[i]))
if a not in ["Account", "Region"]
]
if not all([k in references.keys() for k in fk]):
continue
# TODO: Handle Types that make use of more than one
# variable identifier
if len(fk) != 1:
raise NotImplementedError
fk = fk[0]
for v in list(references[fk]):
# Find the first resource matching the reference
r = next((r
for r in self if re.compile(
RESOURCES.definition(rel[i]).
format(Account=self.account_id,
Region=self.session.region_name,
**{
**{
x: list(y)[0]
for x, y in references.items(
) if len(y) == 1
},
**{
fk: v
}
})).match(r.id()) is not None), None)
if r is None:
# print("Failed to match (%s: %s) against any resources" % (k, v))
# print("Its likely that the resource was missed during ingestion")
continue
# Delete the properties that are responsible for the edge's existence.
properties = self._extract_property_value(
resource.properties(), fk)
# Even though direction is irrelavent when dealing with Associative
# edges, neo4j is directed. We need to ensure the direction is kept
# in order to eliminate duplicate edges.
(source, target) = (resource, r) if i == 1 else (r,
resource)
edge = Associative(properties={"Name": "Attached"},
source=source,
target=target)
opposite_edge = Associative(
properties={"Name": "Attached"},
source=target,
target=source)
if (edge not in self and opposite_edge
not in self) and edge not in edges:
edges.append(edge)
self.extend(edges)
def run_ingestor(collections, model):
if not len(collections) > 0:
return
for attr, v in model.items():
label = list(v.keys())[0]
collection_managers = []
if len(self.types) > 0 and label not in self.types:
collateral = [
rt for rt in [
list(k.keys())[0]
for k in list(v.values())[0].values()
] if rt in self.types and rt not in
[list(k.keys())[0] for k in model.values()]
]
self.console.debug(''.join(
(f"Skipped {label} ingestion ",
f"({', '.join(collateral)} will also be skipped)."
if len(collateral) > 0 else "")))
continue
rt = ''.join(''.join([
f" {c}" if c.isupper() else c for c in getattr(
collections[0], attr)._model.request.operation
]).split()[1:])
for operation, collection in self.console.tasklist(
f"Adding {rt}",
iterables=map(lambda c: (getattr(c, attr).all, c),
collections),
wait=
f"Awaiting response to {self.__class__.__name__.lower()}:"
f"{getattr(collections[0], attr)._model.request.operation}",
done=f"Added {rt}"):
for cm in SessionClientWrapper(operation(),
console=self.console):
collection_managers.append(cm)
if 'meta' not in dir(cm) or cm.meta.data is None:
self.console.warn(
f"Skipping ServiceResource {cm}: "
"it has no properties")
continue
cm.meta.data["Name"] = [getattr(cm, i)
for i in cm.meta.identifiers
][-1] if "Name" not in cm.meta.data.keys() \
else cm.meta.data["Name"]
properties = {
**cm.meta.data,
**dict(collection.meta.data
if collection is not None
and not collection.__class__.__name__.endswith("ServiceResource")
and collection.meta.data is not None
else {}),
}
try:
cm.meta.data["Arn"] = RESOURCES.definition(
label).format(Region=self.session.region_name,
Account=self.account,
**properties)
except KeyError as p:
self.console.warn(
f"Failed to construct resource ARN: defintion for type '{label}' is malformed - "
f"boto collection '{cm.__class__.__name__}' does not have property {p}, "
f"maybe you meant one of the following ({', '.join(properties.keys())}) instead?"
)
continue
# Add Resource
resource = Resource(labels=[label],
properties=cm.meta.data)
self.add(resource)
for _, attrs in v.items():
run_ingestor(collection_managers, attrs)