def _get_resource_arn(self, resource, base_resource):
resource_type = resource.__class__.__name__.split(".")[-1]
properties = resource.meta.data
keys = properties.keys()
label = self._get_resource_type_label(resource_type)
arn = None
if "Arn" in keys:
arn = properties["Arn"]
elif f"{resource_type}Arn" in keys:
arn = properties[f"{resource_type}Arn"]
elif f"{resource_type}Id" in keys and properties[
f"{resource_type}Id"].startswith("arn:aws"):
arn = properties[f"{resource_type}Id"]
elif label in RESOURCES.keys():
parent = base_resource.meta.data if base_resource.meta.data is not None else {}
combined = {**parent, **properties}
arn = RESOURCES.definition(label).format(
Region=self.session.region_name,
Account=self.account_id,
**combined)
if isinstance(arn, str) \
and re.compile("arn:aws:([a-zA-Z0-9]+):([a-z0-9-]*):(\d{12}|aws)?:(.*)"
).match(arn) is not None:
return arn
return None
def list_user_mfa_devices(self):
if not any([
r in self.types
for r in ["AWS::Iam::MfaDevice", "AWS::Iam::VirtualMfaDevice"]
]):
return
for user in self.console.tasklist(
"Adding MfaDevices",
iterables=self.get("AWS::Iam::User").get("Resource"),
wait="Awaiting response to iam:ListMFADevices",
done="Added MFA devices",
):
for mfa_device in self.client.list_mfa_devices(
UserName=user.get("Name"))["MFADevices"]:
label = RESOURCES.label(mfa_device["SerialNumber"])
mfa_device["Arn"] = mfa_device["SerialNumber"]
mfa_device["Name"] = mfa_device["Arn"].split('/')[-1] if label == "AWS::Iam::MfaDevice" \
else "Virtual Device" if label == "AWS::Iam::VirtualMfaDevice" \
else "Device"
if label is None:
continue
del mfa_device["SerialNumber"]
resource = Resource(labels=[label], properties=mfa_device)
self.add(resource)
def validate_types(types):
"""
Check types against known resources & print invalid ones.
"""
invalid = set(types) - set(RESOURCES.keys())
if invalid != set():
print(f"Invalid resource type(s): {list(invalid)}.")
sys.exit()
def load_generics(self, types=None):
for k in self.console.tasklist(f"Adding Generic resources",
self.types,
done=f"Added Generic resources"):
self.add(
Generic(properties={
"Name": f"${k.split(':')[-1]}",
"Arn": RESOURCES.definition(k),
},
labels=[k]))
def _load_generics(self, types=None):
labels = [t for t in types if t in RESOURCES] \
if types is not None else \
[t for t in RESOURCES if t.startswith(
"AWS::%s::" % self.__class__.__name__.capitalize())]
for k in labels:
self.add(Generic(properties={
"Name": "$%s" % k.split(':')[-1],
"Arn": RESOURCES.definition(k)
}, labels=[k]))
def _get_principals(self):
'''https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html'''
principals = Elements()
key = list(filter(lambda x: "Principal" in x,
self.__statement.keys()))[0]
statement = self.__statement[key]
if isinstance(statement, str) and statement == "*":
statement = {"AWS": "*"}
assert isinstance(statement, dict)
if "AWS" in statement:
if not isinstance(statement["AWS"], list):
statement["AWS"] = [statement["AWS"]]
if '*' in statement["AWS"]:
external = External(
key="Arn",
labels=["AWS::Account"],
properties={
"Name": "All AWS Accounts",
"Description":
"Pseudo-Account representing anyone who possesses an AWS account",
"Arn": "arn:aws:iam::{Account}:root"
})
principals = Elements([
*self.__resources.get('AWS::Iam::User').get("Resource"),
*self.__resources.get('AWS::Iam::Role').get("Resource"),
external
])
for principal in [
p for p in statement["AWS"] if '*' not in statement["AWS"]
]:
if '*' in principal:
continue
node = next((a for a in self.__resources.get("Resource")
if a.id() == principal), None)
# We haven't seen this node before. It may belong to another account,
# or it belongs to a service that was not loaded.
if node is None:
name = principal
labels = ["AWS::Account"]
if re.compile(f"^{RESOURCES.regex['Account']}$").match(
principal) is not None:
principal = f"arn:aws:iam::{principal}:root"
labels += ["AWS::Account"]
elif re.compile(
f"^arn:aws:iam::{RESOURCES.regex['Account']}:root$"
).match(principal) is not None:
name = str(principal.split(":")[4])
labels += ["AWS::Account"]
else:
for k, v in RESOURCES.items():
if re.compile(v).match(principal):
name = principal.replace('/',
':').split(':')[-1]
labels = [k]
break
node = External(
key=str("Arn" if principal.
startswith("arn") else "CanonicalUser"),
labels=labels,
properties={
"Name":
str(name),
str("Arn" if principal.startswith("arn") else "CanonicalUser"):
principal,
})
principals.add(node)
elif "Service" in statement:
services = statement["Service"] if isinstance(
statement["Service"], list) else [statement["Service"]]
for service in services:
if service.lower().endswith("amazonaws.com"):
labels = ["AWS::Domain"]
else:
labels = ["Internet::Domain"]
principals.add(
External(key="Name",
labels=labels,
properties={"Name": service}))
elif "Federated" in statement:
node = None
labels = []
statements = statement["Federated"] \
if isinstance(statement["Federated"], list) \
else [statement["Federated"]]
for federated in statements:
if re.compile(RESOURCES["AWS::Iam::SamlProvider"]).match(
federated) is not None:
base = Resource if (next(
(a for a in self.__resources.get("Resoure")
if a.account() == federated.split(':')[4]),
False)) else External
node = base(key="Arn",
labels=["AWS::Iam::SamlProvider"],
properties={
"Name": federated.split('/')[-1],
"Arn": federated
})
elif re.compile(
"^(?=.{1,253}\.?$)(?:(?!-|[^.]+_)[A-Za-z0-9-_]{1,63}(?<!-)(?:\.|$)){2,}$"
).match(federated):
node = External(key="Name",
labels=["Internet::Domain"],
properties={"Name": federated})
else:
node = External(key="Name",
properties={
"Name": federated,
})
principals.add(node)
# TODO:
elif "CanonicalUser" in statement:
principals.add(
External(key="CanonicalUser",
labels=["AWS::Account"],
properties={
"Name": statement["CanonicalUser"],
"CanonicalUser": statement["CanonicalUser"],
}))
else:
console.warn("Unknown principal: ", statement)
return principals
def _resolve_principal_statement(self):
principals = Elements()
key = list(filter(lambda x: "Principal" in x,
self._statement.keys()))[0]
statement = self._statement[key]
if isinstance(statement, str) and statement == "*":
statement = {"AWS": "*"}
if not isinstance(statement, dict):
raise ValueError
if "AWS" in statement:
if not isinstance(statement["AWS"], list):
statement["AWS"] = [statement["AWS"]]
if '*' in statement["AWS"]:
principals = self._resources.get('AWS::Iam::User').get(
"Resource") + self._resources.get('AWS::Iam::Role').get(
"Resource") + [
External(key="Arn",
labels=["AWS::Account"],
properties={
"Name": "All AWS Accounts",
"Arn": "arn:aws:iam::{Account}:root"
})
]
for principal in [
p for p in statement["AWS"] if '*' not in statement["AWS"]
]:
if '*' in principal:
continue
node = next(
(a for a in self._resources if a.id() == principal), None)
# We haven't seen this node before. It may belong to another account,
# or it belongs to a service that was not loaded.
if node is None:
name = principal
labels = []
if re.compile("^%s$" % RESOURCES.regex["Account"]).match(
principal) is not None:
labels += ["AWS::Account"]
principal = "arn:aws:iam::{Account}:root".format(
Account=principal)
elif re.compile(
"^%s$" % "arn:aws:iam::{Account}:root".format(
Account=RESOURCES.regex["Account"])).match(
principal) is not None:
name = str(principal.split(":")[4])
labels += ["AWS::Account"]
else:
for k, v in RESOURCES.items():
if re.compile(v).match(principal):
name = principal.replace('/',
':').split(':')[-1]
labels = [k]
break
node = External(key="Arn",
labels=labels,
properties={
"Name": str(name),
"Arn": principal
})
principals.add(node)
elif "Service" in statement:
services = statement["Service"] if isinstance(
statement["Service"], list) else [statement["Service"]]
for service in services:
if service.endswith("amazonaws.com"):
labels = ["AWS::Domain"]
else:
labels = ["Internet::Domain"]
principals.add(
External(labels=labels, properties={"Name": service}))
elif "Federated" in statement:
node = None
labels = []
if re.compile(RESOURCES["AWS::Iam::SamlProvider"]).match(
statement["Federated"]) is not None:
base = Resource if (next(
(a for a in self._resources
if a.id().split(':')[4] == statement["Federated"].split(
':')[4]), False)) else External
node = base(key="Arn",
labels=["AWS::Iam::SamlProvider"],
properties={
"Name": statement["Federated"].split('/')[-1],
"Arn": statement["Federated"]
})
elif re.compile(
"^(?=.{1,253}\.?$)(?:(?!-|[^.]+_)[A-Za-z0-9-_]{1,63}(?<!-)(?:\.|$)){2,}$"
).match(statement["Federated"]):
node = External(labels=["Internet::Domain"],
properties={"Name": statement["Federated"]})
else:
node = External(properties={
"Name": statement["Federated"],
})
principals.add(node)
# TODO:
elif "CanonicalUser" in statement:
principals.add(
External(labels=["AWS::Account"],
properties={
"Name": statement["CanonicalUser"],
"CanonicalUser": statement["CanonicalUser"],
"Arn": ""
}))
else:
print("Unknown pricipal: ", statement)
self._explicit_principals = principals
def _load_associations(self):
if len(self.associates) == 0:
return
self._print(
f"[*] Adding {self.__class__.__name__} associative relationships")
edges = Elements()
for resource in self.get("Resource"):
references = {}
label = [l for l in resource.labels() if l != "Resource"][0]
# Find references to other resources in the form of a dictionary (refs)
self._references(resource.properties(), references)
# Create an edge, for all known associations (as defined by self.rels).
for rel in [
r for r in self.associates
if r[0] == label or r[1] == label
]:
i = 1 if label == rel[0] else 0
# Get a list of foreign keys that we must be capable of referencing
# in order to create an association
fk = [
a for a in re.compile("{([A-Za-z]+)}").findall(
RESOURCES.definition(rel[i]))
if a not in ["Account", "Region"]
]
if not all([k in references.keys() for k in fk]):
continue
# TODO: Handle Types that make use of more than one
# variable identifier
if len(fk) != 1:
raise NotImplementedError
fk = fk[0]
for v in list(references[fk]):
# Find the first resource matching the reference
r = next((r
for r in self if re.compile(
RESOURCES.definition(rel[i]).
format(Account=self.account_id,
Region=self.session.region_name,
**{
**{
x: list(y)[0]
for x, y in references.items(
) if len(y) == 1
},
**{
fk: v
}
})).match(r.id()) is not None), None)
if r is None:
# print("Failed to match (%s: %s) against any resources" % (k, v))
# print("Its likely that the resource was missed during ingestion")
continue
# Delete the properties that are responsible for the edge's existence.
properties = self._extract_property_value(
resource.properties(), fk)
# Even though direction is irrelavent when dealing with Associative
# edges, neo4j is directed. We need to ensure the direction is kept
# in order to eliminate duplicate edges.
(source, target) = (resource, r) if i == 1 else (r,
resource)
edge = Associative(properties={"Name": "Attached"},
source=source,
target=target)
opposite_edge = Associative(
properties={"Name": "Attached"},
source=target,
target=source)
if (edge not in self and opposite_edge
not in self) and edge not in edges:
edges.append(edge)
self.extend(edges)
def run_ingestor(collections, model):
if not len(collections) > 0:
return
for attr, v in model.items():
label = list(v.keys())[0]
collection_managers = []
if len(self.types) > 0 and label not in self.types:
collateral = [
rt for rt in [
list(k.keys())[0]
for k in list(v.values())[0].values()
] if rt in self.types and rt not in
[list(k.keys())[0] for k in model.values()]
]
self.console.debug(''.join(
(f"Skipped {label} ingestion ",
f"({', '.join(collateral)} will also be skipped)."
if len(collateral) > 0 else "")))
continue
rt = ''.join(''.join([
f" {c}" if c.isupper() else c for c in getattr(
collections[0], attr)._model.request.operation
]).split()[1:])
for operation, collection in self.console.tasklist(
f"Adding {rt}",
iterables=map(lambda c: (getattr(c, attr).all, c),
collections),
wait=
f"Awaiting response to {self.__class__.__name__.lower()}:"
f"{getattr(collections[0], attr)._model.request.operation}",
done=f"Added {rt}"):
for cm in SessionClientWrapper(operation(),
console=self.console):
collection_managers.append(cm)
if 'meta' not in dir(cm) or cm.meta.data is None:
self.console.warn(
f"Skipping ServiceResource {cm}: "
"it has no properties")
continue
cm.meta.data["Name"] = [getattr(cm, i)
for i in cm.meta.identifiers
][-1] if "Name" not in cm.meta.data.keys() \
else cm.meta.data["Name"]
properties = {
**cm.meta.data,
**dict(collection.meta.data
if collection is not None
and not collection.__class__.__name__.endswith("ServiceResource")
and collection.meta.data is not None
else {}),
}
try:
cm.meta.data["Arn"] = RESOURCES.definition(
label).format(Region=self.session.region_name,
Account=self.account,
**properties)
except KeyError as p:
self.console.warn(
f"Failed to construct resource ARN: defintion for type '{label}' is malformed - "
f"boto collection '{cm.__class__.__name__}' does not have property {p}, "
f"maybe you meant one of the following ({', '.join(properties.keys())}) instead?"
)
continue
# Add Resource
resource = Resource(labels=[label],
properties=cm.meta.data)
self.add(resource)
for _, attrs in v.items():
run_ingestor(collection_managers, attrs)
def __init__(self,
session,
console=None,
services=[],
db="default.db",
quick=False,
skip_actions=False,
only_types=[],
skip_types=[],
only_arns=[],
skip_arns=[]):
try:
if console is None:
from lib.util.console import console
self.console = console
identity = self.console.task(
"Awaiting response to sts:GetCallerIdentity",
session.client('sts').get_caller_identity,
done=lambda r: '\n'.join([
f"Identity: {r['Arn']}",
f"Services: {', '.join([s.__name__ for s in services])}",
f"Database: {db}",
f"Account: {r['Account']}",
f"Region: {session.region_name}",
]))
self.account = identity["Account"]
self.console.spacer()
except (ClientError, PartialCredentialsError, ProfileNotFound) as e:
self.console.error(str(e))
sys.exit(1)
if len(only_arns) > 0:
only_types = list(
set(only_types + [RESOURCES.label(arn) for arn in only_arns]))
for ingestor in services:
elements = ingestor(session=session,
console=self.console,
account=self.account,
quick=quick,
only_types=only_types,
skip_types=skip_types,
only_arns=only_arns,
skip_arns=skip_arns)
super().update(elements)
elements.destroy()
self.load_transitives()
if not skip_actions:
self.load_actions()
self.zip = self.save(db)
self.console.spacer()
