def __init__(self, session, resources=None, db="default.db"):
super().__init__(session=session, default=False)
self._db = db
if resources is None:
self.client = self.session.client("iam")
print(
"[+] Ingesting AWS::Iam::Users, AWS::Iam::Roles, ",
"AWS::Iam::Groups, AWS::Iam::Policies, "
"AWS::Iam::InstanceProfiles from IAM")
self += self.get_account_authorization_details()
self.get_login_profile()
self.list_access_keys()
elif len(resources) > 0:
self += resources
# Set IAM entities
self.entities = Elements(
self.get('AWS::Iam::User') +
self.get('AWS::Iam::Role')).get("Resource")
# Set Account
for a in set([e.account() for e in self.entities.get("Resource")]):
self.root = Resource(properties={
"Name": a,
"Arn": "arn:aws:iam::%s:root" % a
},
labels=["Resource", "AWS::Iam::Root"])
break
def list_user_mfa_devices(self):
if not any([
r in self.types
for r in ["AWS::Iam::MfaDevice", "AWS::Iam::VirtualMfaDevice"]
]):
return
for user in self.console.tasklist(
"Adding MfaDevices",
iterables=self.get("AWS::Iam::User").get("Resource"),
wait="Awaiting response to iam:ListMFADevices",
done="Added MFA devices",
):
for mfa_device in self.client.list_mfa_devices(
UserName=user.get("Name"))["MFADevices"]:
label = RESOURCES.label(mfa_device["SerialNumber"])
mfa_device["Arn"] = mfa_device["SerialNumber"]
mfa_device["Name"] = mfa_device["Arn"].split('/')[-1] if label == "AWS::Iam::MfaDevice" \
else "Virtual Device" if label == "AWS::Iam::VirtualMfaDevice" \
else "Device"
if label is None:
continue
del mfa_device["SerialNumber"]
resource = Resource(labels=[label], properties=mfa_device)
self.add(resource)
def list_functions(self):
if 'AWS::Lambda::Function' not in self.types:
return
functions = Elements()
for function in [
f for r in self.console.tasklist(
"Adding Functions",
iterables=self.client.get_paginator(
"list_functions").paginate(),
wait="Awaiting response to lambda:ListFunctions",
done="Added Functions") for f in r["Functions"]
]:
function["Name"] = function["FunctionName"]
function["Arn"] = function["FunctionArn"]
del function["FunctionName"]
del function["FunctionArn"]
function = Resource(properties=function,
labels=["AWS::Lambda::Function"])
self.add(function)
def get_aad_element(label, entry):
properties = dict()
for pk, pv in sorted(entry.items()):
if pk.endswith("PolicyList"):
properties["Documents"] = [{
p["PolicyName"]: p["PolicyDocument"]
for p in pv
}]
elif pk == "AssumeRolePolicyDocument":
properties["Trusts"] = pv
elif pk in [
"GroupList", "InstanceProfileList",
"AttachedManagedPolicies"
]:
continue
elif pk == "PolicyVersionList":
properties["Document"] = [{
"DefaultVersion":
[p for p in pv if p["IsDefaultVersion"]][0]["Document"]
}]
else:
properties[pk.replace(label, "")] = pv
element = Resource(properties=properties,
labels=["Resource",
"AWS::Iam::%s" % label])
if "GroupList" in entry.keys():
edges["Groups"].extend([(element, g)
for g in entry["GroupList"]])
if "InstanceProfileList" in entry.keys():
edges["InstanceProfiles"].extend([
(get_aad_element("InstanceProfile", ip), element)
for ip in entry["InstanceProfileList"]
])
if "AttachedManagedPolicies" in entry.keys():
edges["Policies"].extend([
(element, p["PolicyArn"])
for p in entry["AttachedManagedPolicies"]
])
if element not in elements:
print(f" \-> Adding {element}")
elements.append(element)
return element
def get_aad_element(label, entry):
properties = dict()
for pk, pv in sorted(entry.items()):
if pk.endswith("PolicyList"):
properties["Documents"] = [{
p["PolicyName"]: p["PolicyDocument"]
for p in pv}]
elif pk == "AssumeRolePolicyDocument":
properties["Trusts"] = pv
elif pk in ["GroupList", "InstanceProfileList", "AttachedManagedPolicies"]:
continue
elif pk == "PolicyVersionList":
properties["Document"] = [{
"DefaultVersion": [p
for p in pv
if p["IsDefaultVersion"]
][0]["Document"]
}]
else:
properties[pk.replace(label, "")] = pv
element = Resource(
properties=properties,
labels=["Resource", f"AWS::Iam::{label}"])
if f"AWS::Iam::Group" in self.run and "GroupList" in entry.keys():
edges["Groups"].extend([(element, g)
for g in entry["GroupList"]])
if f"AWS::Iam::InstanceProfile" in self.run \
and "InstanceProfileList" in entry.keys():
edges["InstanceProfiles"].extend([(get_aad_element("InstanceProfile", ip), element)
for ip in entry["InstanceProfileList"]])
if f"AWS::Iam::Policy" in self.run \
and "AttachedManagedPolicies" in entry.keys():
edges["Policies"].extend([(element, p["PolicyArn"])
for p in entry["AttachedManagedPolicies"]])
if (str(f"AWS::Iam::{label}") in self.run
and (len(except_arns) == 0 or properties["Arn"] not in except_arns)
and (len(only_arns) == 0 or properties["Arn"] in only_arns)
and element not in elements):
self._print(f"[*] Adding {element}")
elements.add(element)
return element
def __init__(self,
session,
account="0000000000000",
only_types=[],
except_types=[],
only_arns=[],
except_arns=[]):
super().__init__(session=session, default=False)
if not (super()._resolve_type_selection(only_types, except_types)
and super()._resolve_arn_selection(only_arns, except_arns)):
return
self.client = self.session.client('lambda')
for rt in self.run:
print(f"{self.__class__.__name__}: Loading {rt}")
resources = self._get_paginated(rt)
for resource in resources:
resource_type = rt.capitalize()[0:-1]
properties = resource
label = "AWS::%s::%s" % (
self.__class__.__name__.capitalize(),
resource_type.replace("Info", "").replace(
"Version", "").replace("Summary", ""))
keys = [
re.sub("^" + resource_type, "", k)
for k in properties.keys()
]
key = "%sArn" % resource_type
properties["Arn"] = properties[key]
del properties[key]
key = "%sName" % resource_type
properties["Name"] = properties[key]
del properties[key]
print(
f"{self.__class__.__name__}: Loading {properties['Arn']}")
r = Resource(labels=[label], properties=properties)
if r not in self:
self.append(r)
def get_aad_resources(item, label):
resources = []
properties = {}
for k in sorted(item.keys()):
# Rename PolicyLists to Documents
if k.endswith("PolicyList"):
properties["Documents"] = [{
p["PolicyName"]: p["PolicyDocument"]
for p in item[k]
}]
# Rename AssumeRolePolicyDocument to Trusts
elif k == "AssumeRolePolicyDocument":
properties["Trusts"] = item[k]
# Add instance profiles
elif k == "InstanceProfileList":
[
resources.extend(
get_aad_resources(instance_profile,
"InstanceProfile"))
for instance_profile in item[k]
]
# Rename PolicyVersionList to Document
elif k == "PolicyVersionList":
properties["Document"] = [{
"DefaultVersion":
[p for p in item[k]
if p["IsDefaultVersion"]][0]["Document"]
}]
# Remove label from property key
elif label in k:
properties[k.replace(label, "")] = item[k]
# No change
else:
properties[k] = item[k]
# Add Resource
resources.append(
Resource(labels=[f"AWS::Iam::{label}"], properties=properties))
return resources
def list_functions(self):
functions = []
self._print("[*] Listing functions (this can take a while)")
for function in [
f for r in self.client.get_paginator(
"list_functions").paginate() for f in r["Functions"]
]:
function["Name"] = function["FunctionName"]
function["Arn"] = function["FunctionArn"]
del function["FunctionName"]
del function["FunctionArn"]
f = Resource(properties=function, labels=["AWS::Lambda::Function"])
if f not in functions:
self._print(f"[*] Adding {f}")
functions.append(f)
return functions
def _load_resource_collection(self,
collection,
boto_base_resource=None,
awspx_base_resource=None):
resources = []
label = "AWS::{service}::{type}".format(
service=self.__class__.__name__.capitalize(),
type=getattr(boto_base_resource, collection)._model.resource.type)
# Known cases of misaligned naming conventions (boto and lib.aws.RESOURCES)
if label.endswith("Version"):
label = label[:-7]
elif label == "AWS::Ec2::KeyPairInfo":
label = "AWS::Ec2::KeyPair"
# Skip excluded types or types unknown to us
if label not in RESOURCES.types \
or (not self.run_all and label not in self.run):
return
try:
# Note: Depending on the stage at which the exception is thrown, we
# may miss certain resources.
resources = [
r for r in getattr(boto_base_resource, collection).all()
]
except Exception as e:
print(f"[!] Couldn't load {collection} of {boto_base_resource} "
"-- probably due to a resource based policy or something.")
for resource in resources:
# Get properties
properties = resource.meta.data
if properties is None:
continue
# Get Arn and Name
arn = self._get_resource_arn(resource, boto_base_resource)
name = self._get_resource_name(resource)
# Include or exclude this ARN
if self.only_arns and arn not in self.only_arns:
continue
elif self.except_arns and arn in self.except_arns:
continue
properties["Arn"] = arn
properties["Name"] = name
if not (properties["Arn"] and properties["Name"]):
print(label)
print(json.dumps(properties, indent=2, default=str))
raise ValueError
# Create resource
r = Resource(labels=[label], properties=properties)
if r not in self:
self._print(f"[*] Adding {r}")
self.append(r)
# Add associative relationship with parent
if awspx_base_resource:
assocs = [set(a) for a in self.associates]
if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \
or not self.associates:
e = Associative(properties={"Name": "Attached"},
source=r,
target=awspx_base_resource)
oe = Associative(properties={"Name": "Attached"},
source=awspx_base_resource,
target=r)
if not (e in self or oe in self):
self.append(e)
# Load resources from this one's collections
if self._get_collections(resource):
self._load_resources(resource, r)
# Return when we've seen all explicit resources
if self.only_arns and all(
[r in map(lambda x: x.id(), self) for r in self.only_arns]):
return
def run_ingestor(collections, model):
if not len(collections) > 0:
return
for attr, v in model.items():
label = list(v.keys())[0]
collection_managers = []
if len(self.types) > 0 and label not in self.types:
collateral = [
rt for rt in [
list(k.keys())[0]
for k in list(v.values())[0].values()
] if rt in self.types and rt not in
[list(k.keys())[0] for k in model.values()]
]
self.console.debug(''.join(
(f"Skipped {label} ingestion ",
f"({', '.join(collateral)} will also be skipped)."
if len(collateral) > 0 else "")))
continue
rt = ''.join(''.join([
f" {c}" if c.isupper() else c for c in getattr(
collections[0], attr)._model.request.operation
]).split()[1:])
for operation, collection in self.console.tasklist(
f"Adding {rt}",
iterables=map(lambda c: (getattr(c, attr).all, c),
collections),
wait=
f"Awaiting response to {self.__class__.__name__.lower()}:"
f"{getattr(collections[0], attr)._model.request.operation}",
done=f"Added {rt}"):
for cm in SessionClientWrapper(operation(),
console=self.console):
collection_managers.append(cm)
if 'meta' not in dir(cm) or cm.meta.data is None:
self.console.warn(
f"Skipping ServiceResource {cm}: "
"it has no properties")
continue
cm.meta.data["Name"] = [getattr(cm, i)
for i in cm.meta.identifiers
][-1] if "Name" not in cm.meta.data.keys() \
else cm.meta.data["Name"]
properties = {
**cm.meta.data,
**dict(collection.meta.data
if collection is not None
and not collection.__class__.__name__.endswith("ServiceResource")
and collection.meta.data is not None
else {}),
}
try:
cm.meta.data["Arn"] = RESOURCES.definition(
label).format(Region=self.session.region_name,
Account=self.account,
**properties)
except KeyError as p:
self.console.warn(
f"Failed to construct resource ARN: defintion for type '{label}' is malformed - "
f"boto collection '{cm.__class__.__name__}' does not have property {p}, "
f"maybe you meant one of the following ({', '.join(properties.keys())}) instead?"
)
continue
# Add Resource
resource = Resource(labels=[label],
properties=cm.meta.data)
self.add(resource)
for _, attrs in v.items():
run_ingestor(collection_managers, attrs)
class IAM(Ingestor):
def __init__(self, session, resources=None, db="default.db"):
super().__init__(session=session, default=False)
self._db = db
if resources is None:
self.client = self.session.client("iam")
print(
"[+] Ingesting AWS::Iam::Users, AWS::Iam::Roles, ",
"AWS::Iam::Groups, AWS::Iam::Policies, "
"AWS::Iam::InstanceProfiles from IAM")
self += self.get_account_authorization_details()
self.get_login_profile()
self.list_access_keys()
elif len(resources) > 0:
self += resources
# Set IAM entities
self.entities = Elements(
self.get('AWS::Iam::User') +
self.get('AWS::Iam::Role')).get("Resource")
# Set Account
for a in set([e.account() for e in self.entities.get("Resource")]):
self.root = Resource(properties={
"Name": a,
"Arn": "arn:aws:iam::%s:root" % a
},
labels=["Resource", "AWS::Iam::Root"])
break
def get_account_authorization_details(self):
elements = Elements()
edges = {"Groups": [], "Policies": [], "InstanceProfiles": []}
def get_aad_element(label, entry):
properties = dict()
for pk, pv in sorted(entry.items()):
if pk.endswith("PolicyList"):
properties["Documents"] = [{
p["PolicyName"]: p["PolicyDocument"]
for p in pv
}]
elif pk == "AssumeRolePolicyDocument":
properties["Trusts"] = pv
elif pk in [
"GroupList", "InstanceProfileList",
"AttachedManagedPolicies"
]:
continue
elif pk == "PolicyVersionList":
properties["Document"] = [{
"DefaultVersion":
[p for p in pv if p["IsDefaultVersion"]][0]["Document"]
}]
else:
properties[pk.replace(label, "")] = pv
element = Resource(properties=properties,
labels=["Resource",
"AWS::Iam::%s" % label])
if "GroupList" in entry.keys():
edges["Groups"].extend([(element, g)
for g in entry["GroupList"]])
if "InstanceProfileList" in entry.keys():
edges["InstanceProfiles"].extend([
(get_aad_element("InstanceProfile", ip), element)
for ip in entry["InstanceProfileList"]
])
if "AttachedManagedPolicies" in entry.keys():
edges["Policies"].extend([
(element, p["PolicyArn"])
for p in entry["AttachedManagedPolicies"]
])
if element not in elements:
print(f" \-> Adding {element}")
elements.append(element)
return element
account_authorization_details = [
aad for aad in self.client.get_paginator(
"get_account_authorization_details").paginate()
]
account_authorization_details = [
(label.replace("DetailList", "").replace("Policies",
"Policy"), entry)
for aad in account_authorization_details
for (label, v) in aad.items() if isinstance(v, list) for entry in v
]
for label, entry in account_authorization_details:
get_aad_element(label, entry)
# User|Group|Role - Attached -> Policy
for (s, t) in edges["Policies"]:
t = next(entry for entry in elements if entry.id() == t)
elements.append(
Transitive(properties={"Name": "Attached"}, source=s,
target=t))
# User - [MemberOf] -> Group
for (s, t) in edges["Groups"]:
t = next(entry for entry in elements.get("AWS::Iam::Group")
if str(entry).endswith(t))
elements.append(
Transitive(properties={"Name": "MemberOf"}, source=s,
target=t))
# InstanceProfile - [Attached] -> Role
for (s, t) in edges["InstanceProfiles"]:
del s.properties()["Roles"]
elements.append(
Transitive(properties={"Name": "Attached"}, source=s,
target=t))
return elements
def get_login_profile(self):
for user in self.get("AWS::Iam::User").get("Resource"):
try:
login_profile = self.client.get_login_profile(
UserName=user.get("Name"))["LoginProfile"]
del login_profile["UserName"]
user.set("LoginProfile", login_profile)
except self.client.exceptions.NoSuchEntityException:
pass
def list_access_keys(self):
for user in self.get("AWS::Iam::User").get("Resource"):
try:
access_keys = self.client.list_access_keys(
UserName=user.get("Name"))["AccessKeyMetadata"]
for i in range(len(access_keys)):
k = access_keys[i]["AccessKeyId"]
del access_keys[i]["AccessKeyId"]
del access_keys[i]["UserName"]
access_keys[i] = {k: access_keys[i]}
user.set("AccessKeys", access_keys)
except self.client.exceptions.NoSuchEntityException:
pass
def post(self):
self.resolve()
self.transitive()
return self.save(self._db)
def transitive(self):
instances = self.get("AWS::Ec2::Instance").get("Resource")
functions = self.get("AWS::Lambda::Function").get("Resource")
roles = self.get("AWS::Iam::Role").get("Resource")
instance_profiles = self.get("AWS::Iam::InstanceProfile").get(
"Resource")
# Instance - [TRANSITIVE] -> Iam Instance Profile
for instance in instances:
if "IamInstanceProfile" not in instance.properties():
continue
target = next((ip for ip in instance_profiles if ip.id() ==
instance.properties()["IamInstanceProfile"]["Arn"]),
None)
del instance.properties()["IamInstanceProfile"]
self.append(
Transitive({"Name": "Attached"},
source=instance,
target=target))
# Lambda - [TRANSITIVE] -> Role
for function in functions:
if "Role" not in function.properties():
continue
role = next(
(r for r in roles if r.id() == function.properties()["Role"]),
None)
del function.properties()["Role"]
self.append(
Transitive({"Name": "Attached"}, source=function, target=role))
def resolve(self):
IDP = [
"AWS::Iam::User", "AWS::Iam::Role", "AWS::Iam::Group",
"AWS::Iam::Policy"
]
RBP = {"AWS::S3::Bucket": "Policy", "AWS::Iam::Role": "Trusts"}
(principals, actions, trusts) = (Elements(), Elements(), Elements())
resources = self.get("Resource") + self.get("Generic")
print("Resolving actions and resources")
# Resolve actions
for resource in self.get("Resource"):
# Identity Based Policies (Inline and Managed)
if resource.labels()[0] in IDP:
count = len(actions)
actions.extend(
IdentityBasedPolicy(resource, resources).resolve())
diff = len(actions) - count
if diff > 0:
print(f"[+] Identity based Policy for `{resource}` "
f"resolved to {diff} action(s)")
if resource.labels()[0] in RBP.keys():
# Bucket ACLs
if resource.type("AWS::S3::Bucket"):
count = len(actions)
acl = BucketACL(resource, resources)
principals.extend(
[p for p in acl.principals() if p not in principals])
actions.extend(
[a for a in acl.resolve() if a not in actions])
diff = len(actions) - count
if diff > 0:
print(f"[+] Bucket ACL for `{resource}` "
f"resolved to {diff} action(s)")
# Resource Based Policies
rbp = ResourceBasedPolicy(resource,
resources,
keys=[RBP[resource.labels()[0]]])
if len(rbp.principals()) > 0:
count = len(actions)
resolved = rbp.resolve()
principals.extend([
p for p in rbp.principals() if p not in principals
and str(p) != RESOURCES.types["AWS::Account"].format(
Account=self.root.account())
])
# TODO: This code should be moved to 'ResourceBasedPolicy' and override resolve().
# For Roles, actions imply a TRUSTS relationship. For both (ACTION and TRUSTS, only actions beginning
# with sts:Assume are considered valid.
for action in [
a for a in resolved
if "AWS::Iam::Role" not in resource.labels()
or str(a).startswith("sts:AssumeRole")
]:
if action.source().type("AWS::Account") \
and action.source().properties()["Arn"].split(':')[4] == self.root.account():
if "AWS::Iam::Role" in resource.labels():
trusts.extend([
Trusts(properties=action.properties(),
source=action.target(),
target=e) for e in self.entities
])
# This case appears redundant for Buckets
else:
actions.append(action)
if "AWS::Iam::Role" in resource.labels():
trusts.append(
Trusts(properties=action.properties(),
source=action.target(),
target=action.source()))
diff = len(actions) - count
if diff > 0:
print(f"[+] Resource based policy for `{resource}` "
f"resolved to {diff} action(s)")
self.extend([p for p in principals if p not in self])
self.extend([a for a in actions if a not in self])
self.extend(trusts)
def _load_resource_collection(self,
collection,
boto_base_resource=None,
awspx_base_resource=None):
resources = []
try:
# Note: Depending on the stage at which the exception is thrown, we
# may miss certain resources.
resources = [
r for r in getattr(boto_base_resource, collection).all()
]
except Exception as e:
print(f"Couldn't load {collection} of {boto_base_resource} "
"- probably due to a resource based policy or something.")
for resource in resources:
# Get properties
properties = resource.meta.data
if properties is None:
continue
# Get label
resource_type = resource.__class__.__name__.split(".")[-1]
label = self._get_resource_type_label(resource_type)
# Get Arn and Name
arn = self._get_resource_arn(resource, boto_base_resource)
name = self._get_resource_name(resource)
# Include or exclude this ARN
if self.only_arns and arn not in self.only_arns:
continue
elif self.except_arns and arn in self.except_arns:
continue
properties["Arn"] = arn
properties["Name"] = name
if not (properties["Arn"] and properties["Name"]):
print(label)
print(json.dumps(properties, indent=2, default=str))
raise ValueError
# Create resource
r = Resource(labels=[label], properties=properties)
if r not in self:
print(f" \-> Adding {r}")
self.append(r)
# Add associative relationship with parent
if awspx_base_resource:
assocs = [set(a) for a in self.associates]
if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \
or not self.associates:
e = Associative(properties={"Name": "Attached"},
source=r,
target=awspx_base_resource)
oe = Associative(properties={"Name": "Attached"},
source=awspx_base_resource,
target=r)
if not (e in self or oe in self):
self.append(e)
# Load resources from this one's collections
if self._get_collections(resource):
self._load_resources(resource, r)
# Return when we've seen all explicit resources
if self.only_arns and all(
[r in map(lambda x: x.id(), self) for r in self.only_arns]):
return