def __init__(self, session, resources=None, db="default.db"): super().__init__(session=session, default=False) self._db = db if resources is None: self.client = self.session.client("iam") print( "[+] Ingesting AWS::Iam::Users, AWS::Iam::Roles, ", "AWS::Iam::Groups, AWS::Iam::Policies, " "AWS::Iam::InstanceProfiles from IAM") self += self.get_account_authorization_details() self.get_login_profile() self.list_access_keys() elif len(resources) > 0: self += resources # Set IAM entities self.entities = Elements( self.get('AWS::Iam::User') + self.get('AWS::Iam::Role')).get("Resource") # Set Account for a in set([e.account() for e in self.entities.get("Resource")]): self.root = Resource(properties={ "Name": a, "Arn": "arn:aws:iam::%s:root" % a }, labels=["Resource", "AWS::Iam::Root"]) break
def list_user_mfa_devices(self): if not any([ r in self.types for r in ["AWS::Iam::MfaDevice", "AWS::Iam::VirtualMfaDevice"] ]): return for user in self.console.tasklist( "Adding MfaDevices", iterables=self.get("AWS::Iam::User").get("Resource"), wait="Awaiting response to iam:ListMFADevices", done="Added MFA devices", ): for mfa_device in self.client.list_mfa_devices( UserName=user.get("Name"))["MFADevices"]: label = RESOURCES.label(mfa_device["SerialNumber"]) mfa_device["Arn"] = mfa_device["SerialNumber"] mfa_device["Name"] = mfa_device["Arn"].split('/')[-1] if label == "AWS::Iam::MfaDevice" \ else "Virtual Device" if label == "AWS::Iam::VirtualMfaDevice" \ else "Device" if label is None: continue del mfa_device["SerialNumber"] resource = Resource(labels=[label], properties=mfa_device) self.add(resource)
def list_functions(self): if 'AWS::Lambda::Function' not in self.types: return functions = Elements() for function in [ f for r in self.console.tasklist( "Adding Functions", iterables=self.client.get_paginator( "list_functions").paginate(), wait="Awaiting response to lambda:ListFunctions", done="Added Functions") for f in r["Functions"] ]: function["Name"] = function["FunctionName"] function["Arn"] = function["FunctionArn"] del function["FunctionName"] del function["FunctionArn"] function = Resource(properties=function, labels=["AWS::Lambda::Function"]) self.add(function)
def get_aad_element(label, entry): properties = dict() for pk, pv in sorted(entry.items()): if pk.endswith("PolicyList"): properties["Documents"] = [{ p["PolicyName"]: p["PolicyDocument"] for p in pv }] elif pk == "AssumeRolePolicyDocument": properties["Trusts"] = pv elif pk in [ "GroupList", "InstanceProfileList", "AttachedManagedPolicies" ]: continue elif pk == "PolicyVersionList": properties["Document"] = [{ "DefaultVersion": [p for p in pv if p["IsDefaultVersion"]][0]["Document"] }] else: properties[pk.replace(label, "")] = pv element = Resource(properties=properties, labels=["Resource", "AWS::Iam::%s" % label]) if "GroupList" in entry.keys(): edges["Groups"].extend([(element, g) for g in entry["GroupList"]]) if "InstanceProfileList" in entry.keys(): edges["InstanceProfiles"].extend([ (get_aad_element("InstanceProfile", ip), element) for ip in entry["InstanceProfileList"] ]) if "AttachedManagedPolicies" in entry.keys(): edges["Policies"].extend([ (element, p["PolicyArn"]) for p in entry["AttachedManagedPolicies"] ]) if element not in elements: print(f" \-> Adding {element}") elements.append(element) return element
def get_aad_element(label, entry): properties = dict() for pk, pv in sorted(entry.items()): if pk.endswith("PolicyList"): properties["Documents"] = [{ p["PolicyName"]: p["PolicyDocument"] for p in pv}] elif pk == "AssumeRolePolicyDocument": properties["Trusts"] = pv elif pk in ["GroupList", "InstanceProfileList", "AttachedManagedPolicies"]: continue elif pk == "PolicyVersionList": properties["Document"] = [{ "DefaultVersion": [p for p in pv if p["IsDefaultVersion"] ][0]["Document"] }] else: properties[pk.replace(label, "")] = pv element = Resource( properties=properties, labels=["Resource", f"AWS::Iam::{label}"]) if f"AWS::Iam::Group" in self.run and "GroupList" in entry.keys(): edges["Groups"].extend([(element, g) for g in entry["GroupList"]]) if f"AWS::Iam::InstanceProfile" in self.run \ and "InstanceProfileList" in entry.keys(): edges["InstanceProfiles"].extend([(get_aad_element("InstanceProfile", ip), element) for ip in entry["InstanceProfileList"]]) if f"AWS::Iam::Policy" in self.run \ and "AttachedManagedPolicies" in entry.keys(): edges["Policies"].extend([(element, p["PolicyArn"]) for p in entry["AttachedManagedPolicies"]]) if (str(f"AWS::Iam::{label}") in self.run and (len(except_arns) == 0 or properties["Arn"] not in except_arns) and (len(only_arns) == 0 or properties["Arn"] in only_arns) and element not in elements): self._print(f"[*] Adding {element}") elements.add(element) return element
def __init__(self, session, account="0000000000000", only_types=[], except_types=[], only_arns=[], except_arns=[]): super().__init__(session=session, default=False) if not (super()._resolve_type_selection(only_types, except_types) and super()._resolve_arn_selection(only_arns, except_arns)): return self.client = self.session.client('lambda') for rt in self.run: print(f"{self.__class__.__name__}: Loading {rt}") resources = self._get_paginated(rt) for resource in resources: resource_type = rt.capitalize()[0:-1] properties = resource label = "AWS::%s::%s" % ( self.__class__.__name__.capitalize(), resource_type.replace("Info", "").replace( "Version", "").replace("Summary", "")) keys = [ re.sub("^" + resource_type, "", k) for k in properties.keys() ] key = "%sArn" % resource_type properties["Arn"] = properties[key] del properties[key] key = "%sName" % resource_type properties["Name"] = properties[key] del properties[key] print( f"{self.__class__.__name__}: Loading {properties['Arn']}") r = Resource(labels=[label], properties=properties) if r not in self: self.append(r)
def get_aad_resources(item, label): resources = [] properties = {} for k in sorted(item.keys()): # Rename PolicyLists to Documents if k.endswith("PolicyList"): properties["Documents"] = [{ p["PolicyName"]: p["PolicyDocument"] for p in item[k] }] # Rename AssumeRolePolicyDocument to Trusts elif k == "AssumeRolePolicyDocument": properties["Trusts"] = item[k] # Add instance profiles elif k == "InstanceProfileList": [ resources.extend( get_aad_resources(instance_profile, "InstanceProfile")) for instance_profile in item[k] ] # Rename PolicyVersionList to Document elif k == "PolicyVersionList": properties["Document"] = [{ "DefaultVersion": [p for p in item[k] if p["IsDefaultVersion"]][0]["Document"] }] # Remove label from property key elif label in k: properties[k.replace(label, "")] = item[k] # No change else: properties[k] = item[k] # Add Resource resources.append( Resource(labels=[f"AWS::Iam::{label}"], properties=properties)) return resources
def list_functions(self): functions = [] self._print("[*] Listing functions (this can take a while)") for function in [ f for r in self.client.get_paginator( "list_functions").paginate() for f in r["Functions"] ]: function["Name"] = function["FunctionName"] function["Arn"] = function["FunctionArn"] del function["FunctionName"] del function["FunctionArn"] f = Resource(properties=function, labels=["AWS::Lambda::Function"]) if f not in functions: self._print(f"[*] Adding {f}") functions.append(f) return functions
def _load_resource_collection(self, collection, boto_base_resource=None, awspx_base_resource=None): resources = [] label = "AWS::{service}::{type}".format( service=self.__class__.__name__.capitalize(), type=getattr(boto_base_resource, collection)._model.resource.type) # Known cases of misaligned naming conventions (boto and lib.aws.RESOURCES) if label.endswith("Version"): label = label[:-7] elif label == "AWS::Ec2::KeyPairInfo": label = "AWS::Ec2::KeyPair" # Skip excluded types or types unknown to us if label not in RESOURCES.types \ or (not self.run_all and label not in self.run): return try: # Note: Depending on the stage at which the exception is thrown, we # may miss certain resources. resources = [ r for r in getattr(boto_base_resource, collection).all() ] except Exception as e: print(f"[!] Couldn't load {collection} of {boto_base_resource} " "-- probably due to a resource based policy or something.") for resource in resources: # Get properties properties = resource.meta.data if properties is None: continue # Get Arn and Name arn = self._get_resource_arn(resource, boto_base_resource) name = self._get_resource_name(resource) # Include or exclude this ARN if self.only_arns and arn not in self.only_arns: continue elif self.except_arns and arn in self.except_arns: continue properties["Arn"] = arn properties["Name"] = name if not (properties["Arn"] and properties["Name"]): print(label) print(json.dumps(properties, indent=2, default=str)) raise ValueError # Create resource r = Resource(labels=[label], properties=properties) if r not in self: self._print(f"[*] Adding {r}") self.append(r) # Add associative relationship with parent if awspx_base_resource: assocs = [set(a) for a in self.associates] if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \ or not self.associates: e = Associative(properties={"Name": "Attached"}, source=r, target=awspx_base_resource) oe = Associative(properties={"Name": "Attached"}, source=awspx_base_resource, target=r) if not (e in self or oe in self): self.append(e) # Load resources from this one's collections if self._get_collections(resource): self._load_resources(resource, r) # Return when we've seen all explicit resources if self.only_arns and all( [r in map(lambda x: x.id(), self) for r in self.only_arns]): return
def run_ingestor(collections, model): if not len(collections) > 0: return for attr, v in model.items(): label = list(v.keys())[0] collection_managers = [] if len(self.types) > 0 and label not in self.types: collateral = [ rt for rt in [ list(k.keys())[0] for k in list(v.values())[0].values() ] if rt in self.types and rt not in [list(k.keys())[0] for k in model.values()] ] self.console.debug(''.join( (f"Skipped {label} ingestion ", f"({', '.join(collateral)} will also be skipped)." if len(collateral) > 0 else ""))) continue rt = ''.join(''.join([ f" {c}" if c.isupper() else c for c in getattr( collections[0], attr)._model.request.operation ]).split()[1:]) for operation, collection in self.console.tasklist( f"Adding {rt}", iterables=map(lambda c: (getattr(c, attr).all, c), collections), wait= f"Awaiting response to {self.__class__.__name__.lower()}:" f"{getattr(collections[0], attr)._model.request.operation}", done=f"Added {rt}"): for cm in SessionClientWrapper(operation(), console=self.console): collection_managers.append(cm) if 'meta' not in dir(cm) or cm.meta.data is None: self.console.warn( f"Skipping ServiceResource {cm}: " "it has no properties") continue cm.meta.data["Name"] = [getattr(cm, i) for i in cm.meta.identifiers ][-1] if "Name" not in cm.meta.data.keys() \ else cm.meta.data["Name"] properties = { **cm.meta.data, **dict(collection.meta.data if collection is not None and not collection.__class__.__name__.endswith("ServiceResource") and collection.meta.data is not None else {}), } try: cm.meta.data["Arn"] = RESOURCES.definition( label).format(Region=self.session.region_name, Account=self.account, **properties) except KeyError as p: self.console.warn( f"Failed to construct resource ARN: defintion for type '{label}' is malformed - " f"boto collection '{cm.__class__.__name__}' does not have property {p}, " f"maybe you meant one of the following ({', '.join(properties.keys())}) instead?" ) continue # Add Resource resource = Resource(labels=[label], properties=cm.meta.data) self.add(resource) for _, attrs in v.items(): run_ingestor(collection_managers, attrs)
class IAM(Ingestor): def __init__(self, session, resources=None, db="default.db"): super().__init__(session=session, default=False) self._db = db if resources is None: self.client = self.session.client("iam") print( "[+] Ingesting AWS::Iam::Users, AWS::Iam::Roles, ", "AWS::Iam::Groups, AWS::Iam::Policies, " "AWS::Iam::InstanceProfiles from IAM") self += self.get_account_authorization_details() self.get_login_profile() self.list_access_keys() elif len(resources) > 0: self += resources # Set IAM entities self.entities = Elements( self.get('AWS::Iam::User') + self.get('AWS::Iam::Role')).get("Resource") # Set Account for a in set([e.account() for e in self.entities.get("Resource")]): self.root = Resource(properties={ "Name": a, "Arn": "arn:aws:iam::%s:root" % a }, labels=["Resource", "AWS::Iam::Root"]) break def get_account_authorization_details(self): elements = Elements() edges = {"Groups": [], "Policies": [], "InstanceProfiles": []} def get_aad_element(label, entry): properties = dict() for pk, pv in sorted(entry.items()): if pk.endswith("PolicyList"): properties["Documents"] = [{ p["PolicyName"]: p["PolicyDocument"] for p in pv }] elif pk == "AssumeRolePolicyDocument": properties["Trusts"] = pv elif pk in [ "GroupList", "InstanceProfileList", "AttachedManagedPolicies" ]: continue elif pk == "PolicyVersionList": properties["Document"] = [{ "DefaultVersion": [p for p in pv if p["IsDefaultVersion"]][0]["Document"] }] else: properties[pk.replace(label, "")] = pv element = Resource(properties=properties, labels=["Resource", "AWS::Iam::%s" % label]) if "GroupList" in entry.keys(): edges["Groups"].extend([(element, g) for g in entry["GroupList"]]) if "InstanceProfileList" in entry.keys(): edges["InstanceProfiles"].extend([ (get_aad_element("InstanceProfile", ip), element) for ip in entry["InstanceProfileList"] ]) if "AttachedManagedPolicies" in entry.keys(): edges["Policies"].extend([ (element, p["PolicyArn"]) for p in entry["AttachedManagedPolicies"] ]) if element not in elements: print(f" \-> Adding {element}") elements.append(element) return element account_authorization_details = [ aad for aad in self.client.get_paginator( "get_account_authorization_details").paginate() ] account_authorization_details = [ (label.replace("DetailList", "").replace("Policies", "Policy"), entry) for aad in account_authorization_details for (label, v) in aad.items() if isinstance(v, list) for entry in v ] for label, entry in account_authorization_details: get_aad_element(label, entry) # User|Group|Role - Attached -> Policy for (s, t) in edges["Policies"]: t = next(entry for entry in elements if entry.id() == t) elements.append( Transitive(properties={"Name": "Attached"}, source=s, target=t)) # User - [MemberOf] -> Group for (s, t) in edges["Groups"]: t = next(entry for entry in elements.get("AWS::Iam::Group") if str(entry).endswith(t)) elements.append( Transitive(properties={"Name": "MemberOf"}, source=s, target=t)) # InstanceProfile - [Attached] -> Role for (s, t) in edges["InstanceProfiles"]: del s.properties()["Roles"] elements.append( Transitive(properties={"Name": "Attached"}, source=s, target=t)) return elements def get_login_profile(self): for user in self.get("AWS::Iam::User").get("Resource"): try: login_profile = self.client.get_login_profile( UserName=user.get("Name"))["LoginProfile"] del login_profile["UserName"] user.set("LoginProfile", login_profile) except self.client.exceptions.NoSuchEntityException: pass def list_access_keys(self): for user in self.get("AWS::Iam::User").get("Resource"): try: access_keys = self.client.list_access_keys( UserName=user.get("Name"))["AccessKeyMetadata"] for i in range(len(access_keys)): k = access_keys[i]["AccessKeyId"] del access_keys[i]["AccessKeyId"] del access_keys[i]["UserName"] access_keys[i] = {k: access_keys[i]} user.set("AccessKeys", access_keys) except self.client.exceptions.NoSuchEntityException: pass def post(self): self.resolve() self.transitive() return self.save(self._db) def transitive(self): instances = self.get("AWS::Ec2::Instance").get("Resource") functions = self.get("AWS::Lambda::Function").get("Resource") roles = self.get("AWS::Iam::Role").get("Resource") instance_profiles = self.get("AWS::Iam::InstanceProfile").get( "Resource") # Instance - [TRANSITIVE] -> Iam Instance Profile for instance in instances: if "IamInstanceProfile" not in instance.properties(): continue target = next((ip for ip in instance_profiles if ip.id() == instance.properties()["IamInstanceProfile"]["Arn"]), None) del instance.properties()["IamInstanceProfile"] self.append( Transitive({"Name": "Attached"}, source=instance, target=target)) # Lambda - [TRANSITIVE] -> Role for function in functions: if "Role" not in function.properties(): continue role = next( (r for r in roles if r.id() == function.properties()["Role"]), None) del function.properties()["Role"] self.append( Transitive({"Name": "Attached"}, source=function, target=role)) def resolve(self): IDP = [ "AWS::Iam::User", "AWS::Iam::Role", "AWS::Iam::Group", "AWS::Iam::Policy" ] RBP = {"AWS::S3::Bucket": "Policy", "AWS::Iam::Role": "Trusts"} (principals, actions, trusts) = (Elements(), Elements(), Elements()) resources = self.get("Resource") + self.get("Generic") print("Resolving actions and resources") # Resolve actions for resource in self.get("Resource"): # Identity Based Policies (Inline and Managed) if resource.labels()[0] in IDP: count = len(actions) actions.extend( IdentityBasedPolicy(resource, resources).resolve()) diff = len(actions) - count if diff > 0: print(f"[+] Identity based Policy for `{resource}` " f"resolved to {diff} action(s)") if resource.labels()[0] in RBP.keys(): # Bucket ACLs if resource.type("AWS::S3::Bucket"): count = len(actions) acl = BucketACL(resource, resources) principals.extend( [p for p in acl.principals() if p not in principals]) actions.extend( [a for a in acl.resolve() if a not in actions]) diff = len(actions) - count if diff > 0: print(f"[+] Bucket ACL for `{resource}` " f"resolved to {diff} action(s)") # Resource Based Policies rbp = ResourceBasedPolicy(resource, resources, keys=[RBP[resource.labels()[0]]]) if len(rbp.principals()) > 0: count = len(actions) resolved = rbp.resolve() principals.extend([ p for p in rbp.principals() if p not in principals and str(p) != RESOURCES.types["AWS::Account"].format( Account=self.root.account()) ]) # TODO: This code should be moved to 'ResourceBasedPolicy' and override resolve(). # For Roles, actions imply a TRUSTS relationship. For both (ACTION and TRUSTS, only actions beginning # with sts:Assume are considered valid. for action in [ a for a in resolved if "AWS::Iam::Role" not in resource.labels() or str(a).startswith("sts:AssumeRole") ]: if action.source().type("AWS::Account") \ and action.source().properties()["Arn"].split(':')[4] == self.root.account(): if "AWS::Iam::Role" in resource.labels(): trusts.extend([ Trusts(properties=action.properties(), source=action.target(), target=e) for e in self.entities ]) # This case appears redundant for Buckets else: actions.append(action) if "AWS::Iam::Role" in resource.labels(): trusts.append( Trusts(properties=action.properties(), source=action.target(), target=action.source())) diff = len(actions) - count if diff > 0: print(f"[+] Resource based policy for `{resource}` " f"resolved to {diff} action(s)") self.extend([p for p in principals if p not in self]) self.extend([a for a in actions if a not in self]) self.extend(trusts)
def _load_resource_collection(self, collection, boto_base_resource=None, awspx_base_resource=None): resources = [] try: # Note: Depending on the stage at which the exception is thrown, we # may miss certain resources. resources = [ r for r in getattr(boto_base_resource, collection).all() ] except Exception as e: print(f"Couldn't load {collection} of {boto_base_resource} " "- probably due to a resource based policy or something.") for resource in resources: # Get properties properties = resource.meta.data if properties is None: continue # Get label resource_type = resource.__class__.__name__.split(".")[-1] label = self._get_resource_type_label(resource_type) # Get Arn and Name arn = self._get_resource_arn(resource, boto_base_resource) name = self._get_resource_name(resource) # Include or exclude this ARN if self.only_arns and arn not in self.only_arns: continue elif self.except_arns and arn in self.except_arns: continue properties["Arn"] = arn properties["Name"] = name if not (properties["Arn"] and properties["Name"]): print(label) print(json.dumps(properties, indent=2, default=str)) raise ValueError # Create resource r = Resource(labels=[label], properties=properties) if r not in self: print(f" \-> Adding {r}") self.append(r) # Add associative relationship with parent if awspx_base_resource: assocs = [set(a) for a in self.associates] if {awspx_base_resource.labels()[0], r.labels()[0]} in assocs \ or not self.associates: e = Associative(properties={"Name": "Attached"}, source=r, target=awspx_base_resource) oe = Associative(properties={"Name": "Attached"}, source=awspx_base_resource, target=r) if not (e in self or oe in self): self.append(e) # Load resources from this one's collections if self._get_collections(resource): self._load_resources(resource, r) # Return when we've seen all explicit resources if self.only_arns and all( [r in map(lambda x: x.id(), self) for r in self.only_arns]): return