def _get_resource_arn(self, resource, base_resource): resource_type = resource.__class__.__name__.split(".")[-1] properties = resource.meta.data keys = properties.keys() label = self._get_resource_type_label(resource_type) arn = None if "Arn" in keys: arn = properties["Arn"] elif f"{resource_type}Arn" in keys: arn = properties[f"{resource_type}Arn"] elif f"{resource_type}Id" in keys and properties[ f"{resource_type}Id"].startswith("arn:aws"): arn = properties[f"{resource_type}Id"] elif label in RESOURCES.keys(): parent = base_resource.meta.data if base_resource.meta.data is not None else {} combined = {**parent, **properties} arn = RESOURCES.definition(label).format( Region=self.session.region_name, Account=self.account_id, **combined) if isinstance(arn, str) \ and re.compile("arn:aws:([a-zA-Z0-9]+):([a-z0-9-]*):(\d{12}|aws)?:(.*)" ).match(arn) is not None: return arn return None
def list_user_mfa_devices(self): if not any([ r in self.types for r in ["AWS::Iam::MfaDevice", "AWS::Iam::VirtualMfaDevice"] ]): return for user in self.console.tasklist( "Adding MfaDevices", iterables=self.get("AWS::Iam::User").get("Resource"), wait="Awaiting response to iam:ListMFADevices", done="Added MFA devices", ): for mfa_device in self.client.list_mfa_devices( UserName=user.get("Name"))["MFADevices"]: label = RESOURCES.label(mfa_device["SerialNumber"]) mfa_device["Arn"] = mfa_device["SerialNumber"] mfa_device["Name"] = mfa_device["Arn"].split('/')[-1] if label == "AWS::Iam::MfaDevice" \ else "Virtual Device" if label == "AWS::Iam::VirtualMfaDevice" \ else "Device" if label is None: continue del mfa_device["SerialNumber"] resource = Resource(labels=[label], properties=mfa_device) self.add(resource)
def validate_types(types): """ Check types against known resources & print invalid ones. """ invalid = set(types) - set(RESOURCES.keys()) if invalid != set(): print(f"Invalid resource type(s): {list(invalid)}.") sys.exit()
def load_generics(self, types=None): for k in self.console.tasklist(f"Adding Generic resources", self.types, done=f"Added Generic resources"): self.add( Generic(properties={ "Name": f"${k.split(':')[-1]}", "Arn": RESOURCES.definition(k), }, labels=[k]))
def _load_generics(self, types=None): labels = [t for t in types if t in RESOURCES] \ if types is not None else \ [t for t in RESOURCES if t.startswith( "AWS::%s::" % self.__class__.__name__.capitalize())] for k in labels: self.add(Generic(properties={ "Name": "$%s" % k.split(':')[-1], "Arn": RESOURCES.definition(k) }, labels=[k]))
def _get_principals(self): '''https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html''' principals = Elements() key = list(filter(lambda x: "Principal" in x, self.__statement.keys()))[0] statement = self.__statement[key] if isinstance(statement, str) and statement == "*": statement = {"AWS": "*"} assert isinstance(statement, dict) if "AWS" in statement: if not isinstance(statement["AWS"], list): statement["AWS"] = [statement["AWS"]] if '*' in statement["AWS"]: external = External( key="Arn", labels=["AWS::Account"], properties={ "Name": "All AWS Accounts", "Description": "Pseudo-Account representing anyone who possesses an AWS account", "Arn": "arn:aws:iam::{Account}:root" }) principals = Elements([ *self.__resources.get('AWS::Iam::User').get("Resource"), *self.__resources.get('AWS::Iam::Role').get("Resource"), external ]) for principal in [ p for p in statement["AWS"] if '*' not in statement["AWS"] ]: if '*' in principal: continue node = next((a for a in self.__resources.get("Resource") if a.id() == principal), None) # We haven't seen this node before. It may belong to another account, # or it belongs to a service that was not loaded. if node is None: name = principal labels = ["AWS::Account"] if re.compile(f"^{RESOURCES.regex['Account']}$").match( principal) is not None: principal = f"arn:aws:iam::{principal}:root" labels += ["AWS::Account"] elif re.compile( f"^arn:aws:iam::{RESOURCES.regex['Account']}:root$" ).match(principal) is not None: name = str(principal.split(":")[4]) labels += ["AWS::Account"] else: for k, v in RESOURCES.items(): if re.compile(v).match(principal): name = principal.replace('/', ':').split(':')[-1] labels = [k] break node = External( key=str("Arn" if principal. startswith("arn") else "CanonicalUser"), labels=labels, properties={ "Name": str(name), str("Arn" if principal.startswith("arn") else "CanonicalUser"): principal, }) principals.add(node) elif "Service" in statement: services = statement["Service"] if isinstance( statement["Service"], list) else [statement["Service"]] for service in services: if service.lower().endswith("amazonaws.com"): labels = ["AWS::Domain"] else: labels = ["Internet::Domain"] principals.add( External(key="Name", labels=labels, properties={"Name": service})) elif "Federated" in statement: node = None labels = [] statements = statement["Federated"] \ if isinstance(statement["Federated"], list) \ else [statement["Federated"]] for federated in statements: if re.compile(RESOURCES["AWS::Iam::SamlProvider"]).match( federated) is not None: base = Resource if (next( (a for a in self.__resources.get("Resoure") if a.account() == federated.split(':')[4]), False)) else External node = base(key="Arn", labels=["AWS::Iam::SamlProvider"], properties={ "Name": federated.split('/')[-1], "Arn": federated }) elif re.compile( "^(?=.{1,253}\.?$)(?:(?!-|[^.]+_)[A-Za-z0-9-_]{1,63}(?<!-)(?:\.|$)){2,}$" ).match(federated): node = External(key="Name", labels=["Internet::Domain"], properties={"Name": federated}) else: node = External(key="Name", properties={ "Name": federated, }) principals.add(node) # TODO: elif "CanonicalUser" in statement: principals.add( External(key="CanonicalUser", labels=["AWS::Account"], properties={ "Name": statement["CanonicalUser"], "CanonicalUser": statement["CanonicalUser"], })) else: console.warn("Unknown principal: ", statement) return principals
def _resolve_principal_statement(self): principals = Elements() key = list(filter(lambda x: "Principal" in x, self._statement.keys()))[0] statement = self._statement[key] if isinstance(statement, str) and statement == "*": statement = {"AWS": "*"} if not isinstance(statement, dict): raise ValueError if "AWS" in statement: if not isinstance(statement["AWS"], list): statement["AWS"] = [statement["AWS"]] if '*' in statement["AWS"]: principals = self._resources.get('AWS::Iam::User').get( "Resource") + self._resources.get('AWS::Iam::Role').get( "Resource") + [ External(key="Arn", labels=["AWS::Account"], properties={ "Name": "All AWS Accounts", "Arn": "arn:aws:iam::{Account}:root" }) ] for principal in [ p for p in statement["AWS"] if '*' not in statement["AWS"] ]: if '*' in principal: continue node = next( (a for a in self._resources if a.id() == principal), None) # We haven't seen this node before. It may belong to another account, # or it belongs to a service that was not loaded. if node is None: name = principal labels = [] if re.compile("^%s$" % RESOURCES.regex["Account"]).match( principal) is not None: labels += ["AWS::Account"] principal = "arn:aws:iam::{Account}:root".format( Account=principal) elif re.compile( "^%s$" % "arn:aws:iam::{Account}:root".format( Account=RESOURCES.regex["Account"])).match( principal) is not None: name = str(principal.split(":")[4]) labels += ["AWS::Account"] else: for k, v in RESOURCES.items(): if re.compile(v).match(principal): name = principal.replace('/', ':').split(':')[-1] labels = [k] break node = External(key="Arn", labels=labels, properties={ "Name": str(name), "Arn": principal }) principals.add(node) elif "Service" in statement: services = statement["Service"] if isinstance( statement["Service"], list) else [statement["Service"]] for service in services: if service.endswith("amazonaws.com"): labels = ["AWS::Domain"] else: labels = ["Internet::Domain"] principals.add( External(labels=labels, properties={"Name": service})) elif "Federated" in statement: node = None labels = [] if re.compile(RESOURCES["AWS::Iam::SamlProvider"]).match( statement["Federated"]) is not None: base = Resource if (next( (a for a in self._resources if a.id().split(':')[4] == statement["Federated"].split( ':')[4]), False)) else External node = base(key="Arn", labels=["AWS::Iam::SamlProvider"], properties={ "Name": statement["Federated"].split('/')[-1], "Arn": statement["Federated"] }) elif re.compile( "^(?=.{1,253}\.?$)(?:(?!-|[^.]+_)[A-Za-z0-9-_]{1,63}(?<!-)(?:\.|$)){2,}$" ).match(statement["Federated"]): node = External(labels=["Internet::Domain"], properties={"Name": statement["Federated"]}) else: node = External(properties={ "Name": statement["Federated"], }) principals.add(node) # TODO: elif "CanonicalUser" in statement: principals.add( External(labels=["AWS::Account"], properties={ "Name": statement["CanonicalUser"], "CanonicalUser": statement["CanonicalUser"], "Arn": "" })) else: print("Unknown pricipal: ", statement) self._explicit_principals = principals
def _load_associations(self): if len(self.associates) == 0: return self._print( f"[*] Adding {self.__class__.__name__} associative relationships") edges = Elements() for resource in self.get("Resource"): references = {} label = [l for l in resource.labels() if l != "Resource"][0] # Find references to other resources in the form of a dictionary (refs) self._references(resource.properties(), references) # Create an edge, for all known associations (as defined by self.rels). for rel in [ r for r in self.associates if r[0] == label or r[1] == label ]: i = 1 if label == rel[0] else 0 # Get a list of foreign keys that we must be capable of referencing # in order to create an association fk = [ a for a in re.compile("{([A-Za-z]+)}").findall( RESOURCES.definition(rel[i])) if a not in ["Account", "Region"] ] if not all([k in references.keys() for k in fk]): continue # TODO: Handle Types that make use of more than one # variable identifier if len(fk) != 1: raise NotImplementedError fk = fk[0] for v in list(references[fk]): # Find the first resource matching the reference r = next((r for r in self if re.compile( RESOURCES.definition(rel[i]). format(Account=self.account_id, Region=self.session.region_name, **{ **{ x: list(y)[0] for x, y in references.items( ) if len(y) == 1 }, **{ fk: v } })).match(r.id()) is not None), None) if r is None: # print("Failed to match (%s: %s) against any resources" % (k, v)) # print("Its likely that the resource was missed during ingestion") continue # Delete the properties that are responsible for the edge's existence. properties = self._extract_property_value( resource.properties(), fk) # Even though direction is irrelavent when dealing with Associative # edges, neo4j is directed. We need to ensure the direction is kept # in order to eliminate duplicate edges. (source, target) = (resource, r) if i == 1 else (r, resource) edge = Associative(properties={"Name": "Attached"}, source=source, target=target) opposite_edge = Associative( properties={"Name": "Attached"}, source=target, target=source) if (edge not in self and opposite_edge not in self) and edge not in edges: edges.append(edge) self.extend(edges)
def run_ingestor(collections, model): if not len(collections) > 0: return for attr, v in model.items(): label = list(v.keys())[0] collection_managers = [] if len(self.types) > 0 and label not in self.types: collateral = [ rt for rt in [ list(k.keys())[0] for k in list(v.values())[0].values() ] if rt in self.types and rt not in [list(k.keys())[0] for k in model.values()] ] self.console.debug(''.join( (f"Skipped {label} ingestion ", f"({', '.join(collateral)} will also be skipped)." if len(collateral) > 0 else ""))) continue rt = ''.join(''.join([ f" {c}" if c.isupper() else c for c in getattr( collections[0], attr)._model.request.operation ]).split()[1:]) for operation, collection in self.console.tasklist( f"Adding {rt}", iterables=map(lambda c: (getattr(c, attr).all, c), collections), wait= f"Awaiting response to {self.__class__.__name__.lower()}:" f"{getattr(collections[0], attr)._model.request.operation}", done=f"Added {rt}"): for cm in SessionClientWrapper(operation(), console=self.console): collection_managers.append(cm) if 'meta' not in dir(cm) or cm.meta.data is None: self.console.warn( f"Skipping ServiceResource {cm}: " "it has no properties") continue cm.meta.data["Name"] = [getattr(cm, i) for i in cm.meta.identifiers ][-1] if "Name" not in cm.meta.data.keys() \ else cm.meta.data["Name"] properties = { **cm.meta.data, **dict(collection.meta.data if collection is not None and not collection.__class__.__name__.endswith("ServiceResource") and collection.meta.data is not None else {}), } try: cm.meta.data["Arn"] = RESOURCES.definition( label).format(Region=self.session.region_name, Account=self.account, **properties) except KeyError as p: self.console.warn( f"Failed to construct resource ARN: defintion for type '{label}' is malformed - " f"boto collection '{cm.__class__.__name__}' does not have property {p}, " f"maybe you meant one of the following ({', '.join(properties.keys())}) instead?" ) continue # Add Resource resource = Resource(labels=[label], properties=cm.meta.data) self.add(resource) for _, attrs in v.items(): run_ingestor(collection_managers, attrs)
def __init__(self, session, console=None, services=[], db="default.db", quick=False, skip_actions=False, only_types=[], skip_types=[], only_arns=[], skip_arns=[]): try: if console is None: from lib.util.console import console self.console = console identity = self.console.task( "Awaiting response to sts:GetCallerIdentity", session.client('sts').get_caller_identity, done=lambda r: '\n'.join([ f"Identity: {r['Arn']}", f"Services: {', '.join([s.__name__ for s in services])}", f"Database: {db}", f"Account: {r['Account']}", f"Region: {session.region_name}", ])) self.account = identity["Account"] self.console.spacer() except (ClientError, PartialCredentialsError, ProfileNotFound) as e: self.console.error(str(e)) sys.exit(1) if len(only_arns) > 0: only_types = list( set(only_types + [RESOURCES.label(arn) for arn in only_arns])) for ingestor in services: elements = ingestor(session=session, console=self.console, account=self.account, quick=quick, only_types=only_types, skip_types=skip_types, only_arns=only_arns, skip_arns=skip_arns) super().update(elements) elements.destroy() self.load_transitives() if not skip_actions: self.load_actions() self.zip = self.save(db) self.console.spacer()