def _find_search_optimizations(filters): """ Searches through all the filters, and creates white/blacklists of types and IDs, which can be used to optimize the filesystem search. :param filters: An iterable of filter objects representing a query :return: A 2-tuple of AuthSet objects: the first is for object types, and the second is for object IDs. """ # The basic approach to this is to determine what is allowed and # prohibited, independently, and then combine them to create the final # white/blacklists. allowed_types = allowed_ids = None prohibited_types = set() prohibited_ids = set() for filter_ in filters: if filter_.property == "type": if filter_.op in ("=", "in"): allowed_types = _update_allow(allowed_types, filter_.value) elif filter_.op == "!=": prohibited_types.add(filter_.value) elif filter_.property == "id": if filter_.op == "=": # An "allow" ID filter implies a type filter too, since IDs # contain types within them. allowed_ids = _update_allow(allowed_ids, filter_.value) allowed_types = _update_allow(allowed_types, get_type_from_id(filter_.value)) elif filter_.op == "!=": prohibited_ids.add(filter_.value) elif filter_.op == "in": allowed_ids = _update_allow(allowed_ids, filter_.value) allowed_types = _update_allow(allowed_types, (get_type_from_id(id_) for id_ in filter_.value)) opt_types = AuthSet(allowed_types, prohibited_types) opt_ids = AuthSet(allowed_ids, prohibited_ids) # If we have both type and ID whitelists, perform a type-based intersection # on them, to further optimize. (Some of the cross-property constraints # occur above; this is essentially a second pass which operates on the # final whitelists, which among other things, incorporates any of the # prohibitions found above.) if opt_types.auth_type == AuthSet.WHITE and \ opt_ids.auth_type == AuthSet.WHITE: opt_types.values.intersection_update( get_type_from_id(id_) for id_ in opt_ids.values) opt_ids.values.intersection_update( id_ for id_ in opt_ids.values if get_type_from_id(id_) in opt_types.values) return opt_types, opt_ids
def get_techniques_used_by_all_groups(self, stix_format=True): groups = self.get_groups() groups = self.remove_revoked(groups) techniques = self.get_techniques() group_relationships = list() group_techniques_ref = list() groups_use_techniques = list() filters = [ Filter("type", "=", "relationship"), Filter('relationship_type', '=', 'uses') ] relationships = self.COMPOSITE_DS.query(filters) for rel in relationships: if get_type_from_id(rel.source_ref) == 'intrusion-set'\ and get_type_from_id(rel.target_ref) == 'attack-pattern': group_relationships.append(rel) for g in groups: for rel in group_relationships: if g['id'] == rel['source_ref']: gs = json.loads(g.serialize()) gs['technique_ref'] = rel['target_ref'] gs['relationship_description'] = rel['description'] gs['relationship_id'] = rel['id'] group_techniques_ref.append(gs) for gt in group_techniques_ref: for t in techniques: if gt['technique_ref'] == t['id']: if 'revoked' in t.keys(): gt['revoked'] = t['revoked'] tactic_list = list() if 'kill_chain_phases' in t.keys(): tactic_list = t['kill_chain_phases'] gt['technique'] = t['name'] if 'description' in t.keys(): gt['technique_description'] = t['description'] gt['tactic'] = tactic_list gt['technique_id'] = t['external_references'][0][ 'external_id'] gt['matrix'] = t['external_references'][0]['source_name'] if 'x_mitre_platforms' in t.keys(): gt['platform'] = t['x_mitre_platforms'] if 'x_mitre_data_sources' in t.keys(): gt['data_sources'] = t['x_mitre_data_sources'] if 'x_mitre_permissions_required' in t.keys(): gt['permissions_required'] = t[ 'x_mitre_permissions_required'] if 'x_mitre_effective_permissions' in t.keys(): gt['effective_permissions'] = t[ 'x_mitre_effective_permissions'] groups_use_techniques.append(gt) if not stix_format: groups_use_techniques = self.translate_stix_objects( groups_use_techniques) return groups_use_techniques
def get_techniques_mitigated_by_mitigation(self, stix_object, stix_format=True): relationships = self.get_relationships_by_object(stix_object) mitigation_relationships = list() for relation in relationships: if get_type_from_id(relation.source_ref) == 'course-of-action': mitigation_relationships.append(relation) filter_objects = [ Filter('type', '=', 'attack-pattern'), Filter('id', '=', [r.target_ref for r in mitigation_relationships]) ] try: enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query( filter_objects) except: enterprise_stix_objects = [] try: pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects) except: pre_stix_objects = [] try: mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects) except: mobile_stix_objects = [] all_techniques_list = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects if not stix_format: all_techniques_list = self.translate_stix_objects( all_techniques_list) return all_techniques_list
def get_software_used_by_group(self, stix_object, stix_format=True): relationships = self.get_relationships_by_object(stix_object) software_relationships = list() for relation in relationships: if get_type_from_id(relation.target_ref) in ['malware', 'tool']: software_relationships.append(relation) filter_objects = [ Filter('type', 'in', ['malware', 'tool']), Filter('id', '=', [r.target_ref for r in software_relationships]) ] try: enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query( filter_objects) except: enterprise_stix_objects = [] try: pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects) except: pre_stix_objects = [] try: mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects) except: mobile_stix_objects = [] all_software_list = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects if not stix_format: all_software_list = self.translate_stix_objects(all_software_list) return all_software_list
def load_baseline(cls): print("Loading baseline from MITRE. This may take some time...") for act in cls.__get__all_actors(): actor = Actor(ext_id=act.id, name=act.name) if "aliases" in act: actor.alias = json.dumps(act['aliases']) if "description" in act: actor.description = act.description db.session.add(actor) for tec in cls.__get_all_techniques(): behaviour = Behaviour(extId=tec.id, name=tec.name) if "description" in tec: behaviour.description = tec.description if "x_mitre_platforms" in tec: behaviour.platforms = json.dumps(tec['x_mitre_platforms']) for rel in cls.__find_relationships(target_id=tec.id, rel_type='mitigates'): mitigation = cls.__get_mitigation_by_id(rel.source_ref) behaviour.mitigation_name = mitigation[0].name behaviour.mitigation_desc = mitigation[0].description db.session.add(behaviour) for sw in cls.__get_all_software(): software = Software(extId=sw.id, name=sw.name, type=sw.type) if "description" in sw: software.description = sw.description if "x_mitre_aliases" in sw: software.alias = json.dumps(sw['x_mitre_aliases']) if "x_mitre_platforms" in sw: software.platforms = json.dumps(sw['x_mitre_platforms']) db.session.add(software) for actor in Actor.query.all(): #print("MITRE: Setting relationships for actor {}...".format(actor.name)) for rel in cls.__find_relationships(source_id=actor.ext_id, rel_type='uses'): type = utils.get_type_from_id(rel.target_ref) if type in ['attack-pattern']: behaviour = Behaviour.query.filter_by(ext_id=rel.target_ref).first() if behaviour: actor.uses_behaviours.append(behaviour) elif type in ['malware', 'tool']: software = Software.query.filter_by(ext_id=rel.target_ref).first() if software: actor.uses_software.append(software) db.session.add(actor) db.session.commit() print("Successfully loaded baseline from MITRE Att&ck knowledge base")
def get_techniques_used_by_group_software(self, stix_object, stix_format=True): # Get all relationships available for group relationships = self.get_relationships_by_object(stix_object) software_relationships = list() # Get all software relationships from group for relation in relationships: if get_type_from_id(relation.target_ref) in ['malware', 'tool']: software_relationships.append(relation) # Get all used by the software that is used by group filter_objects = [ Filter('type', '=', 'relationship'), Filter('relationship_type', '=', 'uses'), Filter('source_ref', 'in', [r.target_ref for r in software_relationships]) ] try: enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query( filter_objects) except: enterprise_stix_objects = [] try: pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects) except: pre_stix_objects = [] try: mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects) except: mobile_stix_objects = [] software_uses = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects # Get all techniques used by the software that is used by group filter_techniques = [ Filter('type', '=', 'attack-pattern'), Filter('id', 'in', [s.target_ref for s in software_uses]) ] try: enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query( filter_techniques) except: enterprise_stix_objects = [] try: pre_stix_objects = self.TC_PRE_SOURCE.query(filter_techniques) except: pre_stix_objects = [] try: mobile_stix_objects = self.TC_MOBILE_SOURCE.query( filter_techniques) except: mobile_stix_objects = [] all_techniques_list = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects if not stix_format: all_techniques_list = self.translate_stix_objects( all_techniques_list) return all_techniques_list
def get_techniques_by_group(self, stix_id): group_uses = [ r for r in self.src.relationships(stix_id, 'uses', source_only=True) if get_type_from_id(r.target_ref) in ['malware', 'tool'] ] software_uses = self.src.query([ Filter('type', '=', 'relationship'), Filter('relationship_type', '=', 'uses'), Filter('source_ref', 'in', [r.source_ref for r in group_uses]) ]) techniques_query = self.src.query([ Filter('type', '=', 'attack-pattern'), Filter('id', 'in', [r.target_ref for r in software_uses]) ]) dict_techniques = [] for current_technique in techniques_query: string_report = current_technique.serialize() dict_report = json.loads(string_report) dict_techniques.append(dict_report) techniques_df = pd.DataFrame(dict_techniques) if techniques_df.empty: return techniques_df techniques_df = techniques_df.fillna("") techniques_df["kill_chain_phases"] = techniques_df[ "kill_chain_phases"].apply(lambda x: [ y["phase_name"] for y in x if 'mitre' in y['kill_chain_name'] ]) techniques_df = techniques_df.rename( index=str, columns={ "x_mitre_permissions_required": "permissions_required", "x_mitre_platforms": "platforms", "id": "mitre_id", "kill_chain_phases": "tactics" }) column_set = set(techniques_df.columns.values) for i in self.columns_list["techniques"]: column_set.add(i) techniques_df = techniques_df.reindex(columns=list(column_set), fill_value="") return techniques_df[self.columns_list["techniques"]]
def get_techniques_by_group_software(src, group_stix_id): # get the malware, tools that the group uses group_uses = [ r for r in src.relationships(group_stix_id, 'uses', source_only=True) if get_type_from_id(r.target_ref) in ['malware', 'tool'] ] # get the technique stix ids that the malware, tools use software_uses = src.query([ Filter('type', '=', 'relationship'), Filter('relationship_type', '=', 'uses'), Filter('source_ref', 'in', [r.source_ref for r in group_uses]) ]) #get the techniques themselves return src.query([ Filter('type', '=', 'attack-pattern'), Filter('id', 'in', [r.target_ref for r in software_uses]) ])
def get_techniques_mitigated_by_all_mitigations(self, stix_format=True): # Get all relationships available relationships = self.get_relationships() # Get all mitigation relationships mitigation_relationships = list() for relation in relationships: if get_type_from_id(relation.source_ref) in ['course-of-action']: mitigation_relationships.append(relation) # Get all techniques techniques = self.get_techniques() all_techniques_list = list() # loop through mitigation relationships to match technique for mr in mitigation_relationships: for t in techniques: if t['id'] == mr['target_ref']: all_techniques_list.append(t) if not stix_format: all_techniques_list = self.translate_stix_objects(all_techniques_list) return all_techniques_list
def get_software_map(composite_ds): print("Parsing softwares ...") malware_filter = Filter('type', '=', 'malware') tool_filter = Filter('type', '=', 'tool') software_map = {} revoke_map = {} for cur_filter in [malware_filter, tool_filter]: for item in composite_ds.query(cur_filter): name = item['name'] software_id = None for er in item['external_references']: if er['source_name'] in [ "mitre-attack", "mobile-mitre-attack", "mitre-mobile-attack" ]: software_id = er['external_id'] break if software_id: if not item['revoked']: if cur_filter == malware_filter: soft_type = 'malware' else: soft_type = 'tool' desc = item['description'] platforms = item.get('x_mitre_platforms') or [] related_attack_ids = [] attack_pattern_refs = [ r.target_ref for r in composite_ds.relationships( item, 'uses', source_only=True) if get_type_from_id(r.target_ref) == 'attack-pattern' ] for attack_item in composite_ds.query([ Filter('type', '=', 'attack-pattern'), Filter('id', 'in', attack_pattern_refs) ]): for er in attack_item['external_references']: if er['source_name'] in [ "mitre-attack", "mobile-mitre-attack", "mitre-mobile-attack" ]: related_attack_ids.append(er['external_id']) break software_map[software_id] = { "name": name, "description": desc, "platforms": platforms, "software_id": software_id, "type": soft_type, "attack_ids": related_attack_ids } print(f"\tAdding {name.upper()} as ID: {software_id}") else: revoke_id = get_revoked_by(item['id'], composite_ds) if revoke_id is None: print( f"\t[WARN] {name.upper()} ({software_id}) has been revoked without being replaced." ) else: revoke_map[software_id] = revoke_id print( f"\tAdding revoked {name.upper()} to the revoked map: {software_id} => {revoke_id}" ) else: print(f"[ERR] Ignored {name.upper()}: No attack ID found.") return software_map, revoke_map