def get_collection_src(): """ Get collection src from collections objects provided by TAXII_SERVER :return: collection source object """ global CS server = Server(TAXII_SERVER) collections = [ c for c in server.api_roots[0].collections if c._title not in EXCLUDE_COLLECTIONS ] if CS.get_all_data_sources(): print("Reusing collections") return CS else: print("Creating new collections") for collection in collections: print("Adding collection %s %s" % (collection._title, collection.id)) CS.add_data_source(TAXIICollectionSource(collection)) return CS
def __init__(self, source='taxii', local=None): """ Initialization - Creates a matrix generator object :param server: Source to utilize (taxii or local) :param local: string path to local cache of stix data """ self.convert_data = {} if source.lower() not in ['taxii', 'local']: print( '[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii" or "local"' .format(source)) raise ValueError if source.lower() == 'taxii': self.server = Server('https://cti-taxii.mitre.org/taxii') self.api_root = self.server.api_roots[0] self.collections = dict() for collection in self.api_root.collections: if collection.title != "PRE-ATT&CK": tc = Collection( 'https://cti-taxii.mitre.org/stix/collections/' + collection.id) self.collections[collection.title.split(' ') [0].lower()] = TAXIICollectionSource(tc) elif source.lower() == 'local': if local is not None: hd = MemoryStore() if 'mobile' in local.lower(): self.collections['mobile'] = hd.load_from_file(local) else: self.collections['enterprise'] = hd.load_from_file(local) else: print( '[MatrixGen] - "local" source specified, but path to local source not provided' ) raise ValueError self.matrix = {} self._build_matrix()
def attnck_taxii(): # Initialize dictionary to hold Enterprise ATT&CK content ent_attack = {} pre_attack = {} # Establish TAXII2 Collection instance for Enterprise ATT&CK collection ENTERPRISE_ATTCK = "95ecc380-afe9-11e4-9b6c-751b66dd541e" PRE_ATTCK = "062767bd-02d2-4b72-84ba-56caef0f8658" MOBILE_ATTCK = "2f669986-b40b-4423-b720-4396ca6a462b" enterprise_attack_collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) pre_attack_collection = Collection( "https://cti-taxii.mitre.org/stix/collections/062767bd-02d2-4b72-84ba-56caef0f8658/" ) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(enterprise_attack_collection) tc_source2 = TAXIICollectionSource(pre_attack_collection) # Create filters to retrieve content from Enterprise ATT&CK based on type filter_objs = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship") } filter_objs_pre = { "techniques": Filter("type", "=", "attack-pattern"), "groups": Filter("type", "=", "intrusion-set"), "relationships": Filter("type", "=", "relationship") } # Retrieve all Enterprise ATT&CK content for key in filter_objs: ent_attack[key] = tc_source.query(filter_objs[key]) for key in filter_objs_pre: pre_attack[key] = tc_source2.query(filter_objs[key]) return ent_attack, pre_attack
def __init__(self, source='taxii', resource=None): """ Initialization - Creates a matrix generator object :param source: Source to utilize (taxii, remote, or local) :param resource: string path to local cache of stix data (local) or url of an ATT&CK Workbench (remote) """ self.convert_data = {} self.collections = dict() if source.lower() not in ['taxii', 'local', 'remote']: print( '[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii", "remote" or ' '"local"'.format(source)) raise ValueError if source.lower() == 'taxii': self.server = Server('https://cti-taxii.mitre.org/taxii') self.api_root = self.server.api_roots[0] for collection in self.api_root.collections: if collection.title != "PRE-ATT&CK": tc = Collection( 'https://cti-taxii.mitre.org/stix/collections/' + collection.id) self.collections[collection.title.split(' ') [0].lower()] = TAXIICollectionSource(tc) elif source.lower() == 'local': if resource is not None: hd = MemoryStore() hd.load_from_file(resource) if 'mobile' in resource.lower(): self.collections['mobile'] = hd else: self.collections['enterprise'] = hd else: print( '[MatrixGen] - "local" source specified, but path to local source not provided' ) raise ValueError elif source.lower() == 'remote': if resource is not None: if ':' not in resource[6:]: print( '[MatrixGen] - "remote" source missing port; assuming ":3000"' ) resource += ":3000" if not resource.startswith('http'): resource = 'http://' + resource for dataset in ['enterprise', 'mobile']: hd = MemoryStore() response = requests.get( f"{resource}/api/stix-bundles?domain={dataset}-" f"attack&includeRevoked=true&includeDeprecated=true") response.raise_for_status( ) # ensure we notice bad responses _add(hd, json.loads(response.text), True, None) self.collections[dataset] = hd else: print( f'[MatrixGen] - WARNING: "remote" selected without providing a "resource" url. The use of ' f'"remote" requires the inclusion of a "resource" url to an ATT&CK Workbench instance. No matrix ' f'will be generated...') self.matrix = {} self._build_matrix()
def execute(self, quals, columns): collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) tc_source = TAXIICollectionSource(collection) conn_string = _conn_string query = "SELECT id,name FROM mitre_intrusion_set" try: conn = ag.connect(conn_string) cur = conn.cursor() cur.execute(query) while True: records = cur.fetchall() if not records: break for i in range(0, len(records)): line = dict() intrusion_set_id = records[i][0] intrusion_set_name = records[i][1] result = self.get_attack_pattern(tc_source, intrusion_set_id) for j in range(0, len(result)): tmp_attack_pattern = dict(result[j]) for column_name in self.columns: if (column_name == 'external_references'): ext_ref = tmp_attack_pattern[column_name] ref_list = list() for index in range(0, len(ext_ref)): if ('url' in ext_ref[index]): ref_list.append(ext_ref[index]['url']) line[column_name] = ref_list elif (column_name == 'kill_chain_phases'): kc_phase = tmp_attack_pattern[column_name] line[column_name] = kc_phase[0]['phase_name'] elif (column_name == 'platforms'): if ('x_mitre_platforms' in tmp_attack_pattern): line[column_name] = tmp_attack_pattern[ 'x_mitre_platforms'] elif (column_name == 'id'): if ('id' in tmp_attack_pattern): line[column_name] = tmp_attack_pattern[ column_name] elif (column_name == 'name'): if ('name' in tmp_attack_pattern): line[column_name] = tmp_attack_pattern[ column_name] elif (column_name == 'gid'): line[column_name] = intrusion_set_id elif (column_name == 'gname'): line[column_name] = intrusion_set_name else: line[column_name] = tmp_attack_pattern[ column_name] yield line except Exception, e: log_to_postgres(e)
####################### # Getting data ####################### # import the stix2 library and the taxii2client libraries from stix2 import TAXIICollectionSource, Filter from taxii2client import Collection # establish TAXII2 Collection instance collection = Collection("https://limo.anomali.com/api/v1/taxii2/feeds/collections/107/", user="******", password="******") # supply the TAXII2 collection to TAXIICollection tc_source = TAXIICollectionSource(collection) # build your filter -- kinda seems broken now # f1 = Filter("type","=", "indicator") #retrieve the STIX objects results = tc_source.query() print("Retrieving...") print("Got {} results".format(len(results))) ####################### # Adding Data Back ####################### # Note that the password below is incorrect. You can find the real password in the Google Doc collection = Collection('https://ubertaxii.com/scratch/collections/4a26ee4a-924c-4d5e-8519-5d87efe7f6a8/', user='******', password='******') status = collection.add_objects(str(Bundle(sighting)))
class attack_client(object): TC_ENTERPRISE_SOURCE = None TC_PRE_SOURCE = None TC_MOBILE_SOURCE = None COMPOSITE_DS = None def __init__(self, local_path=None): if local_path is not None and os.path.isdir(os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR)) \ and os.path.isdir(os.path.join(local_path, PRE_ATTCK_LOCAL_DIR)) \ and os.path.isdir(os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR)): self.TC_ENTERPRISE_SOURCE = FileSystemSource( os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR)) self.TC_PRE_SOURCE = FileSystemSource( os.path.join(local_path, PRE_ATTCK_LOCAL_DIR)) self.TC_MOBILE_SOURCE = FileSystemSource( os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR)) else: ENTERPRISE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + ENTERPRISE_ATTCK + "/") PRE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + PRE_ATTCK + "/") MOBILE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + MOBILE_ATTCK + "/") self.TC_ENTERPRISE_SOURCE = TAXIICollectionSource( ENTERPRISE_COLLECTION) self.TC_PRE_SOURCE = TAXIICollectionSource(PRE_COLLECTION) self.TC_MOBILE_SOURCE = TAXIICollectionSource(MOBILE_COLLECTION) self.COMPOSITE_DS = CompositeDataSource() self.COMPOSITE_DS.add_data_sources([ self.TC_ENTERPRISE_SOURCE, self.TC_PRE_SOURCE, self.TC_MOBILE_SOURCE ]) def translate_stix_objects(self, stix_objects): technique_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "modified": "modified", "object_marking_refs": "object_marking_refs", "name": "technique", "description": "technique_description", "kill_chain_phases": "tactic", "x_mitre_detection": "technique_detection", "x_mitre_platforms": "platform", "x_mitre_data_sources": "data_sources", "x_mitre_defense_bypassed": "defense_bypassed", "x_mitre_permissions_required": "permissions_required", "x_mitre_effective_permissions": "effective_permissions", "x_mitre_system_requirements": "system_requirements", "x_mitre_network_requirements": "network_requirements", "x_mitre_remote_support": "remote_support", "x_mitre_contributors": "contributors", "x_mitre_detectable_by_common_defenses": "detectable_by_common_defenses", "x_mitre_detectable_by_common_defenses_explanation": "detectable_explanation", "x_mitre_difficulty_for_adversary": "difficulty_for_adversary", "x_mitre_difficulty_for_adversary_explanation": "difficulty_explanation", "x_mitre_tactic_type": "tactic_type", "x_mitre_impact_type": "impact_type", "external_references": "external_references" } mitigation_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "modified": "modified", "name": "mitigation", "description": "mitigation_description", "external_references": "external_references", "x_mitre_old_attack_id": "old_mitigation_id" } group_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "modified": "modified", "name": "group", "description": "group_description", "aliases": "group_aliases", "external_references": "external_references", "x_mitre_contributors": "contributors" } software_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "modified": "modified", "name": "software", "description": "software_description", "labels": "software_labels", "x_mitre_aliases": "software_aliases", "x_mitre_platforms": "software_platform", "external_references": "external_references", "x_mitre_contributors": "contributors", "x_mitre_old_attack_id": "old_software_id" } relationship_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "modified": "modified", "relationship_type": "relationship", "description": "relationship_description", "source_ref": "source_object", "target_ref": "target_object" } tactic_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "modified": "modified", "object_marking_refs": "object_marking_refs", "name": "tactic", "description": "tactic_description", "x_mitre_shortname": "tactic_shortname", "external_references": "external_references" } matrix_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "modified": "modified", "object_marking_refs": "object_marking_refs", "name": "matrix", "description": "matrix_description", "tactic_refs": "tactic_references", "external_references": "external_references" } identity_stix_mapping = { "type": "type", "id": "id", "created_by_ref": "created_by_ref", "created": "created", "definition_type": "marking_definition_type", "definition": "marking_definition" } marking_stix_mapping = { "type": "type", "id": "id", "created": "created", "modified": "modified", "object_marking_refs": "object_marking_refs", "name": "identity", "identity_class": "identity_class" } # ******** Helper Functions ******** def handle_list(list_object, object_type): if object_type == "external_references": obj_dict['url'] = list_object[0]['url'] obj_dict['matrix'] = list_object[0]['source_name'] if obj_dict['type'] == 'attack-pattern': for ref in list_object: if ref['source_name'] == 'capec': obj_dict['capec_id'] = ref['external_id'] obj_dict['capec_url'] = ref['url'] obj_dict['technique_id'] = list_object[0]['external_id'] elif obj_dict['type'] == 'course-of-action': obj_dict['mitigation_id'] = list_object[0]['external_id'] elif obj_dict['type'] == 'group': obj_dict['group_id'] = list_object[0]['external_id'] elif obj_dict['type'] == 'software': obj_dict['software_id'] = list_object[0]['external_id'] elif obj_dict['type'] == 'tactic': obj_dict['tactic_id'] = list_object[0]['external_id'] elif obj_dict['type'] == 'matrix': obj_dict['matrix_id'] = list_object[0]['external_id'] elif object_type == "kill_chain_phases": tactic_list = list() for phase in list_object: tactic_list.append(phase['phase_name']) obj_dict['tactic'] = tactic_list stix_objects_list = list() for obj in stix_objects: if isinstance(obj, dict): obj_dict = obj else: obj_dict = json.loads( obj.serialize()) # From STIX to Python Dict dict_keys = list(obj_dict.keys()) for key in dict_keys: if obj['type'] == "attack-pattern": stix_mapping = technique_stix_mapping elif obj['type'] == "course-of-action": stix_mapping = mitigation_stix_mapping elif obj['type'] == "intrusion-set": stix_mapping = group_stix_mapping elif obj['type'] == "malware" or obj['type'] == "tool": stix_mapping = software_stix_mapping elif obj['type'] == "relationship": stix_mapping = relationship_stix_mapping elif obj['type'] == "x-mitre-tactic": stix_mapping = tactic_stix_mapping elif obj['type'] == "x-mitre-matrix": stix_mapping = matrix_stix_mapping elif obj['type'] == "identity": stix_mapping = identity_stix_mapping elif obj['type'] == "marking-definition": stix_mapping = marking_stix_mapping else: exit if key in stix_mapping.keys(): if key == "external_references" or key == "kill_chain_phases": handle_list(obj_dict[key], key) else: new_key = stix_mapping[key] obj_dict[new_key] = obj_dict.pop(key) stix_objects_list.append(obj_dict) return stix_objects_list def remove_revoked(self, stix_objects, extract=False): handle_revoked = list() for obj in stix_objects: if 'revoked' in obj.keys() and obj['revoked'] == True: if extract: handle_revoked.append(obj) else: continue handle_revoked.append(obj) return handle_revoked # ******** Enterprise ATT&CK Technology Domain ******* def get_enterprise(self, stix_format=True): enterprise_filter_objects = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship"), "tactics": Filter("type", "=", "x-mitre-tactic"), "matrix": Filter("type", "=", "x-mitre-matrix"), "identity": Filter("type", "=", "identity"), "marking-definition": Filter("type", "=", "marking-definition") } enterprise_stix_objects = {} for key in enterprise_filter_objects: enterprise_stix_objects[key] = (self.TC_ENTERPRISE_SOURCE.query( enterprise_filter_objects[key])) if not stix_format: enterprise_stix_objects[key] = self.translate_stix_objects( enterprise_stix_objects[key]) return enterprise_stix_objects def get_enterprise_techniques(self, stix_format=True): enterprise_techniques = self.TC_ENTERPRISE_SOURCE.query( Filter("type", "=", "attack-pattern")) if not stix_format: enterprise_techniques = self.translate_stix_objects( enterprise_techniques) return enterprise_techniques def get_enterprise_mitigations(self, stix_format=True): enterprise_mitigations = self.TC_ENTERPRISE_SOURCE.query( Filter("type", "=", "course-of-action")) if not stix_format: enterprise_mitigations = self.translate_stix_objects( enterprise_mitigations) return enterprise_mitigations def get_enterprise_groups(self, stix_format=True): enterprise_groups = self.TC_ENTERPRISE_SOURCE.query( Filter("type", "=", "intrusion-set")) if not stix_format: enterprise_groups = self.translate_stix_objects(enterprise_groups) return enterprise_groups def get_enterprise_malware(self, stix_format=True): enterprise_malware = self.TC_ENTERPRISE_SOURCE.query( Filter("type", "=", "malware")) if not stix_format: enterprise_malware = self.translate_stix_objects( enterprise_malware) return enterprise_malware def get_enterprise_tools(self, stix_format=True): enterprise_tools = self.TC_ENTERPRISE_SOURCE.query( Filter("type", "=", "tool")) if not stix_format: enterprise_tools = self.translate_stix_objects(enterprise_tools) return enterprise_tools def get_enterprise_relationships(self, stix_format=True): enterprise_relationships = self.TC_ENTERPRISE_SOURCE.query( Filter("type", "=", "relationship")) if not stix_format: enterprise_relationships = self.translate_stix_objects( enterprise_relationships) return enterprise_relationships def get_enterprise_tactics(self, stix_format=True): enterprise_tactics = self.TC_ENTERPRISE_SOURCE.query( Filter("type", "=", "x-mitre-tactic")) if not stix_format: enterprise_tactics = self.translate_stix_objects( enterprise_tactics) return enterprise_tactics # ******** Pre ATT&CK Domain ******* def get_pre(self, stix_format=True): pre_filter_objects = { "techniques": Filter("type", "=", "attack-pattern"), "groups": Filter("type", "=", "intrusion-set"), "relationships": Filter("type", "=", "relationship"), "tactics": Filter("type", "=", "x-mitre-tactic"), "matrix": Filter("type", "=", "x-mitre-matrix"), "identity": Filter("type", "=", "identity"), "marking-definition": Filter("type", "=", "marking-definition") } pre_stix_objects = {} for key in pre_filter_objects: pre_stix_objects[key] = self.TC_PRE_SOURCE.query( pre_filter_objects[key]) if not stix_format: pre_stix_objects[key] = self.translate_stix_objects( pre_stix_objects[key]) return pre_stix_objects def get_pre_techniques(self, stix_format=True): pre_techniques = self.TC_PRE_SOURCE.query( Filter("type", "=", "attack-pattern")) if not stix_format: pre_techniques = self.translate_stix_objects(pre_techniques) return pre_techniques def get_pre_groups(self, stix_format=True): pre_groups = self.TC_PRE_SOURCE.query( Filter("type", "=", "intrusion-set")) if not stix_format: pre_groups = self.translate_stix_objects(pre_groups) return pre_groups def get_pre_relationships(self, stix_format=True): pre_relationships = self.TC_PRE_SOURCE.query( Filter("type", "=", "relationship")) if not stix_format: pre_relationships = self.translate_stix_objects(pre_relationships) return pre_relationships def get_pre_tactics(self, stix_format=True): pre_tactics = self.TC_PRE_SOURCE.query( Filter("type", "=", "x-mitre-tactic")) if not stix_format: pre_tactics = self.translate_stix_objects(pre_tactics) return pre_tactics # ******** Mobile ATT&CK Technology Domain ******* def get_mobile(self, stix_format=True): mobile_filter_objects = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship"), "tactics": Filter("type", "=", "x-mitre-tactic"), "matrix": Filter("type", "=", "x-mitre-matrix"), "identity": Filter("type", "=", "identity"), "marking-definition": Filter("type", "=", "marking-definition") } mobile_stix_objects = {} for key in mobile_filter_objects: mobile_stix_objects[key] = self.TC_MOBILE_SOURCE.query( mobile_filter_objects[key]) if not stix_format: mobile_stix_objects[key] = self.translate_stix_objects( mobile_stix_objects[key]) return mobile_stix_objects def get_mobile_techniques(self, stix_format=True): mobile_techniques = self.TC_MOBILE_SOURCE.query( Filter("type", "=", "attack-pattern")) if not stix_format: mobile_techniques = self.translate_stix_objects(mobile_techniques) return mobile_techniques def get_mobile_mitigations(self, stix_format=True): mobile_mitigations = self.TC_MOBILE_SOURCE.query( Filter("type", "=", "course-of-action")) if not stix_format: mobile_mitigations = self.translate_stix_objects( mobile_mitigations) return mobile_mitigations def get_mobile_groups(self, stix_format=True): mobile_groups = self.TC_MOBILE_SOURCE.query( Filter("type", "=", "intrusion-set")) if not stix_format: mobile_groups = self.translate_stix_objects(mobile_groups) return mobile_groups def get_mobile_malware(self, stix_format=True): mobile_malware = self.TC_MOBILE_SOURCE.query( Filter("type", "=", "malware")) if not stix_format: mobile_malware = self.translate_stix_objects(mobile_malware) return mobile_malware def get_mobile_tools(self, stix_format=True): mobile_tools = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "tool")) if not stix_format: mobile_tools = self.translate_stix_objects(mobile_tools) return mobile_tools def get_mobile_relationships(self, stix_format=True): mobile_relationships = self.TC_MOBILE_SOURCE.query( Filter("type", "=", "relationship")) if not stix_format: mobile_relationships = self.translate_stix_objects( mobile_relationships) return mobile_relationships def get_mobile_tactics(self, stix_format=True): mobile_tactics = self.TC_MOBILE_SOURCE.query( Filter("type", "=", "x-mitre-tactic")) if not stix_format: mobile_tactics = self.translate_stix_objects(mobile_tactics) return mobile_tactics # ******** Get All Functions ******** def get_stix_objects(self, stix_format=True): enterprise_objects = self.get_enterprise() pre_objects = self.get_pre() mobile_objects = self.get_mobile() for keypre in pre_objects.keys(): for preobj in pre_objects[keypre]: if keypre in enterprise_objects.keys(): if preobj not in enterprise_objects[keypre]: enterprise_objects[keypre].append(preobj) for keymob in mobile_objects.keys(): for mobobj in mobile_objects[keymob]: if keymob in enterprise_objects.keys(): if mobobj not in enterprise_objects[keymob]: enterprise_objects[keymob].append(mobobj) if not stix_format: for enterkey in enterprise_objects.keys(): enterprise_objects[enterkey] = self.translate_stix_objects( enterprise_objects[enterkey]) return enterprise_objects def get_techniques(self, stix_format=True): all_techniques = self.COMPOSITE_DS.query( Filter("type", "=", "attack-pattern")) if not stix_format: all_techniques = self.translate_stix_objects(all_techniques) return all_techniques def get_groups(self, stix_format=True): all_groups = self.COMPOSITE_DS.query( Filter("type", "=", "intrusion-set")) if not stix_format: all_groups = self.translate_stix_objects(all_groups) return all_groups def get_mitigations(self, stix_format=True): enterprise_mitigations = self.get_enterprise_mitigations() mobile_mitigations = self.get_mobile_mitigations() for mm in mobile_mitigations: if mm not in enterprise_mitigations: enterprise_mitigations.append(mm) if not stix_format: enterprise_mitigations = self.translate_stix_objects( enterprise_mitigations) return enterprise_mitigations def get_software(self, stix_format=True): enterprise_malware = self.get_enterprise_malware() enterprise_tools = self.get_enterprise_tools() mobile_malware = self.get_mobile_malware() mobile_tools = self.get_mobile_tools() for mt in mobile_tools: if mt not in enterprise_tools: enterprise_tools.append(mt) for mmal in mobile_malware: if mmal not in enterprise_malware: enterprise_malware.append(mmal) all_software = enterprise_tools + enterprise_malware if not stix_format: all_software = self.translate_stix_objects(all_software) return all_software def get_relationships(self, stix_format=True): all_relationships = self.COMPOSITE_DS.query( Filter("type", "=", "relationship")) if not stix_format: all_relationships = self.translate_stix_objects(all_relationships) return all_relationships def get_tactics(self, stix_format=True): all_tactics = self.COMPOSITE_DS.query( Filter("type", "=", "x-mitre-tactic")) if not stix_format: all_tactics = self.translate_stix_objects(all_tactics) return all_tactics # ******** Custom Functions ******** def get_technique_by_name(self, name, case=True, stix_format=True):
def test_ds_taxii(collection): ds = TAXIICollectionSource(collection) assert ds.collection is not None
def test_get_stix2_object(collection): tc_sink = TAXIICollectionSource(collection) objects = tc_sink.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f") assert objects
class attack_client(object): ENTERPRISE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + ENTERPRISE_ATTCK + "/") TC_ENTERPRISE_SOURCE = TAXIICollectionSource(ENTERPRISE_COLLECTION) PRE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + PRE_ATTCK + "/") TC_PRE_SOURCE = TAXIICollectionSource(PRE_COLLECTION) MOBILE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + MOBILE_ATTCK + "/") TC_MOBILE_SOURCE = TAXIICollectionSource(MOBILE_COLLECTION) # ******** Helper Functions ******** def parse_stix_objects(self, stix_objects, stix_object_type): stix_objects_list = list() if stix_object_type == 'techniques': for technique in stix_objects: technique_dict = { 'type': technique['type'], 'id': technique['id'], 'created_by_ref': technique['created_by_ref'], 'created': str(technique['created']), 'modified': str(technique['modified']), 'object_marking_refs': technique['object_marking_refs'], 'url': technique['external_references'][0]['url'], 'matrix': technique['external_references'][0]['source_name'], 'technique': technique['name'], 'technique_description': technique['description'], 'tactic': self.handle_list(technique,'kill_chain_phases'), 'technique_id': technique['external_references'][0]['external_id'], 'platform': self.try_except(technique,'x_mitre_platforms'), 'data_sources': self.try_except(technique,'x_mitre_data_sources'), 'defense_bypassed': self.try_except(technique,'x_mitre_defense_bypassed'), 'permissions_required': self.try_except(technique,'x_mitre_permissions_required'), 'effective_permissions': self.try_except(technique,'x_mitre_effective_permissions'), 'system_requirements': self.try_except(technique,'x_mitre_system_requirements'), 'network_requirements': self.try_except(technique,'x_mitre_network_requirements'), 'remote_support': self.try_except(technique,'x_mitre_remote_support'), 'contributors': self.try_except(technique,'x_mitre_contributors'), 'technique_references': self.try_except(technique,'external_references'), 'detectable_by_common_defenses': self.try_except(technique,'x_mitre_detectable_by_common_defenses'), 'detectable_explanation': self.try_except(technique,'x_mitre_detectable_by_common_defenses_explanation'), 'difficulty_for_adversary': self.try_except(technique,'x_mitre_difficulty_for_adversary'), 'difficulty_explanation': self.try_except(technique,'x_mitre_difficulty_for_adversary_explanation'), 'tactic_type': self.try_except(technique,'x_mitre_tactic_type') } stix_objects_list.append(technique_dict) elif stix_object_type == "mitigations": for mitigation in stix_objects: mitigation_dict = { 'type': mitigation['type'], 'id': mitigation['id'], 'created_by_ref': mitigation['created_by_ref'], 'created': str(mitigation['created']), 'modified': str(mitigation['modified']), 'matrix': mitigation['external_references'][0]['source_name'], 'url': mitigation['external_references'][0]['url'], 'mitigation': mitigation['name'], 'mitigation_description': mitigation['description'], 'mitigation_id': mitigation['external_references'][0]['external_id'], 'mitigation_references': self.handle_list(mitigation,'external_references') } stix_objects_list.append(mitigation_dict) elif stix_object_type == "groups": for group in stix_objects: group_dict = { 'type': group['type'], 'id': group['id'], 'created_by_ref': self.try_except(group, 'created_by_ref'), 'matrix': group['external_references'][0]['source_name'], 'created': str(group['created']), 'modified': str(group['modified']), 'url': group['external_references'][0]['url'], 'group': group['name'], 'group_description': self.try_except(group, 'description'), 'group_aliases': self.try_except(group, 'aliases'), 'group_id': group['external_references'][0]['external_id'], 'group_references': self.try_except(group,'external_references') } stix_objects_list.append(group_dict) elif stix_object_type == "tools" or stix_object_type == "malware" or stix_object_type == "software": for software in stix_objects: software_dict = { 'type': software['type'], 'id': software['id'], 'created_by_ref': software['created_by_ref'], 'created': str(software['created']), 'modified': str(software['modified']), 'matrix': software['external_references'][0]['source_name'], 'software': software['name'], 'software_description': software['description'], 'software_labels': self.try_except(software, 'labels'), 'software_id': software['external_references'][0]['external_id'], 'url': software['external_references'][0]['url'], 'software_aliases': self.try_except(software, 'x_mitre_aliases'), 'software_references': self.try_except(software,'external_references') } stix_objects_list.append(software_dict) elif stix_object_type == "relationships": for relationship in stix_objects: relationship_dict = { 'type': relationship['type'], 'id': relationship['id'], 'created_by_ref': relationship['created_by_ref'], 'created': str(relationship['created']), 'modified': str(relationship['modified']), 'relationship': relationship['relationship_type'], 'relationship_description': self.try_except(relationship, 'description'), 'source_object': relationship['source_ref'], 'target_object': relationship['target_ref'] } stix_objects_list.append(relationship_dict) else: exit return stix_objects_list def handle_list(self, stix_objects, object_type): objects_list = list() if object_type == 'kill_chain_phases': for o in stix_objects[object_type]: objects_list.append(o.phase_name) return objects_list elif object_type == 'external_references': for o in stix_objects[object_type]: if "url" in o: objects_list.append(o.url) else: objects_list.append(o.source_name) return objects_list else: for o in stix_objects[object_type]: objects_list.append(o) return objects_list def try_except(self, stix_objects, object_type): if object_type in stix_objects: specific_stix_object = stix_objects[object_type] if isinstance(specific_stix_object, list): lists = self.handle_list(stix_objects, object_type) return lists else: return stix_objects[object_type] else: return None # ******** Get All Functions ******** def get_all_stix_objects(self): techniques_pre_keys = {"techniques","groups","relationships"} techniques_mobile_keys = {"techniques","mitigations","groups","malware","tools","relationships"} enterprise_objects = self.get_all_enterprise() pre_objects = self.get_all_pre() mobile_objects = self.get_all_mobile() for key in techniques_pre_keys: for pre in pre_objects[key]: if pre not in enterprise_objects[key]: enterprise_objects[key].append(pre) for key in techniques_mobile_keys: for m in mobile_objects[key]: if m not in enterprise_objects[key]: enterprise_objects[key].append(m) return enterprise_objects def get_all_attack(self): techniques = self.get_all_techniques_with_mitigations() software = self.get_all_software() software_use = self.get_techniques_used_by_software() groups = self.get_all_groups() groups_use = self.get_all_used_by_group() for s in software: del s['type'],s['id'],s['created_by_ref'],s['created'],s['modified'] for g in groups: del g['type'],g['id'],g['created_by_ref'],g['created'],g['modified'] all_attack = techniques + software + software_use + groups + groups_use return all_attack def get_all_enterprise(self): enterprise_filter_objects = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship") } enterprise_stix_objects = {} for key in enterprise_filter_objects: enterprise_stix_objects[key] = self.TC_ENTERPRISE_SOURCE.query(enterprise_filter_objects[key]) enterprise_stix_objects[key] = self.parse_stix_objects(enterprise_stix_objects[key], key) return enterprise_stix_objects def get_all_pre(self): pre_filter_objects = { "techniques": Filter("type", "=", "attack-pattern"), "groups": Filter("type", "=", "intrusion-set"), "relationships": Filter("type", "=", "relationship") } pre_stix_objects = {} for key in pre_filter_objects: pre_stix_objects[key] = self.TC_PRE_SOURCE.query(pre_filter_objects[key]) pre_stix_objects[key] = self.parse_stix_objects(pre_stix_objects[key], key) return pre_stix_objects def get_all_mobile(self): mobile_filter_objects = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship") } mobile_stix_objects = {} for key in mobile_filter_objects: mobile_stix_objects[key] = self.TC_MOBILE_SOURCE.query(mobile_filter_objects[key]) mobile_stix_objects[key] = self.parse_stix_objects(mobile_stix_objects[key], key) return mobile_stix_objects def get_all_techniques(self): enterprise_techniques = self.get_all_enterprise_techniques() pre_techniques = self.get_all_pre_techniques() mobile_techniques = self.get_all_mobile_techniques() all_techniques = enterprise_techniques + pre_techniques + mobile_techniques return all_techniques def get_all_groups(self): enterprise_groups = self.get_all_enterprise_groups() pre_groups = self.get_all_pre_groups() mobile_groups = self.get_all_mobile_groups() for pg in pre_groups: if pg not in enterprise_groups: enterprise_groups.append(pg) for mg in mobile_groups: if mg not in enterprise_groups: enterprise_groups.append(mg) return enterprise_groups def get_all_mitigations(self): enterprise_mitigations = self.get_all_enterprise_mitigations() mobile_mitigations = self.get_all_mobile_mitigations() for mm in mobile_mitigations: if mm not in enterprise_mitigations: enterprise_mitigations.append(mm) return enterprise_mitigations def get_all_software(self): enterprise_malware = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "malware")) enterprise_tools = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "tool")) mobile_malware = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "malware")) mobile_tools = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "tool")) for mt in mobile_tools: if mt not in enterprise_tools: enterprise_tools.append(mt) for mmal in mobile_malware: if mmal not in enterprise_malware: enterprise_malware.append(mmal) all_software = enterprise_tools + enterprise_malware all_software = self.parse_stix_objects(all_software, 'software') return all_software def get_all_relationships(self): enterprise_relationships = self.get_all_enterprise_relationships() pre_relationships = self.get_all_pre_relationships() mobile_relationships = self.get_all_mobile_relationships() for pr in pre_relationships: if pr not in enterprise_relationships: enterprise_relationships.append(pr) for mr in mobile_relationships: if mr not in enterprise_relationships: enterprise_relationships.append(mr) return enterprise_relationships def get_all_techniques_with_mitigations(self): all_mitigations_mitigate = [] technique_ids = [] all_mitigations_relationships = self.get_relationships_by_object('mitigations') techniques = self.get_all_techniques() for mr in all_mitigations_relationships: for t in techniques: if t['id'] == mr['target_object']: all_mitigations_dict = { 'matrix': t['matrix'], 'mitigation': mr['mitigation'], 'mitigation_description': mr['mitigation_description'], 'mitigation_id' :mr['mitigation_id'], 'mitigation_references': mr['mitigation_references'], 'technique' : t['technique'], 'technique_description' : t['technique_description'], 'tactic' : t['tactic'], 'url' : t['url'], 'technique_id' : t['technique_id'], 'platform' : t['platform'], 'data_sources' : t['data_sources'], 'defense_bypassed' : t['defense_bypassed'], 'permissions_required' : t['permissions_required'], 'effective_permissions' : t['effective_permissions'], 'system_requirements' : t['system_requirements'], 'network_requirements' : t['network_requirements'], 'remote_support' : t['remote_support'], 'contributors' : t['contributors'], 'technique_references' : t['technique_references'], 'detectable_by_common_defenses' : t['detectable_by_common_defenses'], 'detectable_explanation' : t['detectable_explanation'], 'difficulty_for_adversary' : t['difficulty_for_adversary'], 'difficulty_explanation': t['difficulty_explanation'], 'tactic_type' : t['tactic_type'] } all_mitigations_mitigate.append(all_mitigations_dict) technique_ids.append(t['technique_id']) for t in techniques: if t['technique_id'] not in technique_ids: all_techniques_dict = { 'matrix': t['matrix'], 'technique' : t['technique'], 'technique_description' : t['technique_description'], 'tactic' : t['tactic'], 'url' : t['url'], 'technique_id' : t['technique_id'], 'platform' : t['platform'], 'data_sources' : t['data_sources'], 'defense_bypassed' : t['defense_bypassed'], 'permissions_required' : t['permissions_required'], 'effective_permissions' : t['effective_permissions'], 'system_requirements' : t['system_requirements'], 'network_requirements' : t['network_requirements'], 'remote_support' : t['remote_support'], 'contributors' : t['contributors'], 'technique_references' : t['technique_references'], 'detectable_by_common_defenses' : t['detectable_by_common_defenses'], 'detectable_explanation' : t['detectable_explanation'], 'difficulty_for_adversary' : t['difficulty_for_adversary'], 'difficulty_explanation': t['difficulty_explanation'], 'tactic_type' : t['tactic_type'] } all_mitigations_mitigate.append(all_techniques_dict) return all_mitigations_mitigate # ******** Enterprise Matrix Functions ******** def get_all_enterprise_techniques(self): enterprise_techniques = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "attack-pattern")) enterprise_techniques = self.parse_stix_objects(enterprise_techniques, 'techniques') return enterprise_techniques def get_all_enterprise_groups(self): enterprise_groups = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "intrusion-set")) enterprise_groups = self.parse_stix_objects(enterprise_groups, 'groups') return enterprise_groups def get_all_enterprise_mitigations(self): enterprise_mitigations = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "course-of-action")) enterprise_mitigations = self.parse_stix_objects(enterprise_mitigations, 'mitigations') return enterprise_mitigations def get_all_enterprise_relationships(self): enterprise_relationships = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "relationship")) enterprise_relationships = self.parse_stix_objects(enterprise_relationships, 'relationships') return enterprise_relationships # ******** Pre Matrix Functions ******** def get_all_pre_techniques(self): pre_techniques = self.TC_PRE_SOURCE.query(Filter("type", "=", "attack-pattern")) pre_techniques = self.parse_stix_objects(pre_techniques, 'techniques') return pre_techniques def get_all_pre_groups(self): pre_groups = self.TC_PRE_SOURCE.query(Filter("type", "=", "intrusion-set")) pre_groups = self.parse_stix_objects(pre_groups, 'groups') return pre_groups def get_all_pre_relationships(self): pre_relationships = self.TC_PRE_SOURCE.query(Filter("type", "=", "relationship")) pre_relationships = self.parse_stix_objects(pre_relationships, 'relationships') return pre_relationships # ******** Mobile Matrix Functions ******** def get_all_mobile_techniques(self): mobile_techniques = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "attack-pattern")) mobile_techniques = self.parse_stix_objects(mobile_techniques, 'techniques') return mobile_techniques def get_all_mobile_groups(self): mobile_groups = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "intrusion-set")) mobile_groups = self.parse_stix_objects(mobile_groups, 'groups') return mobile_groups def get_all_mobile_mitigations(self): mobile_mitigations = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "course-of-action")) mobile_mitigations = self.parse_stix_objects(mobile_mitigations, 'mitigations') return mobile_mitigations def get_all_mobile_relationships(self): mobile_relationships = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "relationship")) mobile_relationships = self.parse_stix_objects(mobile_relationships, 'relationships') return mobile_relationships # ******** Custom Functions ******** def get_technique_by_name(self, name): filter_objects = [ Filter('type', '=', 'attack-pattern'), Filter('name', '=', name) ] enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(filter_objects) pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects) mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects) all_stix_objects = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects all_stix_objects = self.parse_stix_objects(all_stix_objects, "techniques") for o in all_stix_objects: return o def get_object_by_attack_id(self, object_type, attack_id): valid_objects = {'attack-pattern','course-of-action','intrusion-set','malware','tool'} if object_type not in valid_objects: raise ValueError("ERROR: Valid object must be one of %r" % valid_objects) else: dictionary = { "attack-pattern": "techniques", "course-of-action": "mitigations", "intrusion-set": "groups", "malware": "malware", "tool": "tools" } filter_objects = [ Filter('type', '=', object_type), Filter('external_references.external_id', '=', attack_id) ] enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(filter_objects) enterprise_stix_objects = self.parse_stix_objects(enterprise_stix_objects, dictionary[object_type]) pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects) pre_stix_objects = self.parse_stix_objects(pre_stix_objects, dictionary[object_type]) mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects) mobile_stix_objects = self.parse_stix_objects(mobile_stix_objects, dictionary[object_type]) all_stix_objects = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects for o in all_stix_objects: return o def get_group_by_alias(self, group_alias): filter_objects = [ Filter('type', '=', 'intrusion-set'), Filter('aliases', '=', group_alias) ] enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(filter_objects) pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects) mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects) all_stix_objects = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects all_stix_objects = self.parse_stix_objects(all_stix_objects, 'groups') for o in all_stix_objects: return o def get_relationships_by_object(self, stix_object): valid_objects = {'groups','software','mitigations'} all_relationships = [] relationships = self.get_all_relationships() if stix_object not in valid_objects: raise ValueError("ERROR: Valid object must be one of %r" % valid_objects) else: if stix_object.lower() == 'groups': groups = self.get_all_groups() for g in groups: for r in relationships: if g['id'] == r['source_object'] and r['relationship'] == 'uses': all_groups_relationships_dict = { 'target_object' : r['target_object'], 'relationship_description' : r['relationship_description'], 'matrix': g['matrix'], 'url': g['url'], 'group': g['group'], 'group_description': g['group_description'], 'group_aliases': g['group_aliases'], 'group_id': g['group_id'], 'group_references': g['group_references'] } all_relationships.append(all_groups_relationships_dict) elif stix_object.lower() == 'software': software = self.get_all_software() relationships = self.get_all_relationships() for s in software: for r in relationships: if s['id'] == r['source_object'] and r['relationship'] == 'uses': all_software_relationships_dict = { 'target_object' : r['target_object'], 'relationship_description' : r['relationship_description'], 'software_type': s['type'], 'matrix': s['matrix'], 'software': s['software'], 'software_description': s['software_description'], 'software_labels':s['software_labels'], 'software_id': s['software_id'], 'url': s['url'], 'software_aliases': s['software_aliases'], 'software_references': s['software_references'] } all_relationships.append(all_software_relationships_dict) else: mitigations = self.get_all_mitigations() relationships = self.get_all_relationships() for m in mitigations: for r in relationships: if m['id'] == r['source_object'] and r['relationship'] == 'mitigates': all_mitigations_relationships_dict = { 'target_object' : r['target_object'], 'matrix': m['matrix'], 'mitigation': m['mitigation'], 'mitigation_description': m['mitigation_description'], 'mitigation_id': m['mitigation_id'], 'mitigation_references': m['mitigation_references'] } all_relationships.append(all_mitigations_relationships_dict) return all_relationships def get_all_data_sources(self): techniques = self.get_all_techniques() data_sources = [] for t in techniques: for ds in t['data_sources'] or []: data_sources.append(ds.lower()) return list(set(data_sources)) def get_techniques_used_by_software(self, software_name=None): all_software_use = [] all_techniques_used = [] all_software_relationships = self.get_relationships_by_object('software') techniques = self.get_all_techniques() for sr in all_software_relationships: for t in techniques: if t['id'] == sr['target_object']: all_software_use_dict = { 'matrix': t['matrix'], 'relationship_description': sr['relationship_description'], 'software': sr['software'], 'software_description': sr['software_description'], 'software_labels':sr['software_labels'], 'software_id': sr['software_id'], 'software_aliases': sr['software_aliases'], 'software_references': sr['software_references'], 'technique' : t['technique'], 'technique_description' : t['technique_description'], 'tactic' : t['tactic'], 'technique_id' : t['technique_id'], 'url' : t['url'], #'platform' : t['platform'], #'data_sources' : t['data_sources'], #'defense_bypassed' : t['defense_bypassed'], #'permissions_required' : t['permissions_required'], #'effective_permissions' : t['effective_permissions'], #'system_requirements' : t['system_requirements'], #'network_requirements' : t['network_requirements'], #'remote_support' : t['remote_support'], 'contributors' : t['contributors'], #'technique_references' : t['technique_references'], #'detectable_by_common_defenses' : t['detectable_by_common_defenses'], #'detectable_explanation' : t['detectable_explanation'], #'difficulty_for_adversary' : t['difficulty_for_adversary'], #'difficulty_explanation': t['difficulty_explanation'], #'tactic_type' : t['tactic_type'] } all_software_use.append(all_software_use_dict) if software_name is None: return all_software_use else: for sn in all_software_use: if software_name.lower() in sn['software'].lower(): all_techniques_used.append(sn) return all_techniques_used def get_techniques_used_by_group(self, group_name=None): all_groups_use = [] all_techniques_used = [] all_groups_relationships = self.get_relationships_by_object('groups') techniques = self.get_all_techniques() for gr in all_groups_relationships: for t in techniques: if t['id'] == gr['target_object']: all_groups_use_dict = { 'matrix': t['matrix'], 'relationship_description': gr['relationship_description'], 'group': gr['group'], 'group_description': gr['group_description'], 'group_aliases': gr['group_aliases'], 'group_id': gr['group_id'], 'group_references': gr['group_references'], 'technique' : t['technique'], 'technique_description' : t['technique_description'], 'tactic' : t['tactic'], 'technique_id' : t['technique_id'], 'url' : t['url'], #'platform' : t['platform'], #'data_sources' : t['data_sources'], #'defense_bypassed' : t['defense_bypassed'], #'permissions_required' : t['permissions_required'], #'effective_permissions' : t['effective_permissions'], #'system_requirements' : t['system_requirements'], #'network_requirements' : t['network_requirements'], #'remote_support' : t['remote_support'], 'contributors' : t['contributors'], #'technique_references' : t['technique_references'], #'detectable_by_common_defenses' : t['detectable_by_common_defenses'], #'detectable_explanation' : t['detectable_explanation'], #'difficulty_for_adversary' : t['difficulty_for_adversary'], #'difficulty_explanation': t['difficulty_explanation'], #'tactic_type' : t['tactic_type'] } all_groups_use.append(all_groups_use_dict) if group_name is None: return all_groups_use else: for gn in all_groups_use: if group_name.lower() in gn['group'].lower(): all_techniques_used.append(gn) return all_techniques_used def get_software_used_by_group(self, group_name=None): all_groups_software_use =[] all_groups_use = [] all_software_used = [] all_groups_relationships = self.get_relationships_by_object('groups') software_techniques = self.get_techniques_used_by_software() software = self.get_all_software() for gr in all_groups_relationships: for s in software: if s['id'] == gr['target_object']: all_groups_software = { 'matrix': s['matrix'], 'relationship_description': gr['relationship_description'], 'group': gr['group'], 'group_description': gr['group_description'], 'group_aliases': gr['group_aliases'], 'group_id': gr['group_id'], 'group_references': gr['group_references'], 'software_url': s['url'], 'software': s['software'], 'software_description': s['software_description'], 'software_labels':s['software_labels'], 'software_id': s['software_id'], 'software_aliases': s['software_aliases'], 'software_references': s['software_references'] } all_groups_software_use.append(all_groups_software) for st in software_techniques: for gs in all_groups_software_use: if gs['software_id'] == st['software_id']: all_software_technique = { 'matrix': gs['matrix'], 'relationship_description': gs['relationship_description'], 'group': gs['group'], 'group_description': gs['group_description'], 'group_aliases': gs['group_aliases'], 'group_id': gs['group_id'], 'group_references': gs['group_references'], 'software url': gs['software_url'], 'software': gs['software'], 'software_description': gs['software_description'], 'software_labels':gs['software_labels'], 'software_id': gs['software_id'], 'software_aliases': gs['software_aliases'], 'software_references': gs['software_references'], 'technique' : st['technique'], 'technique_description' : st['technique_description'], 'tactic' : st['tactic'], 'technique_id' : st['technique_id'] } all_groups_use.append(all_software_technique) if group_name is None: return all_groups_use else: for gn in all_groups_use: if group_name.lower() in gn['group'].lower(): all_software_used.append(gn) return all_software_used def get_all_used_by_group(self, group_name=None): all_used = [] if group_name is None: software = self.get_software_used_by_group() techniques = self.get_techniques_used_by_group() all_used = software + techniques else: software = self.get_software_used_by_group(group_name) techniques = self.get_techniques_used_by_group(group_name) all_used = software + techniques return all_used
from stix2 import Filter, TAXIICollectionSource from taxii2client import Collection import argparse import csv #Connect to TAXII server collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) fs = TAXIICollectionSource(collection) techniques_data_sources = {} #Get a group by its name def get_group_by_alias(src, alias): return src.query( [Filter('type', '=', 'intrusion-set'), Filter('aliases', '=', alias)]) #Return group's techniques def get_technique_by_group(src, stix_id): relations = src.relationships(stix_id, 'uses', source_only=True) return src.query([ Filter('type', '=', 'attack-pattern'), Filter('id', 'in', [r.target_ref for r in relations]) ]) #Return all known threat actors
def get_collections(api_roots): for api_root in api_roots: collections = api_root.collections for c in collections: print("\tCOLLECTION:", c.title) c_info = collection_info(c) tc_source = TAXIICollectionSource(c) add_data_source(tc_source) stix_objects = check_stix_object_type() print("\t\tDATA =>", list(stix_objects.keys()), end='\n') # SRO _objects = c.get_objects() objects = _objects['objects'] for o in objects: if o['type'] == 'relationship': sql = """ INSERT IGNORE INTO sro_relationship(source_ref, target_ref, relationship_type) values (%s, %s, %s) """ cur.execute(sql, (o['source_ref'], o['target_ref'], o['relationship_type'])) db.commit() # SDO for stix_object_type in stix_objects: print("\t\t\t", stix_object_type, len(stix_objects[stix_object_type])) for o in stix_objects[stix_object_type]: TYPE = str(type(o)) # cols = [] table = "sdo_" if "AttackPattern" in TYPE: cols = db_format['attack_pattern'] table += 'attack_pattern' elif "Campaign" in TYPE: cols = db_format['campaign'] table += 'campaign' elif "CourseOfAction" in TYPE: cols = db_format['course_of_action'] table += 'course_of_action' elif "Identity" in TYPE: cols = db_format['identity'] table += 'identity' elif "Indicator" in TYPE: cols = db_format['indicator'] table += 'indicator' elif "IntrusionSet" in TYPE: cols = db_format['intrusion_set'] table += 'intrusion_set' elif "Malware" in TYPE: cols = db_format['malware'] table += 'malware' elif "ObservedData" in TYPE: cols = db_format['observed_data'] table += 'observed_data' elif "Report" in TYPE: cols = db_format['report'] table += 'report' elif "ThreatActor" in TYPE: cols = db_format['threat_actor'] table += 'threat_actor' elif "Tool" in TYPE: cols = db_format['tool'] table += 'tool' elif "Vulnerability" in TYPE: cols = db_format['vulnerability'] table += 'vulnerability' else: continue sql = "INSERT IGNORE INTO %s (%s) VALUES(%s)" % ( table, ", ".join(cols + db_format['common']), ", ".join([ "\"" + pymysql.escape_string(str(o.get(c, "None"))) + "\"" for c in cols + db_format['common'] ])) cur.execute(sql) db.commit()
def main(): parser = argparse.ArgumentParser() parser.add_argument("--matrix", type=str, action="store", default="enterprise", help="Matrix to query (enterprise, mobile, pre)") # Techniques parser.add_argument( "--dump-all-techniques", action="store_true", help="Dump a CSV file with technique,subtechnique,name") # Data Sources parser.add_argument( "--dump-data-sources", action="store_true", help="Dump a list of unique data sources to data_sources.txt") parser.add_argument( "--dump-metadata", action="store_true", help= "Dump a CSV file technique-metadata.csv, containing unique technique-metadata pairings." ) parser.add_argument( "--dump-matching-techniques", action="store_true", help= "Dump techniques that map to match-data-sources to matching-techniques.txt" ) parser.add_argument( "--match-data-sources", type=str, action="store", help= "A file containing a list of data sources that to match against techniques." ) args = parser.parse_args() match_data_sources = None if args.match_data_sources: match_data_sources = parse_data_source_list(args.match_data_sources) args.matrix = args.matrix.lower() if args.matrix == 'pre': matrix = "062767bd-02d2-4b72-84ba-56caef0f8658" elif args.matrix == 'mobile': matrix = "2f669986-b40b-4423-b720-4396ca6a462b" elif args.matrix == 'enterprise': matrix = "95ecc380-afe9-11e4-9b6c-751b66dd541e" # Initialize dictionary to hold Enterprise ATT&CK content attack = {} # Establish TAXII2 Collection instance for Enterprise ATT&CK collection = Collection("https://cti-taxii.mitre.org/stix/collections/{0}/"\ .format(matrix)) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(collection) # Create filters to retrieve content from Enterprise ATT&CK filter_objs = {"techniques": Filter("type", "=", "attack-pattern")} # Retrieve all Enterprise ATT&CK content for key in filter_objs: attack[key] = tc_source.query(filter_objs[key]) all_techniques = attack["techniques"] all_techniques = remove_revoked_deprecated(all_techniques) technique_count = 0 techniques_without_data_source = 0 techniques_observable = 0 techniques_with_data_sources = [] data_sources = set() matching_techniques = set() for technique in all_techniques: technique_count += 1 technique_id = technique['external_references'][0]['external_id'] if 'x_mitre_data_sources' in technique.keys(): if match_data_sources is not None: if data_source_match(technique['x_mitre_data_sources'], match_list=match_data_sources) == True: techniques_observable += 1 if args.dump_matching_techniques == True: matching_techniques.add(technique_id) if args.dump_data_sources == True: [ data_sources.add(data_source) for data_source in technique['x_mitre_data_sources'] ] if args.dump_metadata == True: [ techniques_with_data_sources.append( (technique_id, data_source)) for data_source in technique['x_mitre_data_sources'] ] else: techniques_without_data_source += 1 # Output files based on input arguments. if match_data_sources is not None: print('Techniques: {0}'.format(technique_count)) print('Techniques Observable: {0} ({1}%)'\ .format(techniques_observable, round((techniques_observable / technique_count) * 100))) if args.dump_data_sources == True: with open('data_sources.txt', 'w') as fh_data_sources: data_sources = list(data_sources) data_sources.sort() for data_source in data_sources: fh_data_sources.write('{0}\n'.format(data_source)) if args.dump_matching_techniques == True: with open('matching_techniques.txt', 'w') as fh_matching_techniques: matching_techniques = list(matching_techniques) matching_techniques.sort() for data_source in matching_techniques: fh_matching_techniques.write('{0}\n'.format(data_source)) if args.dump_metadata == True: with open('technique_metadata.csv', 'w') as fh_techniques: csvwriter = csv.writer(fh_techniques, quoting=csv.QUOTE_ALL) csvwriter.writerow(['id', 'data_source']) for technique in techniques_with_data_sources: csvwriter.writerow(technique) if args.dump_all_techniques == True: with open('all_techniques.csv', 'w') as fh_all_techniques: csvwriter = csv.writer(fh_all_techniques, quoting=csv.QUOTE_ALL) csvwriter.writerow([ 'technique_id', 'technique_name', 'technique_url', 'technique_description' ]) for technique in all_techniques: # Handle techniques that do not have a description (these probably # should not exist and should be contributed). try: description = technique.description except AttributeError: description = 'NONE' csvwriter.writerow([ technique.external_references[0].external_id, technique.name, technique.external_references[0].url, description ])
def taxii_import(server_url, collection_url): if not (server_url or collection_url): print('Please specify one of --server_url or --collection_url') exit(-1) if server_url: collection_url = _get_collection_url(server_url) if collection_url: print( 'Importing data from collection at: {0:s}'.format(collection_url)) collection = Collection(collection_url) tc_source = TAXIICollectionSource(collection) object_classes = { 'attack-pattern': attack_pattern.AttackPattern, 'campaign': campaign.Campaign, 'course-of-action': course_of_action.CourseOfAction, 'identity': identity.Identity, 'intrusion-set': intrusion_set.IntrusionSet, 'malware': malware.Malware, 'threat-actor': threat_actor.ThreatActor, 'tool': tool.Tool, 'vulnerability': vulnerability.Vulnerability, } all_objects = {} for name, yeti_class in object_classes.items(): print('Fetching', name) stats = { 'updated': 0, 'new': 0, 'skipped': 0, } try: for item in tc_source.query(Filter('type', '=', name)): item_json = json.loads(item.serialize()) obj = yeti_class.get(item.id) if not obj: obj = yeti_class(**item).save() stats['new'] += 1 all_objects[item['id']] = obj if obj.modified >= item.modified or obj.revoked or obj.equals( item_json): stats['skipped'] += 1 elif obj.modified < item.modified: obj.update(item_json) stats['updated'] += 1 except requests.exceptions.HTTPError as error: print(f'HTTPError: {error}') except datastore.DataSourceError as error: print(f'DataSourceError: {error}') print( f"[{name}] New: {stats['new']}, Updated: {stats['updated']}, " f"Skipped: {stats['skipped']}") print('Getting relationships') stats = 0 taxii_filter = Filter('type', '=', 'relationship') for relationship in tc_source.query(taxii_filter): stats += 1 source = all_objects[relationship.source_ref] target = all_objects[relationship.target_ref] source.link_to(target, stix_rel=json.loads(relationship.serialize())) print('Added {0:d} relationships'.format(stats))
def build_iterator(self, limit: int = -1) -> List: """Retrieves all entries from the feed. Returns: A list of objects, containing the indicators. """ indicators: List[Dict] = list() mitre_id_list: Set[str] = set() indicator_values_list: Set[str] = set() external_refs: Set[str] = set() counter = 0 # For each collection for collection in self.collections: # Stop when we have reached the limit defined if 0 < limit <= counter: break # Establish TAXII2 Collection instance collection_url = urljoin(self.base_url, f'stix/collections/{collection.id}/') collection_data = Collection(collection_url, verify=self.verify, proxies=self.proxies) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(collection_data) # Create filters to retrieve content filter_objs = { "Technique": { "name": "attack-pattern", "filter": Filter("type", "=", "attack-pattern") }, "Mitigation": { "name": "course-of-action", "filter": Filter("type", "=", "course-of-action") }, "Group": { "name": "intrusion-set", "filter": Filter("type", "=", "intrusion-set") }, "Malware": { "name": "malware", "filter": Filter("type", "=", "malware") }, "Tool": { "name": "tool", "filter": Filter("type", "=", "tool") }, } # Retrieve content for concept in filter_objs: # Stop when we have reached the limit defined if 0 < limit <= counter: break input_filter = filter_objs[concept]['filter'] try: mitre_data = tc_source.query(input_filter) except Exception: continue # For each item in the MITRE list, add an indicator to the indicators list for mitreItem in mitre_data: # Stop when we have reached the limit defined if 0 < limit <= counter: break mitre_item_json = json.loads(str(mitreItem)) value = None # Try and map a friendly name to the value before the real ID try: externals = [ x['external_id'] for x in mitre_item_json.get( 'external_references', []) if x['source_name'] == 'mitre-attack' and x['external_id'] ] value = externals[0] except Exception: value = None if not value: value = mitre_item_json.get('x_mitre_old_attack_id', None) if not value: value = mitre_item_json.get('id') if mitre_item_json.get('id') not in mitre_id_list: # If the indicator already exists, then append the new data # to the existing indicator. if value in indicator_values_list: # Append data to the original item original_item = [ x for x in indicators if x.get('value') == value ][0] if original_item['rawJSON'].get('id', None): try: original_item['rawJSON'][ 'id'] += f"\n{mitre_item_json.get('id', '')}" except Exception: pass if original_item['rawJSON'].get('created', None): try: original_item['rawJSON'][ 'created'] += f"\n{mitre_item_json.get('created', '')}" except Exception: pass if original_item['rawJSON'].get('modified', None): try: original_item['rawJSON'][ 'modified'] += f"\n{mitre_item_json.get('modified', '')}" except Exception: pass if original_item['rawJSON'].get( 'description', None): try: if not original_item['rawJSON'].get( 'description').startswith("###"): original_item['rawJSON']['description'] = \ f"### {original_item['rawJSON'].get('type')}\n" \ f"{original_item['rawJSON']['description']}" original_item['rawJSON']['description'] += \ f"\n\n_____\n\n### {mitre_item_json.get('type')}\n" \ f"{mitre_item_json.get('description', '')}" except Exception: pass if original_item['rawJSON'].get( 'external_references', None): try: original_item['rawJSON'][ 'external_references'].extend( mitre_item_json.get( 'external_references', [])) except Exception: pass if original_item['rawJSON'].get( 'kill_chain_phases', None): try: original_item['rawJSON'][ 'kill_chain_phases'].extend( mitre_item_json.get( 'kill_chain_phases', [])) except Exception: pass if original_item['rawJSON'].get('aliases', None): try: original_item['rawJSON']['aliases'].extend( mitre_item_json.get('aliases', [])) except Exception: pass else: indicators.append({ "value": value, "score": self.reputation, "type": "MITRE ATT&CK", "rawJSON": mitre_item_json, }) indicator_values_list.add(value) counter += 1 mitre_id_list.add(mitre_item_json.get('id')) # Create a duplicate indicator using the "external_id" from the # original indicator, if the user has selected "includeAPT" as True if self.include_apt: ext_refs = [ x.get('external_id') for x in mitre_item_json.get( 'external_references') if x.get('external_id') and x.get('source_name') != "mitre-attack" ] for x in ext_refs: if x not in external_refs: indicators.append({ "value": x, "score": self.reputation, "type": "MITRE ATT&CK", "rawJSON": mitre_item_json, }) external_refs.add(x) # Finally, map all the fields from the indicator # rawjson to the fields in the indicator for indicator in indicators: indicator['fields'] = dict() for field, value in mitre_field_mapping.items(): try: # Try and map the field value_type = value['type'] value_name = value['name'] if value_type == "list": indicator['fields'][field] = "\n".join( indicator['rawJSON'][value_name]) else: indicator['fields'][field] = indicator['rawJSON'][ value_name] except KeyError: # If the field does not exist in the indicator # then move on pass except Exception as err: demisto.error(f"Error when mapping Mitre Fields - {err}") return indicators
from stix2 import TAXIICollectionSource, Filter from taxii2client import Server, Collection import json from openpyxl import Workbook from openpyxl.styles import Font #enterprise attack source collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) # supply the TAXII2 collection to TAXIICollection tc_source = TAXIICollectionSource(collection) count = 1 def parse_mitre_item(filter_type): global count wb = Workbook() for item in filter_type: name = item[2] print name category = tc_source.query(item) #generate list of deduped list of keys key_list = [] deduped_keys = [] for lst in category: for k, v in lst.items(): key_list.append(k)
def load_taxii(new=False): collection = Collection("https://cti-taxii.mitre.org/stix/collections/" + domainToTaxiiCollectionId[domain]) data_store = TAXIICollectionSource(collection) parse_subtechniques(data_store, new) return load_datastore(data_store)
# Import needed libraries from stix2 import TAXIICollectionSource, Filter from taxii2client.v20 import Server, Collection import json import re from openpyxl import Workbook from openpyxl.styles import Font from openpyxl.styles import Alignment #__________________________________________________________________________________________________________________________ # Enterprise attack source collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) # Supply the TAXII2 collection to TAXIICollection tc_source = TAXIICollectionSource(collection) # Filter to only techniques filt = Filter('type', '=', 'attack-pattern') techniques = tc_source.query([filt]) #__________________________________________________________________________________________________________________________ # Generate list of all kill chain phases and put them in proper order kc_list = [] for technique in techniques: for k, v in technique.items(): if k == "kill_chain_phases": for i in v: for k, v in i.items(): kc_list.append(v)
async def insert_attack_stix_data(self): """ Function to pull stix/taxii information and insert in to the local db :return: status code """ logging.info('Downloading ATT&CK data from STIX/TAXII...') attack = {} collection = Collection("https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/") tc_source = TAXIICollectionSource(collection) filter_objs = {"techniques": Filter("type", "=", "attack-pattern"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship")} for key in filter_objs: attack[key] = tc_source.query(filter_objs[key]) references = {} # add all of the patterns and dictionary keys/values for each technique and software for i in attack["techniques"]: references[i["id"]] = {"name": i["name"], "id": i["external_references"][0]["external_id"], "example_uses": [], "description": i['description'].replace('<code>', '').replace('</code>', '').replace( '\n', '').encode('ascii', 'ignore').decode('ascii'), "similar_words": [i["name"]]} for i in attack["relationships"]: if i["relationship_type"] == 'uses': if 'attack-pattern' in i["target_ref"]: use = i["description"] # remove unnecessary strings, fix unicode errors use = use.replace('<code>', '').replace('</code>', '').replace('"', "").replace(',', '').replace( '\t', '').replace(' ', ' ').replace('\n', '').encode('ascii', 'ignore').decode('ascii') find_pattern = re.compile('\[.*?\]\(.*?\)') # get rid of att&ck reference (name)[link to site] m = find_pattern.findall(use) if len(m) > 0: for j in m: use = use.replace(j, '') if use[0:2] == '\'s': use = use[3:] elif use[0] == ' ': use = use[1:] # combine all the examples to one list references[i["target_ref"]]["example_uses"].append(use) for i in attack["malware"]: if 'description' not in i: # some software do not have description, example: darkmoon https://attack.mitre.org/software/S0209 continue else: references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i["description"], "examples": [], "example_uses": [], "similar_words": [i["name"]]} for i in attack["tools"]: references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i["description"], "examples": [], "example_uses": [], "similar_words": [i["name"]]} attack_data = references logging.info("Finished...now creating the database.") cur_uids = await self.dao.get('attack_uids') if await self.dao.get('attack_uids') else [] cur_items = [i['uid'] for i in cur_uids] for k, v in attack_data.items(): if k not in cur_items: await self.dao.insert('attack_uids', dict(uid=k, description=defang_text(v['description']), tid=v['id'], name=v['name'])) if 'regex_patterns' in v: [await self.dao.insert('regex_patterns', dict(uid=k, regex_pattern=defang_text(x))) for x in v['regex_patterns']] if 'similar_words' in v: [await self.dao.insert('similar_words', dict(uid=k, similar_word=defang_text(x))) for x in v['similar_words']] if 'false_negatives' in v: [await self.dao.insert('false_negatives', dict(uid=k, false_negative=defang_text(x))) for x in v['false_negatives']] if 'false_positives' in v: [await self.dao.insert('false_positives', dict(uid=k, false_positive=defang_text(x))) for x in v['false_positives']] if 'true_positives' in v: [await self.dao.insert('true_positives', dict(uid=k, true_positive=defang_text(x))) for x in v['true_positives']] if 'example_uses' in v: [await self.dao.insert('true_positives', dict(uid=k, true_positive=defang_text(x))) for x in v['example_uses']] logging.info('[!] DB Item Count: {}'.format(len(await self.dao.get('attack_uids'))))
from taxii2client import Server from stix2 import CustomObject, properties, TAXIICollectionSource from taxii2client import Collection from stix2 import Filter import sys discovery_url = sys.argv[1] poll_url = sys.argv[2] #descovery server = Server(discovery_url, 'user1', 'Password1') print(server.title) #poll collection = Collection(poll_url, 'user1', 'Password1') tc_source = TAXIICollectionSource(collection) f1 = Filter("type", "=", "indicator") indicators = tc_source.query([f1]) for indicator in indicators: print(indicator)
from stix2 import TAXIICollectionSource, Filter from taxii2client.v20 import Collection import argparse # Establish TAXII2 Collection instance for Enterprise ATT&CK collection collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) # Supply the collection to TAXIICollection tc_src = TAXIICollectionSource(collection) def data_sources(): """returns all data sources in Enterprise ATT&CK""" all_data_srcs = [] # Get all techniques in Enterprise ATT&CK techniques = tc_src.query([Filter("type", "=", "attack-pattern")]) # Get all data sources in Enterprise ATT&CK for tech in techniques: if 'x_mitre_data_sources' in tech: all_data_srcs += [ data_src for data_src in tech.x_mitre_data_sources if data_src not in all_data_srcs ] return all_data_srcs
def test_get_stix2_object(collection): tc_sink = TAXIICollectionSource(collection) objects = tc_sink.get("indicator--00000000-0000-4000-8000-000000000001") assert objects
def get_tactic_techniques(self, tactic_name=None, tactic_id=None): """ Get all the techniques for a give tactic Reference: https://github.com/mitre/cti/blob/master/USAGE.md :param tactic_name: tactic name :param tactic_id: tactic ID :return: techs """ if self.attack_server is None: self.connect_server() # # STIX type for technique is attack-pattern # tech_filter = Filter("type", "=", "attack-pattern") # # Find name if id is given # if tactic_name is None: if tactic_id is not None: tactic_name = MitreAttackTactic.get_name(tactic_id) # # Not found in MITRE document. We actually need to use lower case for the tactic # name, and also replace space with -. # t_name = tactic_name.replace(' ', '-') t_name = t_name.lower() tactic_filter = Filter("kill_chain_phases.phase_name", "=", t_name) # # Only look for Enterprise ATT&CK at this point # collection_title = "Enterprise ATT&CK" collection = self.collection_dict[collection_title] tc_source = TAXIICollectionSource(collection) mitre_techs = tc_source.query([tech_filter, tactic_filter]) # # The returned AttackPattern is not serializable. Pick the fields we want # techs = [] for mitre_tech in mitre_techs: refs = [] mitre_tech_id = "" for r in mitre_tech["external_references"]: ref = {"url": r.get("url", "")} if r.get("source_name", None) == "mitre-attack": mitre_tech_id = r.get("external_id", "") refs.append(ref) tech = { "name": mitre_tech.get("name", ""), "description": mitre_tech.get("description", ""), "external_references": refs, "x_mitre_detection": mitre_tech.get("x_mitre_detection", ""), "mitre_tech_id": mitre_tech_id } techs.append(tech) return techs
def build_iterator(self, create_relationships=False, is_up_to_6_2=True, limit: int = -1): """Retrieves all entries from the feed. Returns: A list of objects, containing the indicators. """ indicators: List[Dict] = list() mitre_id_list: Set[str] = set() mitre_relationships_list = [] id_to_name: Dict = {} mitre_id_to_mitre_name: Dict = {} counter = 0 # For each collection for collection in self.collections: # fetch only enterprise objects if collection.id != ENTERPRISE_COLLECTION_ID: continue # Stop when we have reached the limit defined if 0 < limit <= counter: break # Establish TAXII2 Collection instance collection_url = urljoin(self.base_url, f'stix/collections/{collection.id}/') collection_data = Collection(collection_url, verify=self.verify, proxies=self.proxies) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(collection_data) for concept in FILTER_OBJS: if 0 < limit <= counter: break input_filter = FILTER_OBJS[concept]['filter'] try: mitre_data = tc_source.query(input_filter) except Exception: continue for mitre_item in mitre_data: if 0 < limit <= counter: break mitre_item_json = json.loads(str(mitre_item)) if mitre_item_json.get('id') not in mitre_id_list: value = mitre_item_json.get('name') item_type = get_item_type(mitre_item_json.get('type'), is_up_to_6_2) if item_type == 'Relationship' and create_relationships: if mitre_item_json.get('relationship_type') == 'revoked-by': continue mitre_relationships_list.append(mitre_item_json) else: if is_indicator_deprecated_or_revoked(mitre_item_json): continue id_to_name[mitre_item_json.get('id')] = value indicator_obj = self.create_indicator(item_type, value, mitre_item_json) add_obj_to_mitre_id_to_mitre_name(mitre_id_to_mitre_name, mitre_item_json) indicators.append(indicator_obj) counter += 1 mitre_id_list.add(mitre_item_json.get('id')) return indicators, mitre_relationships_list, id_to_name, mitre_id_to_mitre_name
from stix2 import TAXIICollectionSource, Filter from taxii2client import Server server = Server("https://cti-taxii.mitre.org/taxii/") api_root = server.api_roots[0] # # Three collections: Enterprise ATT&CK, PRE-ATT&CK, MOBILE # for collection in api_root.collections: print(collection.title + ": " + collection.id) #collection = Collection("https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/") # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(collection) # Create filters to retrieve content from Enterprise ATT&CK filter_objs = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship") } # filter for tactics #fltr = Filter("type", "=", "tactic") #tactics = tc_source.query(fltr)
def cti_create(collection, client): cti = client['cti'] cve = cti['cve'] cwe = cti['cwe'] capec = cti['capec'] attack_techniques = cti['attack_techniques'] attack_relationships = cti['attack_relationships'] attack_groups = cti['attack_groups'] attack_software = cti['attack_software'] attack_mitigations = cti['attack_mitigations'] misp_threat_actor = cti['misp_threat_actor'] if collection == "CVE": files = [ f for f in os.listdir(NVD_PATH) if os.path.isfile(os.path.join(NVD_PATH, f)) ] files.sort() for file in files: archive = zipfile.ZipFile(os.path.join(NVD_PATH, file), 'r') jsonfile = archive.open(archive.namelist()[0]) nvd_dict = json.loads(jsonfile.read()) for element in nvd_dict['CVE_Items']: cve.insert_one(element) jsonfile.close() return elif collection == "CWE": files = [ f for f in os.listdir(CWE_PATH) if os.path.isfile(os.path.join(CWE_PATH, f)) ] files.sort() for file in files: archive = zipfile.ZipFile(os.path.join(CWE_PATH, file), 'r') jsonfile = archive.open(archive.namelist()[0]) cwe_dict = xmltodict.parse(jsonfile.read()) cwe_list = cwe_dict['Weakness_Catalog']['Weaknesses']['Weakness'] for element in cwe_list: cwe.insert_one(json.loads( json.dumps(element).replace("@", ""))) jsonfile.close() return elif collection == "CAPEC": files = [ f for f in os.listdir(CAPEC_PATH) if os.path.isfile(os.path.join(CAPEC_PATH, f)) ] files.sort() for file in files: archive = zipfile.ZipFile(os.path.join(CAPEC_PATH, file), 'r') jsonfile = archive.open(archive.namelist()[0]) capec_dict = xmltodict.parse(jsonfile.read()) capec_list = capec_dict['Attack_Pattern_Catalog'][ 'Attack_Patterns']['Attack_Pattern'] for element in capec_list: capec.insert_one( json.loads(json.dumps(element).replace("@", ""))) jsonfile.close() return elif collection == "ATTACK": technique_helper = [] software_helper = [] attack_id = {} for collection in API_ROOT.collections: attack_id[collection.title] = collection.id attack_collection = {} enterprise_attack = Collection(MITRE_STIX + attack_id["Enterprise ATT&CK"]) pre_attack = Collection(MITRE_STIX + attack_id["PRE-ATT&CK"]) enterprise_attack_source = TAXIICollectionSource(enterprise_attack) pre_attack_source = TAXIICollectionSource(pre_attack) teacher = dict((requests.get(ATTACK_TEACHER).json())) teacher_dict = {} for technique in teacher['techniques']: if technique["color"] == RED: teacher_dict[technique["techniqueID"]] = '5' elif technique["color"] == ORANGE: teacher_dict[technique["techniqueID"]] = '4' elif technique["color"] == YELLOW: teacher_dict[technique["techniqueID"]] = '3' elif technique["color"] == GREEN: teacher_dict[technique["techniqueID"]] = '2' elif technique["color"] == BLUE: teacher_dict[technique["techniqueID"]] = '1' else: teacher_dict[technique["techniqueID"]] = 'Unknown' filter_objs = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship") } for key in filter_objs: attack_collection[key] = [] try: attack_collection[key] += enterprise_attack_source.query( filter_objs[key]) except: pass try: attack_collection[key] += pre_attack_source.query( filter_objs[key]) except: pass for entity in attack_collection["relationships"]: attack_relationships.insert_one(dict(entity)) for entity in attack_collection["techniques"]: dict_entity = dict(entity) for references in dict_entity['external_references']: if 'attack' in references['source_name']: attack_id = references['external_id'] if attack_id in teacher_dict.keys(): dict_entity['level'] = teacher_dict[attack_id] else: dict_entity['level'] = 'Unknown' technique_helper.append(dict_entity) attack_techniques.insert_one(dict_entity) for entity in attack_collection["malware"]: dict_entity = dict(entity) sw_id = dict_entity['id'] techniques = [] measured_techniques = 0 ttp_score_sum = 0 for relationship in attack_collection["relationships"]: dict_rel = dict(relationship) if dict_rel['source_ref'] == sw_id: if 'attack-pattern' in dict_rel['target_ref']: techniques.append(dict_rel['target_ref']) elif dict_rel['target_ref'] == sw_id: if 'attack-pattern' in dict_rel['source_ref']: techniques.append(dict_rel['source_ref']) for ttp_id in techniques: for technique in technique_helper: if technique['id'] == ttp_id and technique[ 'level'] != "Unknown": measured_techniques += 1 ttp_score_sum += int(technique['level']) if measured_techniques == 0: dict_entity['score'] = 'Unknown' else: dict_entity['score'] = round( ttp_score_sum / measured_techniques, 2) software_helper.append(dict_entity) attack_software.insert_one(dict_entity) for entity in attack_collection["tools"]: dict_entity = dict(entity) sw_id = dict_entity['id'] techniques = [] measured_techniques = 0 ttp_score_sum = 0 for relationship in attack_collection["relationships"]: dict_rel = dict(relationship) if dict_rel['source_ref'] == sw_id: if 'attack-pattern' in dict_rel['target_ref']: techniques.append(dict_rel['target_ref']) elif dict_rel['target_ref'] == sw_id: if 'attack-pattern' in dict_rel['source_ref']: techniques.append(dict_rel['source_ref']) for ttp_id in techniques: for technique in technique_helper: if technique['id'] == ttp_id and technique[ 'level'] != "Unknown": measured_techniques += 1 ttp_score_sum += int(technique['level']) if measured_techniques == 0: dict_entity['score'] = 'Unknown' else: dict_entity['score'] = round( ttp_score_sum / measured_techniques, 2) software_helper.append(dict_entity) attack_software.insert_one(dict_entity) for entity in attack_collection["groups"]: dict_entity = dict(entity) gr_id = dict_entity['id'] measured_techniques = 0 ttp_score_sum = 0 used_techniques = [] measured_software = 0 sw_score_sum = 0 used_software = [] for relationship in attack_collection["relationships"]: dict_rel = dict(relationship) if dict_rel['source_ref'] == gr_id: if 'attack-pattern' in dict_rel['target_ref']: used_techniques.append(dict_rel['target_ref']) elif dict_rel['target_ref'] == gr_id: if 'attack-pattern' in dict_rel['source_ref']: used_techniques.append(dict_rel['source_ref']) for ttp_id in used_techniques: for technique in technique_helper: if technique['id'] == ttp_id and technique[ 'level'] != "Unknown": measured_techniques += 1 ttp_score_sum += float(technique['level']) if measured_techniques == 0: dict_entity['skill_level'] = 'Unknown' else: dict_entity['skill_level'] = float(ttp_score_sum / measured_techniques) for relationship in attack_collection["relationships"]: dict_rel = dict(relationship) if dict_rel['source_ref'] == sw_id: if 'malware' in dict_rel[ 'target_ref'] or 'tool' in dict_rel['target_ref']: used_software.append(dict_rel['target_ref']) elif dict_rel['target_ref'] == sw_id: if 'malware' in dict_rel[ 'target_ref'] or 'tool' in dict_rel['target_ref']: used_software.append(dict_rel['source_ref']) for sw_id in used_software: for software in software_helper: if software[ 'id'] == sw_id and software['score'] != "Unknown": measured_software += 1 sw_score_sum += float(software['score']) if measured_software == 0: dict_entity['opportunity'] = 'Unknown' else: dict_entity['opportunity'] = float(sw_score_sum / measured_software) if (dict_entity['skill_level'] == "Unknown") and ( dict_entity['opportunity'] != 'Unknown'): dict_entity['skill_level'] = dict_entity['opportunity'] if (dict_entity['opportunity'] == "Unknown") and ( dict_entity['skill_level'] != 'Unknown'): dict_entity['opportunity'] = dict_entity['skill_level'] if dict_entity['opportunity'] != 'Unknown' and dict_entity[ 'skill_level'] != 'Unknown': dict_entity['skill_level'] = int( modf(OWASP * dict_entity['skill_level'])[1]) dict_entity['opportunity'] = int( modf(OWASP * dict_entity['opportunity'])[1]) stix_group_id = dict_entity['id'].split("--")[1] attack_group_id = None for external_reference in dict_entity['external_references']: if 'attack' in external_reference['source_name']: attack_group_id = external_reference['external_id'] break threat_actors = [] try: misp_list = misp_threat_actor.distinct( 'uuid', {"meta.refs": GROUP_URL(attack_group_id) }) + misp_threat_actor.distinct( 'uuid', {"related.dest-uuid": stix_group_id}) for threat_actor in misp_list: if threat_actor not in threat_actors: threat_actors.append(threat_actor) except: pass motive_list = [] size_list = [] for threat_actor in threat_actors: threat_actor_list = misp_threat_actor.find( {'uuid': threat_actor}) for threat_actor_dict in threat_actor_list: motive_list.append(int(threat_actor_dict['motive'])) size_list.append(int(threat_actor_dict['size'])) if len(motive_list) > 0: dict_entity['motive'] = str( sum(motive_list) / len(motive_list)) else: dict_entity['motive'] = '0' if dict_entity['motive'] == '0': dict_entity['motive'] = 'Unknown' if len(size_list) > 0: dict_entity['size'] = str(sum(size_list) / len(size_list)) else: dict_entity['size'] = '0' if dict_entity['size'] == '0': dict_entity['size'] = 'Unknown' attack_groups.insert_one(dict_entity) for entity in attack_collection["mitigations"]: attack_mitigations.insert_one(dict(entity)) return elif collection == "MISP": with open(os.path.join(MISP_PATH, MISP_THREAT_ACTOR), 'r', encoding="UTF-8") as misp_file: threat_actors = json.loads(misp_file.read()) for threat_actor in threat_actors['values']: dict_threat_actor = threat_actor_category(dict(threat_actor)) misp_threat_actor.insert_one(dict_threat_actor) misp_file.close() else: print("Collection type not set") return
attack_dict = {} attack_stix_root_url = "https://cti-taxii.mitre.org/stix/collections/" attack_enterprise_collection_url = attack_stix_root_url + api_root.collections[ 0].id + '/' # Print ATT&CK Enterprise Collection URL print("ATT&CK Enterprise collection URL: " + attack_enterprise_collection_url) print( "------------------------------------------------------------------------") #Establish TAXII2 Collection instance for Enterprise ATT&CK collection Enterprise_collection = Collection(attack_enterprise_collection_url) #Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(Enterprise_collection) print("TAXII Collection Source :" + str(type(tc_source))) print( "------------------------------------------------------------------------") # Create filters to retrieve content from Enterprise ATT&CK based on type # filter_objs = {"techniques": Filter("type", "=", "attack-pattern"), # Technique # "mitigations": Filter("type", "=", "course-of-action"), # Mitigation # "groups": Filter("type", "=", "intrusion-set"), # Group # "malware": Filter("type", "=", "malware"), # Software # "tools": Filter("type", "=", "tool"), # Software # "relationships": Filter("type", "=", "relationship") # } #