def __init__(self, local_path=None): if local_path is not None and os.path.isdir(os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR)) \ and os.path.isdir(os.path.join(local_path, PRE_ATTCK_LOCAL_DIR)) \ and os.path.isdir(os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR)): self.TC_ENTERPRISE_SOURCE = FileSystemSource( os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR)) self.TC_PRE_SOURCE = FileSystemSource( os.path.join(local_path, PRE_ATTCK_LOCAL_DIR)) self.TC_MOBILE_SOURCE = FileSystemSource( os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR)) else: ENTERPRISE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + ENTERPRISE_ATTCK + "/") PRE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + PRE_ATTCK + "/") MOBILE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + MOBILE_ATTCK + "/") self.TC_ENTERPRISE_SOURCE = TAXIICollectionSource( ENTERPRISE_COLLECTION) self.TC_PRE_SOURCE = TAXIICollectionSource(PRE_COLLECTION) self.TC_MOBILE_SOURCE = TAXIICollectionSource(MOBILE_COLLECTION) self.COMPOSITE_DS = CompositeDataSource() self.COMPOSITE_DS.add_data_sources([ self.TC_ENTERPRISE_SOURCE, self.TC_PRE_SOURCE, self.TC_MOBILE_SOURCE ])
def load_taxii(new=False): collection = Collection( "https://cti-taxii.mitre.org/stix/collections/" + domainToTaxiiCollectionId[domain]) data_store = TAXIICollectionSource(collection) parse_subtechniques(data_store, new) return load_datastore(data_store)
def build_iterator(self, create_relationships=False, is_up_to_6_2=True, limit: int = -1): """Retrieves all entries from the feed. Returns: A list of objects, containing the indicators. """ indicators: List[Dict] = list() mitre_id_list: Set[str] = set() mitre_relationships_list = [] id_to_name: Dict = {} counter = 0 # For each collection for collection in self.collections: # Stop when we have reached the limit defined if 0 < limit <= counter: break # Establish TAXII2 Collection instance collection_url = urljoin(self.base_url, f'stix/collections/{collection.id}/') collection_data = Collection(collection_url, verify=self.verify, proxies=self.proxies) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(collection_data) for concept in FILTER_OBJS: if 0 < limit <= counter: break input_filter = FILTER_OBJS[concept]['filter'] try: mitre_data = tc_source.query(input_filter) except Exception: continue for mitre_item in mitre_data: if 0 < limit <= counter: break mitre_item_json = json.loads(str(mitre_item)) if mitre_item_json.get('id') not in mitre_id_list: value = mitre_item_json.get('name') item_type = get_item_type(mitre_item_json.get('type'), is_up_to_6_2) if item_type == 'Relationship' and create_relationships: if mitre_item_json.get('relationship_type') == 'revoked-by': continue mitre_relationships_list.append(mitre_item_json) else: if is_indicator_deprecated_or_revoked(mitre_item_json): continue id_to_name[mitre_item_json.get('id')] = value indicator_obj = self.create_indicator(item_type, value, mitre_item_json) indicators.append(indicator_obj) counter += 1 mitre_id_list.add(mitre_item_json.get('id')) return indicators, mitre_relationships_list, id_to_name
def test_collection_missing_can_write_property(collection_dict): with pytest.raises(ValidationError) as excinfo: collection_dict.pop("can_write") Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/", user="******", password="******", verify=False, collection_info=collection_dict) assert "No 'can_write' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value)
def attnck_taxii(): # Initialize dictionary to hold Enterprise ATT&CK content ent_attack = {} pre_attack = {} # Establish TAXII2 Collection instance for Enterprise ATT&CK collection ENTERPRISE_ATTCK = "95ecc380-afe9-11e4-9b6c-751b66dd541e" PRE_ATTCK = "062767bd-02d2-4b72-84ba-56caef0f8658" MOBILE_ATTCK = "2f669986-b40b-4423-b720-4396ca6a462b" enterprise_attack_collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) pre_attack_collection = Collection( "https://cti-taxii.mitre.org/stix/collections/062767bd-02d2-4b72-84ba-56caef0f8658/" ) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(enterprise_attack_collection) tc_source2 = TAXIICollectionSource(pre_attack_collection) # Create filters to retrieve content from Enterprise ATT&CK based on type filter_objs = { "techniques": Filter("type", "=", "attack-pattern"), "mitigations": Filter("type", "=", "course-of-action"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship") } filter_objs_pre = { "techniques": Filter("type", "=", "attack-pattern"), "groups": Filter("type", "=", "intrusion-set"), "relationships": Filter("type", "=", "relationship") } # Retrieve all Enterprise ATT&CK content for key in filter_objs: ent_attack[key] = tc_source.query(filter_objs[key]) for key in filter_objs_pre: pre_attack[key] = tc_source2.query(filter_objs[key]) return ent_attack, pre_attack
def test_collection_missing_trailing_slash(): set_collection_response() collection = Collection(COLLECTION_URL[:-1]) responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE, status=200, content_type="%s; charset=utf-8" % MEDIA_TYPE_STIX_V20) response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61") indicator = response["objects"][0] assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61"
def view_new_ioc(): collection = Collection( 'http://127.0.0.1:5000/trustgroup1/collections/365fed99-08fa-fdcd-a1b3-fb247eb41d01', user='******', password='******') new_objects = collection.get_objects() print(new_objects) return prepare_response({'data': new_objects})
def get_mitre_data_by_filter(client, mitre_filter): for collection in client.collections: collection_url = urljoin(client.base_url, f'stix/collections/{collection.id}/') collection_data = Collection(collection_url, verify=client.verify, proxies=client.proxies) tc_source = TAXIICollectionSource(collection_data) if tc_source.query(mitre_filter): mitre_data = tc_source.query(mitre_filter)[0] return mitre_data return {}
def get_data(self): collections = { "enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e", "pre_attack": "062767bd-02d2-4b72-84ba-56caef0f8658", "mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b" } collection = Collection( f"https://cti-taxii.mitre.org/stix/collections/{collections['enterprise_attack']}/" ) src = TAXIICollectionSource(collection) return src
def build_taxii_source(collection_name): """Downloads latest Enterprise or Mobile ATT&CK content from MITRE TAXII Server.""" # Establish TAXII2 Collection instance for Enterprise ATT&CK collection collection_map = { "enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e", "mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b" } collection_url = "https://cti-taxii.mitre.org/stix/collections/" + collection_map[ collection_name] + "/" collection = Collection(collection_url) taxii_ds = TAXIICollectionSource(collection) # Create an in-memory source (to prevent multiple web requests) return MemorySource(stix_data=taxii_ds.query())
def establish_connection(collection: str): """establish a connection with the TAXII server. arguments: collection: string, the url of the collection with which to connect. returns: connection to the taxii collection """ # Establish TAXII2 Collection instance for Enterprise ATT&CK collection collection = Collection(collection) # Supply the collection to TAXIICollection tc_src = TAXIICollectionSource(collection) return tc_src
def create_ioc(): collection = Collection( 'http://127.0.0.1:5000/trustgroup1/collections/365fed99-08fa-fdcd-a1b3-fb247eb41d01', user='******', password='******') name = request.form.get('name') rtype = request.form.get('rtype') pattern = request.form.get('pattern') labels = request.form.get('labels') # return prepare_response({'status': 'error', 'message': 'Something went wrong while trying to save IoC'}) if rtype == "indicator": indicator = Indicator(name=name, pattern=pattern, labels=labels) bundle = Bundle([indicator]).serialize() collection.add_objects(bundle) elif rtype == "malware": malware = Malware(name=name, labels=labels) bundle = Bundle([malware]).serialize() collection.add_objects(bundle) elif rtype == "attack-pattern": attack_pattern = AttackPattern(name=name, pattern=pattern, labels=labels) bundle = Bundle([attack_pattern]).serialize() collection.add_objects(bundle) elif rtype == "identity": identity = Identity(name=name, labels=labels) bundle = Bundle([identity]).serialize() collection.add_objects(bundle) elif rtype == "threat-actor": threat_actor = ThreatActor(name=name, pattern=pattern, labels=labels) bundle = Bundle([threat_actor]).serialize() collection.add_objects(bundle) elif rtype == "tool": tool = Tool(name=name, pattern=pattern, labels=labels) bundle = Bundle([tool]).serialize() collection.add_objects(bundle) elif rtype == "vulnerability": vuln = Vulnerability(name=name, pattern=pattern, labels=labels) bundle = Bundle([vuln]).serialize() collection.add_objects(bundle) return prepare_response({ 'status': 'success', 'message': 'IoC was saved successfully' })
def get_mitre_value_from_id(client, args): attack_ids = argToList(args.get('attack_ids', [])) attack_values = [] for attack_id in attack_ids: collection_id = f"stix/collections/{ENTERPRISE_COLLECTION_ID}/" collection_url = urljoin(client.base_url, collection_id) collection_data = Collection(collection_url, verify=client.verify, proxies=client.proxies) tc_source = TAXIICollectionSource(collection_data) attack_pattern_obj = tc_source.query([ Filter("external_references.external_id", "=", attack_id), Filter("type", "=", "attack-pattern") ]) attack_pattern_name = attack_pattern_obj[0][ 'name'] if attack_pattern_obj else None if attack_pattern_name and len(attack_id) > 5: # sub-technique parent_name = tc_source.query([ Filter("external_references.external_id", "=", attack_id[:5]), Filter("type", "=", "attack-pattern") ])[0]['name'] attack_pattern_name = f'{parent_name}: {attack_pattern_name}' if attack_pattern_name: attack_values.append({ 'id': attack_id, 'value': attack_pattern_name }) if attack_values: return CommandResults(outputs=attack_values, outputs_key_field='id', outputs_prefix='MITREATTACK', readable_output=tableToMarkdown( 'MITRE ATTACK Attack Patterns values:', attack_values)) return CommandResults( readable_output=f'MITRE ATTACK Attack Patterns values: ' f'No Attack Patterns found for {attack_ids}.')
def get_taxii_collection_source(cls): # あらかじめ ATT&CK の TAXIICOllectionSourceを取得する try: proxies = System.get_request_proxies() attck_txs = Server("%s/taxii/" % (cls.ATT_CK_TAXII_SERVER), proxies=proxies) print('>>> attck_txs: ' + str(attck_txs)) api_root = attck_txs.api_roots[0] for collection in api_root.collections: if collection.title == cls.COLLCETION_TITLE: collection = Collection( "%s/stix/collections/%s/" % (cls.ATT_CK_TAXII_SERVER, collection.id), proxies=proxies) return TAXIICollectionSource(collection) return None except Exception: import traceback traceback.print_exc() return None
def __init__(self, source='taxii', local=None): """ Initialization - Creates a matrix generator object :param server: Source to utilize (taxii or local) :param local: string path to local cache of stix data """ self.convert_data = {} if source.lower() not in ['taxii', 'local']: print( '[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii" or "local"' .format(source)) raise ValueError if source.lower() == 'taxii': self.server = Server('https://cti-taxii.mitre.org/taxii') self.api_root = self.server.api_roots[0] self.collections = dict() for collection in self.api_root.collections: if collection.title != "PRE-ATT&CK": tc = Collection( 'https://cti-taxii.mitre.org/stix/collections/' + collection.id) self.collections[collection.title.split(' ') [0].lower()] = TAXIICollectionSource(tc) elif source.lower() == 'local': if local is not None: hd = MemoryStore() if 'mobile' in local.lower(): self.collections['mobile'] = hd.load_from_file(local) else: self.collections['enterprise'] = hd.load_from_file(local) else: print( '[MatrixGen] - "local" source specified, but path to local source not provided' ) raise ValueError self.matrix = {} self._build_matrix()
def collection(): """Default Collection object""" # The collection response is needed to get information about the collection set_collection_response() return Collection(COLLECTION_URL)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--matrix", type=str, action="store", default="enterprise", help="Matrix to query (enterprise, mobile, pre)") # Techniques parser.add_argument( "--dump-all-techniques", action="store_true", help="Dump a CSV file with technique,subtechnique,name") # Data Sources parser.add_argument( "--dump-data-sources", action="store_true", help="Dump a list of unique data sources to data_sources.txt") parser.add_argument( "--dump-metadata", action="store_true", help= "Dump a CSV file technique-metadata.csv, containing unique technique-metadata pairings." ) parser.add_argument( "--dump-matching-techniques", action="store_true", help= "Dump techniques that map to match-data-sources to matching-techniques.txt" ) parser.add_argument( "--match-data-sources", type=str, action="store", help= "A file containing a list of data sources that to match against techniques." ) args = parser.parse_args() match_data_sources = None if args.match_data_sources: match_data_sources = parse_data_source_list(args.match_data_sources) args.matrix = args.matrix.lower() if args.matrix == 'pre': matrix = "062767bd-02d2-4b72-84ba-56caef0f8658" elif args.matrix == 'mobile': matrix = "2f669986-b40b-4423-b720-4396ca6a462b" elif args.matrix == 'enterprise': matrix = "95ecc380-afe9-11e4-9b6c-751b66dd541e" # Initialize dictionary to hold Enterprise ATT&CK content attack = {} # Establish TAXII2 Collection instance for Enterprise ATT&CK collection = Collection("https://cti-taxii.mitre.org/stix/collections/{0}/"\ .format(matrix)) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(collection) # Create filters to retrieve content from Enterprise ATT&CK filter_objs = {"techniques": Filter("type", "=", "attack-pattern")} # Retrieve all Enterprise ATT&CK content for key in filter_objs: attack[key] = tc_source.query(filter_objs[key]) all_techniques = attack["techniques"] all_techniques = remove_revoked_deprecated(all_techniques) technique_count = 0 techniques_without_data_source = 0 techniques_observable = 0 techniques_with_data_sources = [] data_sources = set() matching_techniques = set() for technique in all_techniques: technique_count += 1 technique_id = technique['external_references'][0]['external_id'] if 'x_mitre_data_sources' in technique.keys(): if match_data_sources is not None: if data_source_match(technique['x_mitre_data_sources'], match_list=match_data_sources) == True: techniques_observable += 1 if args.dump_matching_techniques == True: matching_techniques.add(technique_id) if args.dump_data_sources == True: [ data_sources.add(data_source) for data_source in technique['x_mitre_data_sources'] ] if args.dump_metadata == True: [ techniques_with_data_sources.append( (technique_id, data_source)) for data_source in technique['x_mitre_data_sources'] ] else: techniques_without_data_source += 1 # Output files based on input arguments. if match_data_sources is not None: print('Techniques: {0}'.format(technique_count)) print('Techniques Observable: {0} ({1}%)'\ .format(techniques_observable, round((techniques_observable / technique_count) * 100))) if args.dump_data_sources == True: with open('data_sources.txt', 'w') as fh_data_sources: data_sources = list(data_sources) data_sources.sort() for data_source in data_sources: fh_data_sources.write('{0}\n'.format(data_source)) if args.dump_matching_techniques == True: with open('matching_techniques.txt', 'w') as fh_matching_techniques: matching_techniques = list(matching_techniques) matching_techniques.sort() for data_source in matching_techniques: fh_matching_techniques.write('{0}\n'.format(data_source)) if args.dump_metadata == True: with open('technique_metadata.csv', 'w') as fh_techniques: csvwriter = csv.writer(fh_techniques, quoting=csv.QUOTE_ALL) csvwriter.writerow(['id', 'data_source']) for technique in techniques_with_data_sources: csvwriter.writerow(technique) if args.dump_all_techniques == True: with open('all_techniques.csv', 'w') as fh_all_techniques: csvwriter = csv.writer(fh_all_techniques, quoting=csv.QUOTE_ALL) csvwriter.writerow([ 'technique_id', 'technique_name', 'technique_url', 'technique_description' ]) for technique in all_techniques: # Handle techniques that do not have a description (these probably # should not exist and should be contributed). try: description = technique.description except AttributeError: description = 'NONE' csvwriter.writerow([ technique.external_references[0].external_id, technique.name, technique.external_references[0].url, description ])
def build_iterator(self, limit: int = -1) -> List: """Retrieves all entries from the feed. Returns: A list of objects, containing the indicators. """ indicators: List[Dict] = list() mitre_id_list: Set[str] = set() indicator_values_list: Set[str] = set() external_refs: Set[str] = set() counter = 0 # For each collection for collection in self.collections: # Stop when we have reached the limit defined if 0 < limit <= counter: break # Establish TAXII2 Collection instance collection_url = urljoin(self.base_url, f'stix/collections/{collection.id}/') collection_data = Collection(collection_url, verify=self.verify, proxies=self.proxies) # Supply the collection to TAXIICollection tc_source = TAXIICollectionSource(collection_data) # Create filters to retrieve content filter_objs = { "Technique": { "name": "attack-pattern", "filter": Filter("type", "=", "attack-pattern") }, "Mitigation": { "name": "course-of-action", "filter": Filter("type", "=", "course-of-action") }, "Group": { "name": "intrusion-set", "filter": Filter("type", "=", "intrusion-set") }, "Malware": { "name": "malware", "filter": Filter("type", "=", "malware") }, "Tool": { "name": "tool", "filter": Filter("type", "=", "tool") }, } # Retrieve content for concept in filter_objs: # Stop when we have reached the limit defined if 0 < limit <= counter: break input_filter = filter_objs[concept]['filter'] try: mitre_data = tc_source.query(input_filter) except Exception: continue # For each item in the MITRE list, add an indicator to the indicators list for mitreItem in mitre_data: # Stop when we have reached the limit defined if 0 < limit <= counter: break mitre_item_json = json.loads(str(mitreItem)) value = None # Try and map a friendly name to the value before the real ID try: externals = [ x['external_id'] for x in mitre_item_json.get( 'external_references', []) if x['source_name'] == 'mitre-attack' and x['external_id'] ] value = externals[0] except Exception: value = None if not value: value = mitre_item_json.get('x_mitre_old_attack_id', None) if not value: value = mitre_item_json.get('id') if mitre_item_json.get('id') not in mitre_id_list: # If the indicator already exists, then append the new data # to the existing indicator. if value in indicator_values_list: # Append data to the original item original_item = [ x for x in indicators if x.get('value') == value ][0] if original_item['rawJSON'].get('id', None): try: original_item['rawJSON'][ 'id'] += f"\n{mitre_item_json.get('id', '')}" except Exception: pass if original_item['rawJSON'].get('created', None): try: original_item['rawJSON'][ 'created'] += f"\n{mitre_item_json.get('created', '')}" except Exception: pass if original_item['rawJSON'].get('modified', None): try: original_item['rawJSON'][ 'modified'] += f"\n{mitre_item_json.get('modified', '')}" except Exception: pass if original_item['rawJSON'].get( 'description', None): try: if not original_item['rawJSON'].get( 'description').startswith("###"): original_item['rawJSON']['description'] = \ f"### {original_item['rawJSON'].get('type')}\n" \ f"{original_item['rawJSON']['description']}" original_item['rawJSON']['description'] += \ f"\n\n_____\n\n### {mitre_item_json.get('type')}\n" \ f"{mitre_item_json.get('description', '')}" except Exception: pass if original_item['rawJSON'].get( 'external_references', None): try: original_item['rawJSON'][ 'external_references'].extend( mitre_item_json.get( 'external_references', [])) except Exception: pass if original_item['rawJSON'].get( 'kill_chain_phases', None): try: original_item['rawJSON'][ 'kill_chain_phases'].extend( mitre_item_json.get( 'kill_chain_phases', [])) except Exception: pass if original_item['rawJSON'].get('aliases', None): try: original_item['rawJSON']['aliases'].extend( mitre_item_json.get('aliases', [])) except Exception: pass else: indicator_obj = { "value": value, "score": self.reputation, "type": "MITRE ATT&CK", "rawJSON": mitre_item_json, "fields": { "tags": self.tags, } } if self.tlp_color: indicator_obj['fields'][ 'trafficlightprotocol'] = self.tlp_color indicators.append(indicator_obj) indicator_values_list.add(value) counter += 1 mitre_id_list.add(mitre_item_json.get('id')) # Create a duplicate indicator using the "external_id" from the # original indicator, if the user has selected "includeAPT" as True if self.include_apt: ext_refs = [ x.get('external_id') for x in mitre_item_json.get( 'external_references') if x.get('external_id') and x.get('source_name') != "mitre-attack" ] for x in ext_refs: if x not in external_refs: indicator_obj = { "value": x, "score": self.reputation, "type": "MITRE ATT&CK", "rawJSON": mitre_item_json, "fields": { "tags": self.tags, } } if self.tlp_color: indicator_obj['fields'][ 'trafficlightprotocol'] = self.tlp_color indicators.append(indicator_obj) external_refs.add(x) # Finally, map all the fields from the indicator # rawjson to the fields in the indicator for indicator in indicators: indicator['fields'] = dict() for field, value in mitre_field_mapping.items(): try: # Try and map the field value_type = value['type'] value_name = value['name'] if value_type == "list": indicator['fields'][field] = "\n".join( indicator['rawJSON'][value_name]) else: if value_name in ['created', 'modified']: indicator['fields'][ field] = handle_multiple_dates_in_one_field( value_name, indicator['rawJSON'][value_name]) else: indicator['fields'][field] = indicator['rawJSON'][ value_name] except KeyError: # If the field does not exist in the indicator # then move on pass except Exception as err: demisto.error(f"Error when mapping Mitre Fields - {err}") return indicators
# Import needed libraries from stix2 import TAXIICollectionSource, Filter from taxii2client.v20 import Server, Collection import json import re from openpyxl import Workbook from openpyxl.styles import Font from openpyxl.styles import Alignment #__________________________________________________________________________________________________________________________ # Enterprise attack source collection = Collection( "https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/" ) # Supply the TAXII2 collection to TAXIICollection tc_source = TAXIICollectionSource(collection) # Filter to only techniques filt = Filter('type', '=', 'attack-pattern') techniques = tc_source.query([filt]) #__________________________________________________________________________________________________________________________ # Generate list of all kill chain phases and put them in proper order kc_list = [] for technique in techniques: for k, v in technique.items(): if k == "kill_chain_phases": for i in v: for k, v in i.items(): kc_list.append(v)
from taxii2client.v20 import Collection from stix2 import * coll = Collection('http://127.0.0.1:5000/trustgroup1/collections/365fed99-08fa-fdcd-a1b3-fb247eb41d01', user='******', password='******') indicator = Indicator(name="File hash for malware variant", labels=["malicious-activity"], pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']") #print(indicator) bundle = Bundle([indicator]).serialize() coll.add_objects(bundle) obj = coll.get_objects() print(obj)
def __init__(self, source='taxii', resource=None): """ Initialization - Creates a matrix generator object :param source: Source to utilize (taxii, remote, or local) :param resource: string path to local cache of stix data (local) or url of an ATT&CK Workbench (remote) """ self.convert_data = {} self.collections = dict() if source.lower() not in ['taxii', 'local', 'remote']: print( '[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii", "remote" or ' '"local"'.format(source)) raise ValueError if source.lower() == 'taxii': self.server = Server('https://cti-taxii.mitre.org/taxii') self.api_root = self.server.api_roots[0] for collection in self.api_root.collections: if collection.title != "PRE-ATT&CK": tc = Collection( 'https://cti-taxii.mitre.org/stix/collections/' + collection.id) self.collections[collection.title.split(' ') [0].lower()] = TAXIICollectionSource(tc) elif source.lower() == 'local': if resource is not None: hd = MemoryStore() hd.load_from_file(resource) if 'mobile' in resource.lower(): self.collections['mobile'] = hd else: self.collections['enterprise'] = hd else: print( '[MatrixGen] - "local" source specified, but path to local source not provided' ) raise ValueError elif source.lower() == 'remote': if resource is not None: if ':' not in resource[6:]: print( '[MatrixGen] - "remote" source missing port; assuming ":3000"' ) resource += ":3000" if not resource.startswith('http'): resource = 'http://' + resource for dataset in ['enterprise', 'mobile']: hd = MemoryStore() response = requests.get( f"{resource}/api/stix-bundles?domain={dataset}-" f"attack&includeRevoked=true&includeDeprecated=true") response.raise_for_status( ) # ensure we notice bad responses _add(hd, json.loads(response.text), True, None) self.collections[dataset] = hd else: print( f'[MatrixGen] - WARNING: "remote" selected without providing a "resource" url. The use of ' f'"remote" requires the inclusion of a "resource" url to an ATT&CK Workbench instance. No matrix ' f'will be generated...') self.matrix = {} self._build_matrix()
def test_collection_unexpected_kwarg(): with pytest.raises(TypeError): Collection(url="", conn=None, foo="bar")
def bad_writable_collection(): """Collection with 'can_write=true', but the COLLECTION_URL is different from the one in the response""" set_collection_response(response=WRITABLE_COLLECTION) return Collection(COLLECTION_URL)
def writable_collection(): """Collection with 'can_write' set to 'true'.""" set_collection_response(WRITABLE_COLLECTION_URL, WRITABLE_COLLECTION) return Collection(WRITABLE_COLLECTION_URL)
async def insert_attack_stix_data(self): """ Function to pull stix/taxii information and insert in to the local db :return: status code """ logging.info('Downloading ATT&CK data from STIX/TAXII...') attack = {} collection = Collection("https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/") tc_source = TAXIICollectionSource(collection) filter_objs = {"techniques": Filter("type", "=", "attack-pattern"), "groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"), "tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship")} for key in filter_objs: attack[key] = tc_source.query(filter_objs[key]) references = {} # add all of the patterns and dictionary keys/values for each technique and software for i in attack["techniques"]: references[i["id"]] = {"name": i["name"], "id": i["external_references"][0]["external_id"], "example_uses": [], "description": i.get('description', NO_DESC).replace('<code>', '').replace( '</code>', '').replace('\n', '').encode('ascii', 'ignore').decode('ascii'), "similar_words": [i["name"]]} for i in attack["relationships"]: if i["relationship_type"] == 'uses': if 'attack-pattern' in i["target_ref"]: use = i.get('description', NO_DESC) # remove unnecessary strings, fix unicode errors use = use.replace('<code>', '').replace('</code>', '').replace('"', "").replace(',', '').replace( '\t', '').replace(' ', ' ').replace('\n', '').encode('ascii', 'ignore').decode('ascii') find_pattern = re.compile('\[.*?\]\(.*?\)') # get rid of att&ck reference (name)[link to site] m = find_pattern.findall(use) if len(m) > 0: for j in m: use = use.replace(j, '') if use[0:2] == '\'s': use = use[3:] elif use[0] == ' ': use = use[1:] # combine all the examples to one list references[i["target_ref"]]["example_uses"].append(use) for i in attack["malware"]: if 'description' not in i: # some software do not have description, example: darkmoon https://attack.mitre.org/software/S0209 continue else: references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i["description"], "examples": [], "example_uses": [], "similar_words": [i["name"]]} for i in attack["tools"]: references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i.get('description', NO_DESC), "examples": [], "example_uses": [], "similar_words": [i["name"]]} attack_data = references logging.info("Finished...now creating the database.") cur_uids = await self.get_technique_uids() for k, v in attack_data.items(): if k not in cur_uids: await self.dao.insert('attack_uids', dict(uid=k, description=defang_text(v.get('description', NO_DESC)), tid=v['id'], name=v['name'])) if 'regex_patterns' in v: [await self.dao.insert_generate_uid('regex_patterns', dict(attack_uid=k, regex_pattern=defang_text(x))) for x in v['regex_patterns']] if 'similar_words' in v: [await self.dao.insert_generate_uid('similar_words', dict(attack_uid=k, similar_word=defang_text(x))) for x in v['similar_words']] if 'false_negatives' in v: [await self.dao.insert_generate_uid('false_negatives', dict(attack_uid=k, false_negative=defang_text(x))) for x in v['false_negatives']] if 'false_positives' in v: [await self.dao.insert_generate_uid('false_positives', dict(attack_uid=k, false_positive=defang_text(x))) for x in v['false_positives']] if 'true_positives' in v: [await self.dao.insert_generate_uid('true_positives', dict(attack_uid=k, true_positive=defang_text(x))) for x in v['true_positives']] if 'example_uses' in v: [await self.dao.insert_generate_uid('true_positives', dict(attack_uid=k, true_positive=defang_text(x))) for x in v['example_uses']] logging.info('[!] DB Item Count: {}'.format(len(await self.dao.get('attack_uids'))))
def test_collection_with_custom_properties(collection_dict): collection_dict["type"] = "domain" col_obj = Collection(url=WRITABLE_COLLECTION_URL, collection_info=collection_dict) assert len(col_obj.custom_properties) == 1 assert col_obj.custom_properties["type"] == "domain"