def _dump(self, event=None): event_path = os.path.join(self.cur_path, 'misp_events') if not os.path.exists(event_path): os.makedirs(event_path) if not event: to_dump = __sessions__.current.misp_event.event elif isinstance(event, MISPEvent): to_dump = event else: to_dump = MISPEvent() to_dump.load(event) if to_dump.id: filename = str(to_dump.id) elif (__sessions__.is_attached_misp(True) and __sessions__.current.misp_event.current_dump_file): filename = __sessions__.current.misp_event.current_dump_file else: i = 1 while True: filename = 'new_event_{}.json'.format(i) if not os.path.exists(os.path.join(event_path, filename)): break i += 1 path = os.path.join(event_path, filename) with open(path, 'w') as f: f.write(to_dump.to_json()) self.log('success', '{} stored successfully.'.format(filename.rstrip('.json'))) return filename
def create_event(self): if self.args.threat is not None: # Dirty trick to keep consistency in the module: the threat level in the upload # API can go from 0 import to 3 but it is 1 to 4 in the event mgmt API. # It will be fixed in a near future, in the meantime, we do that: self.args.threat += 1 if not self.args.info: self.log('error', 'Info field is required for a new event') info = ' '.join(self.args.info) # Check if the following arguments have been set (and correctly set). If not, take the config values self.args.distrib = self.distribution if self.args.distrib is None else self.args.distrib self.args.sharing = self.sharinggroup if self.args.sharing is None else self.args.sharing if self.args.sharing and self.args.distrib != 4: self.args.sharing = None self.log('info', "Sharing group can only be set if distribution is 4. Clearing set value") misp_event = MISPEvent() misp_event.set_all_values(info=info, distribution=self.args.distrib, sharing_group_id=self.args.sharing, threat_level_id=self.args.threat, analysis=self.args.analysis, date=self.args.date) self._search_local_hashes(misp_event) if self.offline_mode: # New event created locally, no ID __sessions__.current.misp_event.current_dump_file = self._dump() __sessions__.current.misp_event.offline() else: misp_event = self.misp.add_event(json.dumps(misp_event, cls=EncodeUpdate)) if self._has_error_message(misp_event): return __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode)) self._dump()
def load_openioc(openioc): # Takes a opened file, or a string if not has_bs4: raise Exception('You need to install BeautifulSoup: pip install bs4') misp_event = MISPEvent() iocreport = BeautifulSoup(openioc, "html.parser") # Set event fields info = extract_field(iocreport, 'short_description') if info: misp_event.info = info date = extract_field(iocreport, 'authored_date') if date: misp_event.set_date(date) # Set special attributes description = extract_field(iocreport, 'description') if description: if not misp_event.info: misp_event.info = description else: misp_event.add_attribute('comment', description) if not misp_event.info: misp_event.info = 'OpenIOC import' author = extract_field(iocreport, 'authored_by') if author: misp_event.add_attribute('comment', author) misp_event = set_all_attributes(iocreport, misp_event) return misp_event
def create_event(self): if self.args.threat is not None: # Dirty trick to keep consistency in the module: the threat level in the upload # API can go from 0 import to 3 but it is 1 to 4 in the event mgmt API. # It will be fixed in a near future, in the meantime, we do that: self.args.threat += 1 if not self.args.info: self.log('error', 'Info field is required for a new event') info = ' '.join(self.args.info) misp_event = MISPEvent() misp_event.set_all_values(info=info, distribution=self.args.distrib, threat_level_id=self.args.threat, analysis=self.args.analysis, date=self.args.date) self._search_local_hashes(misp_event) if self.offline_mode: # New event created locally, no ID __sessions__.current.misp_event.current_dump_file = self._dump() __sessions__.current.misp_event.offline() else: misp_event = self.misp.add_event(json.dumps(misp_event, cls=EncodeUpdate)) if self._has_error_message(misp_event): return __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode)) self._dump()
def test_eventObject(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) misp_event = MISPEvent(pymisp.describe_types) misp_event.load(open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r').read()) json.dumps(misp_event, cls=EncodeUpdate) json.dumps(misp_event, cls=EncodeFull)
def download(self): if self.offline_mode: self.log('error', 'Offline mode, unable to dodnload a sample') return ok = False data = None if self.args.hash: ok, data = self.misp.download_samples(sample_hash=self.args.hash) elif self.args.list is not None: list_events = [] if len(self.args.list) == 0: event_path = os.path.join(self.cur_path, 'misp_events') for eid, path, title in self._get_local_events(event_path): list_events.append(eid) else: list_events = self.args.list all_data = [] for eid in list_events: me = MISPEvent() me.load(self.misp.get(eid)) ok, data = self.misp.download_samples(event_id=me.id) if not ok: self.log('error', data) continue if data: all_data += data data = all_data else: event_id = self._get_eventid() if event_id is None: return ok, data = self.misp.download_samples(event_id=event_id) if not ok: self.log('error', data) return to_print = [] samples_path = os.path.join(self.cur_path, 'misp_samples') for d in data: eid, filename, payload = d path = os.path.join(samples_path, eid, filename) if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) with open(path, 'wb') as f: f.write(payload.getvalue()) to_print.append((eid, path)) if len(to_print) == 1: self.log('success', 'The sample has been downloaded from Event {}'.format(to_print[0][0])) event = self.misp.get(to_print[0][0]) if not self._has_error_message(event): return __sessions__.new(to_print[0][1], MispEvent(event, self.offline_mode)) elif len(to_print) > 1: self.log('success', 'The following files have been downloaded:') self._display_tmp_files() else: self.log('warning', 'No samples available.')
def create_massive_dummy_events(misp, nbattribute): event = MISPEvent() event.info = 'massive dummy event' event = misp.add_event(event) print(event) eventid = event.id distribution = '0' functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] for i in range(nbattribute): choice = randint(0, 5) if choice == 5: floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id) else: functions[choice](misp, event)
def from_remote(self, event_id): from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert misp = PyMISP(misp_url, misp_key, misp_verifycert) result = misp.get(event_id) self.misp_event = MISPEvent() self.misp_event.load(result)
def _change_event(self): if self.offline_mode: self._dump() else: if __sessions__.current.misp_event.event.id: event = self.misp.update(__sessions__.current.misp_event.event) else: event = self.misp.add_event(__sessions__.current.misp_event.event) if self._has_error_message(event): return try: me = MISPEvent() me.load(event) self._check_add(me) except Exception as e: self.log('error', e)
def test_batch_OSINT_events(self): # Test case ONLY for manual testing. Needs to download a full list of OSINT events ! if self.check_python_2(): self.assertTrue(True) elif not manual_testing: self.assertTrue(True) else: self.init_event() file_nb = str(len(os.listdir(self.test_batch_folder))) i = 0 t = time.time() for curr_file in os.listdir(self.test_batch_folder): self.mispevent = MISPEvent() file_path = self.test_batch_folder + curr_file print("Current file : " + file_path + " " + str(i) + " over " + file_nb) i += 1 self.mispevent.load_file(file_path) reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_folder_OSINT + curr_file + ".pdf") print("Elapsed time : " + str(time.time() - t))
def test_batch_OSINT_with_config_events(self): # Test case ONLY for manual testing. Needs to download a full list of OSINT events ! if self.check_python_2(): self.assertTrue(True) elif not manual_testing: self.assertTrue(True) else: self.init_event() config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True config[self.moduleconfig[4]] = True config[self.moduleconfig[5]] = True file_nb = str(len(os.listdir(self.test_batch_folder))) i = 0 t = time.time() for curr_file in os.listdir(self.test_batch_folder): self.mispevent = MISPEvent() file_path = self.test_batch_folder + curr_file print("Current file : " + file_path + " " + str(i) + " over " + file_nb) i += 1 self.mispevent.load_file(file_path) reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder_OSINT + curr_file + ".pdf") print("Elapsed time : " + str(time.time() - t))
class MispEvent(object): def __init__(self, event, offline=False): if isinstance(event, MISPEvent): self.event = event else: self.event = MISPEvent() if isinstance(event, six.string_types) and os.path.exists(event): self.event.load_file(event) else: self.event.load(event) self.off = offline if self.event.id: self.current_dump_file = '{}.json'.format(self.event.id) else: self.current_dump_file = None def online(self): self.off = False def offline(self): self.off = True def get_all_ips(self): return [a.value for a in self.event.attributes if a.type in ['ip-dst', 'ip-src']] def get_all_domains(self): return [a.value for a in self.event.attributes if a.type in ['domain', 'hostname']] def get_all_urls(self): return [a.value for a in self.event.attributes if a.type == 'url'] def get_all_hashes(self): event_hashes = [] sample_hashes = [] for a in self.event.attributes: h = None if a.type in ('md5', 'sha1', 'sha256'): h = a.value event_hashes.append(h) elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'): h = a.value.split('|')[1] event_hashes.append(h) elif a.type == 'malware-sample': h = a.value.split('|')[1] sample_hashes.append(h) return event_hashes, sample_hashes
def __init__(self, event, offline=False): if isinstance(event, MISPEvent): self.event = event else: self.event = MISPEvent() self.event.load(event) self.off = offline if self.event.id: self.current_dump_file = '{}.json'.format(self.event.id) else: self.current_dump_file = None
def _search(self, query): if self.offline_mode: self.log('error', 'Offline mode, unable to search') return result = self.misp.search_all(query) if self._has_error_message(result): return self.log('success', '{} matches on the following events:'.format(query)) for e in result['response']: nb_samples = 0 nb_hashes = 0 me = MISPEvent() me.load(e) for a in me.attributes + [attribute for obj in me.objects for attribute in obj.attributes]: if a.type == 'malware-sample': nb_samples += 1 if a.type in ('md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256'): nb_hashes += 1 self.log('item', '{} ({} samples, {} hashes) - {}{}{}'.format(me.info, nb_samples, nb_hashes, self.url, '/events/view/', me.id))
def create_new_event(): me = MISPEvent() me.info = "Fail2Ban blocking" me.add_tag(args.tag) start = datetime.now() me.add_attribute('datetime', start.isoformat(), comment='Start Time') return me
def _search_local_hashes(self, event, open_session=True): local = [] samples_count = 0 if isinstance(event, MISPEvent): misp_event = event elif event.get('Event') is None: self.log('error', event) return else: misp_event = MISPEvent() misp_event.load(event) if not hasattr(misp_event, 'id'): # The event doesn't exists upstream, breaking. return for a in misp_event.attributes + [attribute for obj in misp_event.objects for attribute in obj.attributes]: row = None if a.type == 'malware-sample': samples_count += 1 if a.type in ('md5', 'sha1', 'sha256'): row = Database().find(key=a.type, value=a.value) elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'): row = Database().find(key=a.type.split('|')[1], value=a.value.split('|')[1]) elif a.type == 'malware-sample': row = Database().find(key='md5', value=a.value.split('|')[1]) if row: local.append(row[0]) self.log('info', 'Event {} contains {} samples.'.format(misp_event.id, samples_count)) if not open_session: return shas = set([l.sha256 for l in local]) if len(shas) == 1: __sessions__.new(get_sample_path(shas.pop()), MispEvent(misp_event, self.offline_mode)) elif len(shas) > 1: self.log('success', 'The following samples are in this viper instance:') __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode)) for s in shas: self.log('item', s) else: __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode)) self.log('info', 'No known (in Viper) samples in that event.')
def __init__(self, event, offline=False): if isinstance(event, MISPEvent): self.event = event else: self.event = MISPEvent() if isinstance(event, six.string_types) and os.path.exists(event): self.event.load_file(event) else: self.event.load(event) self.off = offline if self.event.id: self.current_dump_file = '{}.json'.format(self.event.id) else: self.current_dump_file = None
def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() if not manual_testing: self.root = "tests/" else: self.root = "" self.test_folder = self.root + "reportlab_testfiles/" self.test_batch_folder = self.root + "OSINT_output/" self.storage_folder_OSINT = self.root + "OSINT_PDF/" self.test_image_folder = self.root + "image_json/" self.storage_folder = self.root + "reportlab_testoutputs/" self.storage_image_folder = self.root + "reportlab_test_image_outputs/" self.moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"]
def load_openioc(openioc): if not has_bs4: raise Exception('You need to install BeautifulSoup: pip install bs4') misp_event = MISPEvent() with open(openioc, "r") as ioc_file: iocreport = BeautifulSoup(ioc_file, "lxml") # Set event fields info = extract_field(iocreport, 'short_description') if info: misp_event.info = info date = extract_field(iocreport, 'authored_date') if date: misp_event.set_date(date) # Set special attributes description = extract_field(iocreport, 'description') if description: misp_event.add_attribute('comment', description) author = extract_field(iocreport, 'authored_by') if author: misp_event.add_attribute('comment', author) misp_event = set_all_attributes(iocreport, misp_event) return misp_event
def store(self): try: event_path = os.path.join(self.cur_path, 'misp_events') if not os.path.exists(event_path): os.mkdir(event_path) if self.args.list: header = ['Event ID', 'Title'] rows = [] for eid, path, title in self._get_local_events(event_path): rows.append((eid, title)) self.log('table', dict(header=header, rows=sorted(rows, key=lambda i: (int(i[0].split('_')[-1]))))) elif self.args.update: if self.offline_mode: self.log('error', 'Offline mode, cannot update locally stored events.') return for eid, path, title in self._get_local_events(event_path): event = self.misp.get(eid) with open(path, 'w') as f: f.write(json.dumps(event)) self.log('success', '{} updated successfully.'.format(eid)) elif self.args.sync: if self.offline_mode: self.log('error', 'Offline mode, cannot synchronize locally stored events.') return for eid, path, title in self._get_local_events(event_path): __sessions__.close() event = MISPEvent() event.load(path) if 'new_event_' in path: event = self.misp.add_event(json.dumps(event, cls=EncodeUpdate)) try: self._dump(event) os.remove(path) except Exception as e: self.log('error', 'Unable to create new event: {}.'.format(e)) else: eid = event.id try: event = self.misp.update(event._json()) except Exception as e: self.log('error', 'Unable to update event {}: {}.'.format(eid, e)) if self._has_error_message(event): return elif self.args.delete: path = os.path.join(event_path, '{}.json'.format(self.args.delete)) if os.path.exists(path): os.remove(path) self.log('success', '{} removed successfully.'.format(self.args.delete)) else: self.log('error', '{} does not exists.'.format(self.args.delete)) elif self.args.open: filename = '{}.json'.format(self.args.open) path = os.path.join(event_path, filename) if os.path.exists(path): try: with open(path, 'r') as f: e_json = json.load(f) __sessions__.new(misp_event=MispEvent(e_json, self.offline_mode)) __sessions__.current.misp_event.current_dump_file = filename except Exception as e: self.log('error', 'Unable to open {}: {}'.format(path, e)) else: self.log('error', '{} does not exists.'.format(self.args.open)) elif __sessions__.is_attached_misp(): self._dump() except IOError as e: self.log('error', e.strerror)
#!/usr/bin/env python # -*- coding: utf-8 -*- from pymisp import ExpandedPyMISP, MISPEvent from keys import misp_url, misp_key, misp_verifycert import argparse if __name__ == '__main__': parser = argparse.ArgumentParser(description="Update a MISP event.") parser.add_argument("-e", "--event", required=True, help="Event ID to update.") parser.add_argument("-i", "--input", required=True, help="Input file") args = parser.parse_args() misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) me = MISPEvent() me.load_file(args.input) result = misp.update_event(args.event, me)
class Yeti(): def __init__(self, url, key, attribute): self.misp_mapping = { 'Ip': 'ip-dst', 'Domain': 'domain', 'Hostname': 'hostname', 'Url': 'url', 'AutonomousSystem': 'AS', 'File': 'sha256' } self.yeti_client = pyeti.YetiApi(url=url, api_key=key) self.attribute = attribute self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) def search(self, value): obs = self.yeti_client.observable_search(value=value) if obs: return obs[0] def get_neighboors(self, obs_id): neighboors = self.yeti_client.neighbors_observables(obs_id) if neighboors and 'objs' in neighboors: links_by_id = { link['dst']['id']: (link['description'], 'dst') for link in neighboors['links'] if link['dst']['id'] != obs_id } links_by_id.update({ link['src']['id']: (link['description'], 'src') for link in neighboors['links'] if link['src']['id'] != obs_id }) for n in neighboors['objs']: yield n, links_by_id[n['id']] def parse_yeti_result(self): obs = self.search(self.attribute['value']) for obs_to_add, link in self.get_neighboors(obs['id']): object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add) if object_misp_domain_ip: self.misp_event.add_object(object_misp_domain_ip) continue object_misp_url = self.__get_object_url(obs_to_add) if object_misp_url: self.misp_event.add_object(object_misp_url) continue if link[0] == 'NS record': object_ns_record = self.__get_object_ns_record( obs_to_add, link[1]) if object_ns_record: self.misp_event.add_object(object_ns_record) continue self.__get_attribute(obs_to_add, link[0]) def get_result(self): event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if key in event } return results def __get_attribute(self, obs_to_add, link): try: type_attr = self.misp_mapping[obs_to_add['type']] value = None if obs_to_add['type'] == 'File': value = obs_to_add['value'].split(':')[1] else: value = obs_to_add['value'] attr = self.misp_event.add_attribute(value=value, type=type_attr) attr.comment = '%s: %s' % (link, self.attribute['value']) except KeyError: logging.error('type not found %s' % obs_to_add['type']) return for t in obs_to_add['tags']: self.misp_event.add_attribute_tag(t['name'], attr['uuid']) def __get_object_domain_ip(self, obj_to_add): if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or \ (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')): domain_ip_object = MISPObject('domain-ip') domain_ip_object.add_attribute(self.__get_relation(obj_to_add), obj_to_add['value']) domain_ip_object.add_attribute( self.__get_relation(self.attribute, is_yeti_object=False), self.attribute['value']) domain_ip_object.add_reference(self.attribute['uuid'], 'related_to') return domain_ip_object def __get_object_url(self, obj_to_add): if (obj_to_add['type'] == 'Url' and self.attribute['type'] in [ 'hostname', 'domain', 'ip-src', 'ip-dst' ]) or (obj_to_add['type'] in ('Hostname', 'Domain', 'Ip') and self.attribute['type'] == 'url'): url_object = MISPObject('url') obj_relation = self.__get_relation(obj_to_add) if obj_relation: url_object.add_attribute(obj_relation, obj_to_add['value']) obj_relation = self.__get_relation(self.attribute, is_yeti_object=False) if obj_relation: url_object.add_attribute(obj_relation, self.attribute['value']) url_object.add_reference(self.attribute['uuid'], 'related_to') return url_object def __get_object_ns_record(self, obj_to_add, link): queried_domain = None ns_domain = None object_dns_record = MISPObject('dns-record') if link == 'dst': queried_domain = self.attribute['value'] ns_domain = obj_to_add['value'] elif link == 'src': queried_domain = obj_to_add['value'] ns_domain = self.attribute['value'] if queried_domain and ns_domain: object_dns_record.add_attribute('queried-domain', queried_domain) object_dns_record.add_attribute('ns-record', ns_domain) object_dns_record.add_reference(self.attribute['uuid'], 'related_to') return object_dns_record def __get_relation(self, obj, is_yeti_object=True): if is_yeti_object: type_attribute = self.misp_mapping[obj['type']] else: type_attribute = obj['type'] if type_attribute == 'ip-src' or type_attribute == 'ip-dst': return 'ip' elif 'domain' == type_attribute: return 'domain' elif 'hostname' == type_attribute: return 'domain' elif type_attribute == 'url': return type_attribute
class RFEnricher: """Class for enriching an attribute with data from Recorded Future. The enrichment data is returned as a custom MISP object. """ def __init__(self, attribute_props: dict): self.event = MISPEvent() self.enrichment_object = MISPObject("Recorded Future Enrichment") description = ("An object containing the enriched attribute and " "related entities from Recorded Future.") self.enrichment_object.from_dict(**{ "meta-category": "misc", "description": description, "distribution": 0 }) # Create a copy of enriched attribute to add tags to temp_attr = MISPAttribute() temp_attr.from_dict(**attribute_props) self.enriched_attribute = MISPAttribute() self.enriched_attribute.from_dict(**{ "value": temp_attr.value, "type": temp_attr.type, "distribution": 0 }) self.related_attributes: List[Tuple[str, MISPAttribute]] = [] self.color_picker = RFColors() self.galaxy_finder = GalaxyFinder() # Mapping from MISP-type to RF-type self.type_to_rf_category = { "ip": "ip", "ip-src": "ip", "ip-dst": "ip", "ip-src|port": "ip", "ip-dst|port": "ip", "domain": "domain", "hostname": "domain", "md5": "hash", "sha1": "hash", "sha256": "hash", "uri": "url", "url": "url", "vulnerability": "vulnerability", "weakness": "vulnerability", } # Related entities have 'Related' as part of the word and Links entities from RF # portrayed as related attributes in MISP self.related_attribute_types = [ "RelatedIpAddress", "RelatedInternetDomainName", "RelatedHash", "RelatedEmailAddress", "RelatedCyberVulnerability", "IpAddress", "InternetDomainName", "Hash", "EmailAddress", "CyberVulnerability", ] # Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP self.galaxy_tag_types = [ "RelatedMalware", "RelatedThreatActor", "Threat Actor", "MitreAttackIdentifier", "Malware", ] def enrich(self) -> None: """Run the enrichment.""" category = self.type_to_rf_category.get(self.enriched_attribute.type, "") enriched_attribute_value = self.enriched_attribute.value # If enriched attribute has a port we need to remove that port # since RF do not support enriching ip addresses with port if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]: enriched_attribute_value = enriched_attribute_value.split("|")[0] json_response = GLOBAL_REQUEST_HANDLER.rf_lookup( category, enriched_attribute_value) response = json.loads(json_response.content) try: # Add risk score and risk rules as tags to the enriched attribute risk_score = response["data"]["risk"]["score"] hex_color = self.color_picker.riskscore_color(risk_score) tag_name = f'recorded-future:risk-score="{risk_score}"' self.add_tag(tag_name, hex_color) risk_criticality = response["data"]["risk"]["criticalityLabel"] hex_color = self.color_picker.criticality_color(risk_criticality) tag_name = f'recorded-future:criticality="{risk_criticality}"' self.add_tag(tag_name, hex_color) for evidence in response["data"]["risk"]["evidenceDetails"]: risk_rule = evidence["rule"] criticality = evidence["criticality"] hex_color = self.color_picker.riskrule_color(criticality) tag_name = f'recorded-future:risk-rule="{risk_rule}"' self.add_tag(tag_name, hex_color) links_data = response["data"].get("links", {}).get("hits") # Check if we have error in links response. If yes, then user do not have right module enabled in token links_access_error = response["data"].get("links", {}).get("error") galaxy_tags = [] if not links_access_error: for hit in links_data: for section in hit["sections"]: for sec_list in section["lists"]: entity_type = sec_list["type"]["name"] for entity in sec_list["entities"]: if entity_type in self.galaxy_tag_types: galaxy = self.galaxy_finder.find_galaxy_match( entity["name"], entity_type) if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) else: self.add_attribute(entity["name"], entity_type) else: # Retrieve related entities for related_entity in response["data"]["relatedEntities"]: related_type = related_entity["type"] if related_type in self.related_attribute_types: # Related entities returned as additional attributes for related in related_entity["entities"]: # filter those entities that have count bigger than 4, to reduce noise # because there can be a huge list of related entities if int(related["count"]) > 4: indicator = related["entity"]["name"] self.add_attribute(indicator, related_type) elif related_type in self.galaxy_tag_types: # Related entities added as galaxy-tags to the enriched attribute galaxy_tags = [] for related in related_entity["entities"]: # filter those entities that have count bigger than 4, to reduce noise # because there can be a huge list of related entities if int(related["count"]) > 4: indicator = related["entity"]["name"] galaxy = self.galaxy_finder.find_galaxy_match( indicator, related_type) # Handle deduplication of galaxy tags if galaxy and galaxy not in galaxy_tags: galaxy_tags.append(galaxy) for galaxy in galaxy_tags: self.add_tag(galaxy) except KeyError: misperrors[ "error"] = "Unexpected format in Recorded Future api response." raise def add_attribute(self, indicator: str, indicator_type: str) -> None: """Helper method for adding an indicator to the attribute list.""" out_type = self.get_output_type(indicator_type, indicator) attribute = MISPAttribute() attribute.from_dict(**{ "value": indicator, "type": out_type, "distribution": 0 }) self.related_attributes.append((indicator_type, attribute)) def add_tag(self, tag_name: str, hex_color: str = None) -> None: """Helper method for adding a tag to the enriched attribute.""" tag = MISPTag() tag_properties = {"name": tag_name} if hex_color: tag_properties["colour"] = hex_color tag.from_dict(**tag_properties) self.enriched_attribute.add_tag(tag) def get_output_type(self, related_type: str, indicator: str) -> str: """Helper method for translating a Recorded Future related type to a MISP output type.""" output_type = "text" if related_type in ["RelatedIpAddress", "IpAddress"]: output_type = "ip-dst" elif related_type in [ "RelatedInternetDomainName", "InternetDomainName" ]: output_type = "domain" elif related_type in ["RelatedHash", "Hash"]: hash_len = len(indicator) if hash_len == 64: output_type = "sha256" elif hash_len == 40: output_type = "sha1" elif hash_len == 32: output_type = "md5" elif related_type in ["RelatedEmailAddress", "EmailAddress"]: output_type = "email-src" elif related_type in [ "RelatedCyberVulnerability", "CyberVulnerability" ]: signature = indicator.split("-")[0] if signature == "CVE": output_type = "vulnerability" elif signature == "CWE": output_type = "weakness" elif related_type == "MalwareSignature": output_type = "malware-sample" elif related_type == "Organization": output_type = "target-org" elif related_type == "Username": output_type = "target-user" return output_type def get_results(self) -> dict: """Build and return the enrichment results.""" self.enrichment_object.add_attribute("Enriched attribute", **self.enriched_attribute) for related_type, attribute in self.related_attributes: self.enrichment_object.add_attribute(related_type, **attribute) self.event.add_object(**self.enrichment_object) event = json.loads(self.event.to_json()) result = {key: event[key] for key in ["Object"] if key in event} return {"results": result}
def handle_url(self, url, event: pymisp.MISPEvent): """Handle a single URL.""" event.add_attribute("url", str(url)) return event
def handle_domain(self, domain, event: pymisp.MISPEvent): """Handle a single domain.""" event.add_attribute("domain", str(domain)) return event
def import_all(self, stations_short_names, interval, data_type): object_creator = getattr(self, f'{interval}_flask_{data_type}') if data_type == 'co2': base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/' elif data_type in ['c13', 'o18']: base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/' for station in stations_short_names: url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv' infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations' filepath = self.fetch(url) if not filepath: continue if infofield in self.scrippts_meta: event = MISPEvent() event.load_file(str(self.output_dir / self.scrippts_meta[infofield])) location = event.get_objects_by_name('geolocation')[0] update = True else: event = MISPEvent() event.uuid = str(uuid4()) event.info = infofield event.Orgc = self.misp_org event.add_tag(getattr(self, f'tag_{station}')()) location = getattr(self, f'geolocation_{station}')() event.add_object(location) event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}') update = False with self.scrippts_meta_file.open('a') as f: writer = csv.writer(f) writer.writerow([infofield, f'{event.uuid}.json']) object_creator(event, location, filepath, update) if update: # Bump the publish timestamp event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now()) feed_output = event.to_feed(with_meta=False) with (self.output_dir / f'{event.uuid}.json').open('w') as f: # json.dump(feed_output, f, indent=2, sort_keys=True) # For testing json.dump(feed_output, f)
def from_file(self, path): self.misp_event = MISPEvent() self.misp_event.load_file(path)
def run(self, results): """Run analysis. @return: MISP results dict. """ url = self.options.get("url", "") apikey = self.options.get("apikey", "") if not url or not apikey: log.error("MISP URL or API key not configured.") return self.misp = PyMISP(url, apikey, False, "json") self.threads = self.options.get("threads", "") if not self.threads: self.threads = 5 self.iocs = deque() self.misper = dict() try: if self.options.get("upload_iocs", False) and results.get("malscore", 0) >= self.options.get("min_malscore", 0): distribution = int(self.options.get("distribution", 0)) threat_level_id = int(self.options.get("threat_level_id", 4)) analysis = int(self.options.get("analysis", 0)) tag = self.options.get("tag") or "CAPEv2" info = self.options.get("title", "") upload_sample = self.options.get("upload_sample") malfamily = "" if results.get("detections", ""): malfamily = results["detections"] response = self.misp.search("attributes", value=results["target"]["file"]["sha256"], return_format="json", pythonify=True) if response: event = self.misp.get_event(response[0].event_id, pythonify=True) else: event = MISPEvent() event.distribution = distribution event.threat_level_id = threat_level_id event.analysis = analysis event.info = "{} {} - {}".format(info, malfamily, results.get("info", {}).get("id")) event = self.misp.add_event(event, pythonify=True) # Add a specific tag to flag Cuckoo's event if tag: self.misp.tag(event, tag) # malpedia galaxy if malpedia_json: self.malpedia(results, event, malfamily) # ToDo? self.signature(results, event) self.sample_hashes(results, event) self.all_network(results, event) self.dropped_files(results, event) if upload_sample: target = results.get("target", {}) f = target.get("file", {}) if target.get("category") == "file" and f: with open(f["path"], "rb") as f: event.add_attribute( "malware-sample", value=os.path.basename(f["path"]), data=BytesIO(f.read()), expand="binary", comment="Sample run", ) if results.get("target", {}).get("url", "") and results["target"]["url"] not in whitelist: event.add_attribute("url", results["target"]["url"]) # ToDo migth be outdated! # if self.options.get("ids_files", False) and "suricata" in results.keys(): # for surifile in results["suricata"]["files"]: # if "file_info" in surifile.keys(): # self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]}) # self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]}) # self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]}) if self.options.get("mutexes", False) and "behavior" in results and "summary" in results["behavior"]: if "mutexes" in results.get("behavior", {}).get("summary", {}): for mutex in results["behavior"]["summary"]["mutexes"]: if mutex not in whitelist: event.add_attribute("mutex", mutex) if self.options.get("registry", False) and "behavior" in results and "summary" in results["behavior"]: if "read_keys" in results["behavior"].get("summary", {}): for regkey in results["behavior"]["summary"]["read_keys"]: event.add_attribute("regkey", regkey) event.run_expansions() self.misp.update_event(event) # Make event public if self.options.get("published", True): self.misp.publish(event) except Exception as e: log.error("Failed to generate JSON report: %s" % e, exc_info=True)
def create_misp_event(misp_instance, isight_report_instance, event_tags): # No MISP event for this iSight report ID exists yet. # Alas, create a new MISP event. # Convert the publication date of the iSight report into a datetime object. if isight_report_instance.publishDate: date = datetime.datetime.fromtimestamp( isight_report_instance.publishDate) else: # If iSight doesn't provide a date, use today's date. date = datetime.datetime.now(datetime.timezone.utc) # Create a MISP event from the FireEye iSight report with the following parameters. event = MISPEvent() event.distribution = 1 # This community only if isight_report_instance.riskRating == 'CRITICAL' or isight_report_instance.riskRating == 'Critical': event.threat_level_id = 1 # High elif isight_report_instance.riskRating == 'HIGH' or isight_report_instance.riskRating == 'High': event.threat_level_id = 1 # High elif isight_report_instance.riskRating == 'MEDIUM' or isight_report_instance.riskRating == 'Medium': event.threat_level_id = 2 # Medium elif isight_report_instance.riskRating == 'LOW' or isight_report_instance.riskRating == 'Low': event.threat_level_id = 3 # Low else: event.threat_level_id = 4 # Unknown event.analysis = 2 # Completed event.info = "iSIGHT: " + isight_report_instance.title event.date = date # Push the event to the MISP server. my_event = misp_instance.add_event(event, pythonify=True) PySight_settings.logger.debug('Created MISP event %s for iSight report %s', event, isight_report_instance.reportId) # Add the event ID to the global list of newly created events. global new_events new_events.append(my_event['id']) # Add default tags to the event. if event_tags: for event_tag in event_tags: misp_instance.tag(my_event, event_tag) # Use some iSight ThreatScapes for event tagging. Reports can have multiple ThreatScapes. if 'Cyber Espionage' in isight_report_instance.ThreatScape: # VERIS distinguishes between external, internal or partner actors. This difference is not yet implemented in # MISP. External would be most likely. #misp_instance.tag(my_event, 'veris:actor:external:motive="Espionage"') misp_instance.tag(my_event, 'veris:actor:motive="Espionage"') if 'Hacktivism' in isight_report_instance.ThreatScape: misp_instance.tag(my_event, 'veris:actor:external:variety="Activist"') if 'Critical Infrastructure' in isight_report_instance.ThreatScape: misp_instance.tag(my_event, 'basf:technology="OT"') if 'Cyber Physical' in isight_report_instance.ThreatScape: misp_instance.tag(my_event, 'basf:technology="OT"') if 'Cyber Crime' in isight_report_instance.ThreatScape: misp_instance.tag(my_event, 'veris:actor:external:variety="Organized crime"') # Add the iSight report ID and web link as attributes. if isight_report_instance.reportId: misp_instance.add_attribute(my_event, { 'category': 'External analysis', 'type': 'text', 'to_ids': False, 'value': isight_report_instance.reportId }, pythonify=True) if isight_report_instance.webLink: misp_instance.add_attribute(my_event, { 'category': 'External analysis', 'type': 'link', 'to_ids': False, 'value': isight_report_instance.webLink }, pythonify=True) # Put the ThreatScape into an Attribution attribute, but disable correlation. if isight_report_instance.ThreatScape: misp_instance.add_attribute(my_event, { 'category': 'Attribution', 'type': 'text', 'to_ids': False, 'value': isight_report_instance.ThreatScape, 'disable_correlation': True }, pythonify=True) # Add specific attributes from this iSight report. update_misp_event(misp_instance, my_event, isight_report_instance)
def parse_response(censys_output, attribute): misp_event = MISPEvent() misp_event.add_attribute(**attribute) # Generic fields (for IP/Websites) if "autonomous_system" in censys_output: cen_as = censys_output['autonomous_system'] asn_object = MISPObject('asn') asn_object.add_attribute('asn', value=cen_as["asn"]) asn_object.add_attribute('description', value=cen_as['name']) asn_object.add_attribute('subnet-announced', value=cen_as['routed_prefix']) asn_object.add_attribute('country', value=cen_as['country_code']) asn_object.add_reference(attribute.uuid, 'associated-to') misp_event.add_object(**asn_object) if "ip" in censys_output and "ports" in censys_output: ip_object = MISPObject('ip-port') ip_object.add_attribute('ip', value=censys_output['ip']) for p in censys_output['ports']: ip_object.add_attribute('dst-port', value=p) ip_object.add_reference(attribute.uuid, 'associated-to') misp_event.add_object(**ip_object) # We explore all ports to find https or ssh services for k in censys_output.keys(): if not isinstance(censys_output[k], dict): continue if 'https' in censys_output[k]: try: cert = censys_output[k]['https']['tls']['certificate'] cert_obj = get_certificate_object(cert, attribute) misp_event.add_object(**cert_obj) except KeyError: print("Error !") if 'ssh' in censys_output[k]: try: cert = censys_output[k]['ssh']['v2']['server_host_key'] # TODO enable once the type is merged # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256']) except KeyError: pass # Info from certificate query if "parsed" in censys_output: cert_obj = get_certificate_object(censys_output, attribute) misp_event.add_object(**cert_obj) # Location can be present for IP/Websites results if "location" in censys_output: loc_obj = MISPObject('geolocation') loc = censys_output['location'] loc_obj.add_attribute('latitude', value=loc['latitude']) loc_obj.add_attribute('longitude', value=loc['longitude']) if 'city' in loc: loc_obj.add_attribute('city', value=loc['city']) loc_obj.add_attribute('country', value=loc['country']) if 'postal_code' in loc: loc_obj.add_attribute('zipcode', value=loc['postal_code']) if 'province' in loc: loc_obj.add_attribute('region', value=loc['province']) loc_obj.add_reference(attribute.uuid, 'associated-to') misp_event.add_object(**loc_obj) event = json.loads(misp_event.to_json()) return {'Object': event['Object'], 'Attribute': event['Attribute']}
def __init__(self): super(MISP, self).__init__() self.cur_path = __project__.get_path() self.parser.add_argument("--url", help='URL of the MISP instance') self.parser.add_argument("--off", action='store_true', help='Use offline (can only work on pre-downloaded events)') self.parser.add_argument("--on", action='store_true', help='Switch to online mode') self.parser.add_argument("-k", "--key", help='Your key on the MISP instance') self.parser.add_argument("-v", "--verify", action='store_false', help='Disable certificate verification (for self-signed)') subparsers = self.parser.add_subparsers(dest='subname') # ##### Upload sample to MISP ##### parser_up = subparsers.add_parser('upload', help='Send malware sample to MISP.', formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(''' Distribution levels: * 0: Your organisation only * 1: This community only * 2: Connected communities * 3: All communities * 5: Inherit Sample categories: * 0: Payload delivery * 1: Artifacts dropped * 2: Payload installation * 3: External analysis Analysis levels: * 0: Initial * 1: Ongoing * 2: Completed Threat levels: * 0: High * 1: Medium * 2: Low * 3: Undefined ''')) parser_up.add_argument("-e", "--event", type=int, help="Event ID to update. If None, and you're not connected to a MISP event a new one is created.") parser_up.add_argument("-d", "--distrib", type=int, choices=[0, 1, 2, 3, 5], help="Distribution of the attributes for the new event.") parser_up.add_argument("-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.") parser_up.add_argument("-ids", action='store_true', help="Is eligible for automatically creating IDS signatures.") parser_up.add_argument("-c", "--categ", type=int, choices=[0, 1, 2, 3], default=1, help="Category of the samples.") parser_up.add_argument("-i", "--info", nargs='+', help="Event info field of a new event.") parser_up.add_argument("-o", "--comment", nargs='+', help="Comment associated to the sample.") parser_up.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.") parser_up.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.") # ##### Download samples from event ##### parser_down = subparsers.add_parser('download', help='Download malware samples from MISP.') group = parser_down.add_mutually_exclusive_group() group.add_argument("-e", "--event", type=int, help="Download all the samples related to this event ID.") group.add_argument("-l", "--list", nargs='*', help="Download all the samples related to a list of events. Empty list to download all the samples of all the events stored in the current project.") # noqa group.add_argument("--hash", help="Download the sample related to this hash (only MD5).") # ##### Search in MISP ##### parser_search = subparsers.add_parser('search', help='Search in all the attributes.') parser_search.add_argument("query", nargs='*', help="String to search (if empty, search the hashes of the current file).") # ##### Check hashes on VT ##### parser_checkhashes = subparsers.add_parser('check_hashes', help='Crosscheck hashes on VT.') parser_checkhashes.add_argument("event", nargs='?', default=None, type=int, help="Lookup all the hashes of an event on VT.") parser_checkhashes.add_argument("-p", "--populate", action='store_true', help="Automatically populate event with hashes found on VT.") # ##### Download Yara rules ##### parser_checkhashes = subparsers.add_parser('yara', help='Get YARA rules of an event.') parser_checkhashes.add_argument("event", nargs='?', default=None, type=int, help="Download the yara rules of that event.") # ##### Get Events ##### parser_pull = subparsers.add_parser('pull', help='Initialize the session with an existing MISP event.') parser_pull.add_argument("event", nargs='+', type=int, help="(List of) Event(s) ID.") # ##### Create an Event ##### parser_create_event = subparsers.add_parser('create_event', help='Create a new event on MISP and initialize the session with it.', formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(''' Distribution levels: * 0: Your organisation only * 1: This community only * 2: Connected communities * 3: All communities * 4: Sharing group Sharing Group: * #: ID of sharing group Analysis levels: * 0: Initial * 1: Ongoing * 2: Completed Threat levels: * 0: High * 1: Medium * 2: Low * 3: Undefined ''')) parser_create_event.add_argument("-d", "--distrib", type=int, choices=[0, 1, 2, 3, 4], help="Distribution of the attributes for the new event.") parser_create_event.add_argument("-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.") parser_create_event.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.") parser_create_event.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.") parser_create_event.add_argument("-i", "--info", required=True, nargs='+', help="Event info field of a new event.") parser_create_event.add_argument("--date", help="Date of the event. (Default: today).") # ##### Add Hashes ##### h = subparsers.add_parser("add_hashes", help="If no parameters, add all the hashes of the current session.") h.add_argument("-f", "--filename", help="Filename") h.add_argument("-m", "--md5", help="MD5") h.add_argument("-s", "--sha1", help="SHA1") h.add_argument("-a", "--sha256", help="SHA256") # ##### Add attributes ##### parser_add = subparsers.add_parser('add', help='Add attributes to an existing MISP event.') subparsers_add = parser_add.add_subparsers(dest='add') # Hashes # Generic add temp_me = MISPEvent() if hasattr(temp_me, "types"): known_types = temp_me.types else: # New API known_types = temp_me.get_known_types() for t in known_types: sp = subparsers_add.add_parser(t, help="Add {} to the event.".format(t)) sp.add_argument(t, nargs='+') # ##### Show attributes ##### subparsers.add_parser('show', help='Show attributes to an existing MISP event.') # ##### Open file ##### o = subparsers.add_parser('open', help='Open a sample from the temp directory.') ox = o.add_mutually_exclusive_group(required=True) ox.add_argument("-l", "--list", action='store_true', help="List available files") ox.add_argument("-d", "--delete", help="Delete temporary files (use 'all' to remove all the local samples or an Event ID to only remove the associated samples)") ox.add_argument("sid", nargs='?', type=int, help='Sample ID to open (from the list option).') # ##### Publish an event ##### subparsers.add_parser('publish', help='Publish an existing MISP event.') # ##### Show version ##### subparsers.add_parser('version', help='Returns the version of the MISP instance.') # Store s = subparsers.add_parser('store', help='Store the current MISP event in the current project.') s.add_argument("-l", "--list", action='store_true', help="List stored MISP events") s.add_argument("-u", "--update", action='store_true', help="Update all stored MISP events") s.add_argument("-s", "--sync", action='store_true', help="Sync all MISP Events with the remote MISP instance") s.add_argument("-d", "--delete", type=int, help="Delete a stored MISP event") s.add_argument("-o", "--open", help="Open a stored MISP event") # Tags s = subparsers.add_parser('tag', help='Tag managment using MISP taxonomies.') s.add_argument("-l", "--list", action='store_true', help="List Existing taxonomies.") s.add_argument("-d", "--details", help="Display all values of a taxonomy.") s.add_argument("-s", "--search", help="Search all tags matching a value.") s.add_argument("-e", "--event", help="Add tag to the current event.") s.add_argument("-a", "--attribute", nargs='+', help="Add tag to an attribute of the current event. Syntax: <identifier for the attribute> <machinetag>") # Galaxies s = subparsers.add_parser('galaxies', help='Use misp-galaxy with PyMISPGalaxies.') s.add_argument("-l", "--list", action='store_true', help="List existing galaxies.") s.add_argument("-d", "--details", help="Display all values of a galaxy.") s.add_argument("-v", "--cluster-value", nargs='+', help="Display all details of a cluster value.") s.add_argument("-s", "--search", nargs='+', help="Search all galaxies matching a value.") # Admin s = subparsers.add_parser('admin', help='Administration options.') admin_parser = s.add_subparsers(dest='admin') # Organisation org = admin_parser.add_parser('org', help="Organisation managment.") subparsers_org = org.add_subparsers(dest='org') # Get display = subparsers_org.add_parser('display', help="Display an organisation.") display.add_argument('id', help='ID of the organisation to display. Use "local" to display all local organisations, "external" for all remote organisations, and "all", for both.') # Search search = subparsers_org.add_parser('search', help="Search an organisation by name.") search.add_argument('name', help='(Partial) name of the organisation.') search.add_argument('-t', '--type', default='local', choices=['local', 'external', 'all'], help='Use "local" to search in all local organisations, "external" for remote organisations, and "all", for both.') # Add add_org = subparsers_org.add_parser('add', help="Add an organisation.") add_org.add_argument('name', help='Organisation name.') add_org.add_argument('-u', '--uuid', default=None, help='UUID of the organisation.') add_org.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.') add_org.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.') add_org.add_argument('-n', '--nationality', default=None, help='Nationality of the organisation.') add_org.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.') add_org.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.') add_org.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.') # Delete delete = subparsers_org.add_parser('delete', help="Delete an organisation.") delete.add_argument('id', help='ID of the organisation to delete.') # Edit edit = subparsers_org.add_parser('edit', help="Edit an organisation.") edit.add_argument('id', help='ID of the organisation to edit.') edit.add_argument('-n', '--name', help='Organisation name.') edit.add_argument('-u', '--uuid', help='UUID of the organisation.') edit.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.') edit.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.') edit.add_argument('--nationality', help='Nationality of the organisation.') edit.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.') edit.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.') edit.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.') # User user = admin_parser.add_parser('user', help="User managment.") subparsers_user = user.add_subparsers(dest='user') # Get display = subparsers_user.add_parser('display', help="Display a user.") display.add_argument('id', help='ID of the user to display. Use "all" to display all users.') # Search search = subparsers_user.add_parser('search', help="Search a user by email.") search.add_argument('name', help='(Partial) email of the user.') # Add add_usr = subparsers_user.add_parser('add', help="Add a user.") add_usr.add_argument('email', help='User email address.') add_usr.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.') add_usr.add_argument('-r', '--role-id', default=None, help='Role of the user') add_usr.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export') add_usr.add_argument('-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login') add_usr.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted') add_usr.add_argument('-p', '--password', default=None, help='Set a new password') add_usr.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account') # Delete delete = subparsers_user.add_parser('delete', help="Delete a user.") delete.add_argument('id', help='ID of the user to delete.') # Edit edit = subparsers_user.add_parser('edit', help="Edit a user.") edit.add_argument('id', help='ID of the user to edit.') edit.add_argument('-e', '--email', help='User email address.') edit.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.') edit.add_argument('-r', '--role-id', default=None, help='Role of the user') edit.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export') edit.add_argument('-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login') edit.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted') edit.add_argument('-p', '--password', default=None, help='Set a new password') edit.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account') # Role role = admin_parser.add_parser('role', help="Role managment.") subparsers_role = role.add_subparsers(dest='role') # Get display = subparsers_role.add_parser('display', help="Display all the roles.") # Search search = subparsers_role.add_parser('search', help="Search a role by name.") search.add_argument('name', help='(Partial) name of the role.') # Tags t = admin_parser.add_parser('tag', help="Tag managment.") subparsers_tag = t.add_subparsers(dest='tag') # Get display = subparsers_tag.add_parser('display', help="Display all the tags.") # Search search = subparsers_tag.add_parser('search', help="Search a tag by name.") search.add_argument('name', help='(Partial) name of the tag.') self.categories = {0: 'Payload delivery', 1: 'Artifacts dropped', 2: 'Payload installation', 3: 'External analysis'}
"The distribution setting used for the attributes and for the newly created event, if relevant. [0-3]." ) parser.add_argument( "-i", "--info", help="Used to populate the event info field if no event ID supplied.") parser.add_argument( "-a", "--analysis", type=int, help= "The analysis level of the newly created event, if applicable. [0-2]") parser.add_argument( "-t", "--threat", type=int, help= "The threat level ID of the newly created event, if applicable. [1-4]") args = parser.parse_args() misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) event = MISPEvent() event.distribution = args.distrib event.threat_level_id = args.threat event.analysis = args.analysis event.info = args.info event = misp.add_event(event, pythonify=True) print(event)
def setUp(self): self.maxDiff = None self.mispevent = MISPEvent()
def __init__(self): self.misp_event = MISPEvent() self.event = defaultdict(dict) self.misp_event['Galaxy'] = []
def twitter_account(data): """ Do the things. :param data: :return: """ misp_event_id = data['misp_event_id'] twitter_post_id = data['twitter_post_id'] response_url = data['response_url'] try: # Get the Twitter status. status = twitter_get_account(twitter_post_id) # Extract relevant values from the Twitter status. microblog_data = transform_twitter_account(status._json) # Load the microblog version from it's definition.json file. with open("misp-objects/twitter-account/definition.json") as f: microblog_definition = json.load(f) f.close() # Create the MISP mircroblog object. # TODO: get the object definition from github # misp_objects_path_custom searches the var path for {objectname}/definition.json to load the object definition # This file needs to be updated when the upstream object is updated. microblog = TwitterAccountObject( parameters=microblog_data, misp_objects_path_custom="misp-objects", template_version=str(microblog_definition["version"])) # Get the MISP event. working_event = misp.get_event(misp_event_id, extended=True, pythonify=True) # Get the Slackbot's MISP org ID. user_profile = misp.get_user("me") bot_org_id = user_profile["User"]["org_id"] # If the bot org can update the MISP Event with the new microblog do so. if str(bot_org_id) == str(working_event["org_id"]): working_event.Object.append(microblog) result = misp.update_event(working_event) print(result) else: new_event = True # If an extension exists for Slackbot objects use it. if "extensionEvents" in working_event: for k, event_extension in working_event[ "extensionEvents"].items(): if event_extension["Orgc"]["id"] == bot_org_id: if event_extension[ "info"] == "Covid Slack: Disinfo Bot": extension_event = misp.get_event( event_extension["id"], pythonify=True) extension_event.Object.append(microblog) result = misp.update_event(extension_event) print(result) new_event = False # Create a new extension to the parent event. if new_event: extended_event = MISPEvent() extended_event.info = "Covid Slack: Disinfo Bot" extended_event.extends_uuid = working_event["id"] extended_event.Object.append(microblog) result = misp.add_event(extended_event, pythonify=True) print(result) # Build slack response. response = {'blocks': [], 'response_type': 'in_channel'} response['blocks'].append({ 'type': 'section', 'text': { 'type': 'mrkdwn', 'text': 'Twitter Status: {}'.format(twitter_post_id) } }) twitter_message = "" if microblog_data.get('name'): twitter_message += 'Username: {}\n'.format(microblog_data['name']) if microblog_data.get('display-name'): twitter_message += 'Display Name: {}\n'.format( microblog_data['display-name']) if microblog_data.get('verified'): twitter_message += 'Verified Account: {}\n'.format( microblog_data['verified']) if microblog_data.get('description'): twitter_message += 'Bio: {}\n'.format( microblog_data['description']) if len(microblog_data['hashtag']) > 0: twitter_message += 'Hashtags:\n' for hashtag in microblog_data['hashtag']: twitter_message += '* {}\n'.format(hashtag) if len(microblog_data['embedded-link']) > 0: twitter_message += 'Embedded URLs:\n' for url in microblog_data['embedded-link']: twitter_message += '* {}\n'.format(url) # Add Twitter message as block. response['blocks'].append({ 'type': 'section', 'text': { 'type': 'mrkdwn', 'text': twitter_message } }) requests.post(response_url, json=response) except Exception: logger.info(traceback.print_exc(file=sys.stdout)) message = "An error has occurred!" resp = build_response(message, False) requests.post(response_url, json=resp)
def submit_tf_update(misp: ExpandedPyMISP, attributes: list) -> MISPEvent: """ create/update abuse.ch MISP-Event and append the new attributes """ eventinfo = event_info_template.format( datetime.now().strftime(info_dateformat)) # logging.debug(eventinfo) events = misp.search(controller='events', eventinfo=eventinfo, org=1, pythonify=True) if events: # current event exists already event = events[0] else: # create a new event event = MISPEvent() event.distribution = event_distribution event.threat_level_id = event_threat_level event.analysis = 2 event.info = eventinfo for tag in tagging: event.add_tag(tag) event = misp.add_event(event, pythonify=True) for att in attributes: event.add_attribute(**att) event.published = autopublish return misp.update_event(event)
def load_events_directory(self, directory): self.events = [] for path in glob.glob(os.path.join(directory, '*.json')): e = MISPEvent() e.load(path) self.import_event(e)
"reportname":"top-attacks", "key":"#{API_KEY}" # put your Panorama api key here } # try with "..., verify=False)" if you get an SSL error response = requests.request("GET", url, params=querystring) resp_text = response.text json_data = json.loads(json.dumps(xmltodict.parse(resp_text))) # initialize and set MISPOrganisation orgc = MISPOrganisation() orgc.name = 'Palo Alto' orgc.id = '#{ORGC_ID}' # organisation id orgc.uuid = '#{ORGC_UUID}' # organisation uuid # initialize and set MISPEvent() event = MISPEvent() event.Orgc = orgc event.info = json_data['report']['result']['@name'] + " | " + json_data['report']['result']['@range'] event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config event.threat_level_id = 0 # Optional, defaults to MISP.default_event_threat_level in MISP config event.analysis = 0 # Optional, defaults to 0 (initial analysis) event.add_tag('firewall threats') for threatid in json_data['report']['result']['entry']: attribute = event.add_attribute('comment', threatid['threatid']) attribute.comment = threatid['count'] misp.add_event(event.to_json())
#!/usr/bin/env python3 # -*- coding: utf-8 -*- from pymisp import ExpandedPyMISP, MISPEvent from pymisp import MISPObject from keys import misp_url, misp_key, misp_verifycert from datetime import date misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) event = MISPEvent() event.info = 'IoT malware' # Event Title event.distribution = 1 # 0 = Your Organisation Only, 1 = Community event.threat_level_id = 2 # 1 = High, 2 = Medium, 3 = Low event.analysis = 2 # 0 (initial analysis), 1 (On-Going), 2 (Complete) event.add_tag('malware_classification:malware-category="Botnet"') event.add_tag('tlp:amber') d = date.today() event.set_date(d) attribute_second = event.add_attribute('url', 'http://1.2.3.4/example', disable_correlation=False, comment="Botnet example text", to_ids=False) event = misp.add_event(event, pythonify=True) # Publish event
class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() a = self.mispevent.add_attribute('filename', 'bar.exe') del a.uuid a = self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) a = self.mispevent.objects[0].add_attribute('filename', value='') self.assertEqual(a, None) a = self.mispevent.objects[0].add_attribute('filename', value=None) self.assertEqual(a, None) a = self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) del a.uuid self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation( ['filename'])) self.assertEqual( len(self.mispevent.objects[0].get_attributes_by_relation( 'filename')), 1) self.mispevent.add_object(name='url', strict=True) a = self.mispevent.objects[1].add_attribute( 'url', value='https://www.circl.lu') del a.uuid self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' reference = self.mispevent.objects[0].add_reference( self.mispevent.objects[1], 'baz', comment='foo') del reference.uuid self.assertEqual( self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip( "Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168" ) def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() a = self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) del a.uuid attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file( 'tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation( 'malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP.") def test_shadow_attributes(self): self.init_event() p = self.mispevent.add_proposal(type='filename', value='baz.jpg') del p.uuid a = self.mispevent.add_attribute('filename', 'bar.exe') del a.uuid p = self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') del p.uuid with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) a = self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{ 'name': 'blah' }]) del a.uuid a = self.mispevent.objects[0].add_attribute('pattern-in-file', value='baz') self.assertEqual(a.category, 'Artifacts dropped') del a.uuid self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent. objects[0].attributes[0]) a = self.mispevent.objects[1].add_attribute('filename', value='baz') del a.uuid self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) a = self.mispevent.objects[0].add_attribute( 'registrar', value='registar.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') del a.uuid a = self.mispevent.objects[0].add_attribute( 'nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') del a.uuid self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file( 'tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') def test_userdefined_object(self): self.init_event() self.mispevent.add_object( name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json(sort_keys=True, indent=2) if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: # Python2 bullshit self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') a = self.mispevent.objects[0].add_attribute('member3', value='foo') del a.uuid with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual( e.exception.message, 'At least one of the following attributes is required: member1, member2' ) a = self.mispevent.objects[0].add_attribute('member1', value='bar') del a.uuid a = self.mispevent.objects[0].add_attribute('member1', value='baz') del a.uuid with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple self.mispevent.to_json(sort_keys=True, indent=2) self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[ 0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2), json.dumps(ref_json, sort_keys=True, indent=2))
def handle_ipaddress(self, ipaddress, event: pymisp.MISPEvent): """Handle a single IP address.""" event.add_attribute("ip-dst", str(ipaddress)) return event
def check_hashes(self): if self.offline_mode: self.log('error', 'Offline mode, unable to query VirusTotal') return event_id = self._get_eventid() if event_id is None: return event = self.misp.get(event_id) if self._has_error_message(event): return misp_event = MISPEvent() misp_event.load(event) hashes_to_expand = {} hashes_expanded = [] # Thoses hashes are known and already processed local_samples_hashes = [] partial_objects = {} for o in misp_event.Object: if o.name != 'file': continue if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']): # This object has all the hashes we care about tmphashes = [] tmphashes += [h.value for h in o.get_attributes_by_relation('md5')] tmphashes += [h.value for h in o.get_attributes_by_relation('sha1')] tmphashes += [h.value for h in o.get_attributes_by_relation('sha256')] # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[o.get_attributes_by_relation('sha256')[0].value] = o.get_attributes_by_relation('sha256')[0] if o.has_attributes_by_relation(['malware-sample']): # ... and it has a malware sample local_samples_hashes += tmphashes hashes_expanded += tmphashes elif o.has_attributes_by_relation(['malware-sample']): # This object has a malware sample, but is missing hashes. We can expand locally. # get the MD5 from the malware-sample attribute malware_sample = o.get_attributes_by_relation('malware-sample')[0] # at most one sample/file object local_samples_hashes.append(malware_sample.value.split('|')[1]) local_samples_hashes += [h.value for h in o.get_attributes_by_relation('md5')] local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha1')] local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha256')] if self.args.populate: # The object is missing hashes, keeping track of it for expansion if it isn't already done. partial_objects[o.uuid] = malware_sample else: sha256 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha256')} sha1 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha1')} md5 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('md5')} if sha256: hashes_to_expand.update(sha256) elif sha1: hashes_to_expand.update(sha1) elif md5: hashes_to_expand.update(md5) for ref_uuid, sample in partial_objects.items(): if sample.value.split('|')[1] in hashes_expanded: # Already expanded in an other object continue new_obj, hashes = self._expand_local_sample(pseudofile=sample.malware_binary, filename=sample.value.split('|')[0], refobj=ref_uuid, default_attributes_paramaters=sample) misp_event.Object += new_obj local_samples_hashes += hashes # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[hashes[0]] = sample hashes_expanded += local_samples_hashes for a in misp_event.attributes: if a.type == 'malware-sample' and a.value.split('|')[1] not in hashes_expanded: new_obj, hashes = self._expand_local_sample(pseudofile=a.malware_binary, filename=a.value.split('|')[0], default_attributes_paramaters=a) misp_event.Object += new_obj local_samples_hashes += hashes # Make sure to query VT for the sha256, even if expanded locally hashes_to_expand[hashes[0]] = a elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'): # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway fname, hashval = a.value.split('|') hashes_to_expand[hashval] = a elif a.type in ('md5', 'sha1', 'sha256'): # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway hashes_to_expand[a.value] = a unk_vt_hashes = [] if cfg.virustotal.virustotal_has_private_key is False: quota = 4 timeout = datetime.datetime.now() + datetime.timedelta(minutes=1) hashes_expanded += local_samples_hashes processed_on_vt = [] # Make sure to start getting reports for the longest possible hashes (reduce risks of collisions) for to_expand in sorted(list(set(hashes_to_expand)), key=len): if to_expand in processed_on_vt: # Always run VT, once per sample continue original_attribute = hashes_to_expand[to_expand] if original_attribute.get('object_id'): original_object_id = original_attribute.get('object_id') vt_object = self._make_VT_object(to_expand, original_attribute) if not vt_object: unk_vt_hashes.append(to_expand) continue result = vt_object.get_report() md5 = result['md5'] sha1 = result['sha1'] sha256 = result['sha256'] processed_on_vt += [sha256, sha1, md5] if all(h in local_samples_hashes for h in [md5, sha1, sha256]): self.log('success', 'Sample available in MISP:') else: self.log('success', 'Sample available in VT:') self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5, sha1, sha256)) if self.args.populate: if not all(h in hashes_expanded for h in [md5, sha1, sha256]): # If all the "new" expanded hashes are in the hashes_expanded list, skip file_object = MISPObject('file', default_attributes_paramaters=original_attribute) file_object.add_attribute('md5', value=md5) file_object.add_attribute('sha1', value=sha1) file_object.add_attribute('sha256', value=sha256) file_object.add_reference(vt_object.uuid, 'analysed-with') misp_event.Object.append(file_object) hashes_expanded += [md5, sha1, sha256] else: if not original_object_id or original_object_id == '0': # Not an object, but the hashes are in an other object, skipping continue else: # We already have a MISP object, adding the link to the new VT object file_object = misp_event.get_object_by_id(original_object_id) file_object.add_reference(vt_object.uuid, 'analysed-with') misp_event.Object.append(vt_object) if cfg.virustotal.virustotal_has_private_key is False: if quota > 0: quota -= 1 else: waiting_time = (timeout - datetime.datetime.now()).seconds if waiting_time > 0: self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time)) time.sleep(waiting_time) quota = 4 timeout = datetime.datetime.now() + datetime.timedelta(minutes=1) if self.args.populate: self._populate(misp_event) if len(unk_vt_hashes) > 0: self.log('error', 'Unknown on VT:') for h in unk_vt_hashes: self.log('item', '{}'.format(h))
def handle_yarasignature(self, yarasignature, event: pymisp.MISPEvent): """Handle a single YARA signature.""" event.add_attribute("yara", str(yarasignature)) return event
def create_event(misp): event = MISPEvent() event.distribution = 0 event.threat_level_id = 1 event.analysis = 0 return event
def __init__(self, attribute_props: dict): self.event = MISPEvent() self.enrichment_object = MISPObject("Recorded Future Enrichment") description = ("An object containing the enriched attribute and " "related entities from Recorded Future.") self.enrichment_object.from_dict(**{ "meta-category": "misc", "description": description, "distribution": 0 }) # Create a copy of enriched attribute to add tags to temp_attr = MISPAttribute() temp_attr.from_dict(**attribute_props) self.enriched_attribute = MISPAttribute() self.enriched_attribute.from_dict(**{ "value": temp_attr.value, "type": temp_attr.type, "distribution": 0 }) self.related_attributes: List[Tuple[str, MISPAttribute]] = [] self.color_picker = RFColors() self.galaxy_finder = GalaxyFinder() # Mapping from MISP-type to RF-type self.type_to_rf_category = { "ip": "ip", "ip-src": "ip", "ip-dst": "ip", "ip-src|port": "ip", "ip-dst|port": "ip", "domain": "domain", "hostname": "domain", "md5": "hash", "sha1": "hash", "sha256": "hash", "uri": "url", "url": "url", "vulnerability": "vulnerability", "weakness": "vulnerability", } # Related entities have 'Related' as part of the word and Links entities from RF # portrayed as related attributes in MISP self.related_attribute_types = [ "RelatedIpAddress", "RelatedInternetDomainName", "RelatedHash", "RelatedEmailAddress", "RelatedCyberVulnerability", "IpAddress", "InternetDomainName", "Hash", "EmailAddress", "CyberVulnerability", ] # Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP self.galaxy_tag_types = [ "RelatedMalware", "RelatedThreatActor", "Threat Actor", "MitreAttackIdentifier", "Malware", ]
class VirusTotalParser(object): def __init__(self, apikey, limit): self.apikey = apikey self.limit = limit self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report" self.misp_event = MISPEvent() self.parsed_objects = {} self.input_types_mapping = { 'ip-src': self.parse_ip, 'ip-dst': self.parse_ip, 'domain': self.parse_domain, 'hostname': self.parse_domain, 'md5': self.parse_hash, 'sha1': self.parse_hash, 'sha256': self.parse_hash, 'url': self.parse_url } self.proxies = None def query_api(self, attribute): self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) return self.input_types_mapping[self.attribute.type]( self.attribute.value, recurse=True) def get_result(self): event = json.loads(self.misp_event.to_json()) results = { key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key]) } return {'results': results} ################################################################################ #### Main parsing functions #### # noqa ################################################################################ def parse_domain(self, domain, recurse=False): req = requests.get(self.base_url.format('domain'), params={ 'apikey': self.apikey, 'domain': domain }, proxies=self.proxies) if req.status_code != 200: return req.status_code req = req.json() hash_type = 'sha256' whois = 'whois' feature_types = { 'communicating': 'communicates-with', 'downloaded': 'downloaded-from', 'referrer': 'referring' } siblings = (self.parse_siblings(domain) for domain in req['domain_siblings']) uuid = self.parse_resolutions( req['resolutions'], req['subdomains'] if 'subdomains' in req else None, siblings) for feature_type, relationship in feature_types.items(): for feature in ('undetected_{}_samples', 'detected_{}_samples'): for sample in req.get(feature.format(feature_type), [])[:self.limit]: status_code = self.parse_hash(sample[hash_type], False, uuid, relationship) if status_code != 200: return status_code if req.get(whois): whois_object = MISPObject(whois) whois_object.add_attribute('text', type='text', value=req[whois]) self.misp_event.add_object(**whois_object) return self.parse_related_urls(req, recurse, uuid) def parse_hash(self, sample, recurse=False, uuid=None, relationship=None): req = requests.get(self.base_url.format('file'), params={ 'apikey': self.apikey, 'resource': sample }, proxies=self.proxies) status_code = req.status_code if req.status_code == 200: req = req.json() vt_uuid = self.parse_vt_object(req) file_attributes = [] for hash_type in ('md5', 'sha1', 'sha256'): if req.get(hash_type): file_attributes.append({ 'type': hash_type, 'object_relation': hash_type, 'value': req[hash_type] }) if file_attributes: file_object = MISPObject('file') for attribute in file_attributes: file_object.add_attribute(**attribute) file_object.add_reference(vt_uuid, 'analyzed-with') if uuid and relationship: file_object.add_reference(uuid, relationship) self.misp_event.add_object(**file_object) return status_code def parse_ip(self, ip, recurse=False): req = requests.get(self.base_url.format('ip-address'), params={ 'apikey': self.apikey, 'ip': ip }, proxies=self.proxies) if req.status_code != 200: return req.status_code req = req.json() if req.get('asn'): asn_mapping = { 'network': ('ip-src', 'subnet-announced'), 'country': ('text', 'country') } asn_object = MISPObject('asn') asn_object.add_attribute('asn', type='AS', value=req['asn']) for key, value in asn_mapping.items(): if req.get(key): attribute_type, relation = value asn_object.add_attribute(relation, type=attribute_type, value=req[key]) self.misp_event.add_object(**asn_object) uuid = self.parse_resolutions( req['resolutions']) if req.get('resolutions') else None return self.parse_related_urls(req, recurse, uuid) def parse_url(self, url, recurse=False, uuid=None): req = requests.get(self.base_url.format('url'), params={ 'apikey': self.apikey, 'resource': url }, proxies=self.proxies) status_code = req.status_code if req.status_code == 200: req = req.json() vt_uuid = self.parse_vt_object(req) if not recurse: feature = 'url' url_object = MISPObject(feature) url_object.add_attribute(feature, type=feature, value=url) url_object.add_reference(vt_uuid, 'analyzed-with') if uuid: url_object.add_reference(uuid, 'hosted-in') self.misp_event.add_object(**url_object) return status_code ################################################################################ #### Additional parsing functions #### # noqa ################################################################################ def parse_related_urls(self, query_result, recurse, uuid=None): if recurse: for feature in ('detected_urls', 'undetected_urls'): if feature in query_result: for url in query_result[feature]: value = url['url'] if isinstance(url, dict) else url[0] status_code = self.parse_url(value, False, uuid) if status_code != 200: return status_code else: for feature in ('detected_urls', 'undetected_urls'): if feature in query_result: for url in query_result[feature]: value = url['url'] if isinstance(url, dict) else url[0] self.misp_event.add_attribute('url', value) return 200 def parse_resolutions(self, resolutions, subdomains=None, uuids=None): domain_ip_object = MISPObject('domain-ip') if self.attribute.type in ('domain', 'hostname'): domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') else: domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value) attribute_type, relation, key = ('domain', 'domain', 'hostname') for resolution in resolutions: domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key]) if subdomains: for subdomain in subdomains: attribute = MISPAttribute() attribute.from_dict(**dict(type='domain', value=subdomain)) self.misp_event.add_attribute(**attribute) domain_ip_object.add_reference(attribute.uuid, 'subdomain') if uuids: for uuid in uuids: domain_ip_object.add_reference(uuid, 'sibling-of') self.misp_event.add_object(**domain_ip_object) return domain_ip_object.uuid def parse_siblings(self, domain): attribute = MISPAttribute() attribute.from_dict(**dict(type='domain', value=domain)) self.misp_event.add_attribute(**attribute) return attribute.uuid def parse_vt_object(self, query_result): if query_result['response_code'] == 1: vt_object = MISPObject('virustotal-report') vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio, disable_correlation=True) self.misp_event.add_object(**vt_object) return vt_object.uuid def set_proxy_settings(self, config: dict) -> dict: """Returns proxy settings in the requests format. If no proxy settings are set, return None.""" proxies = None host = config.get('proxy_host') port = config.get('proxy_port') username = config.get('proxy_username') password = config.get('proxy_password') if host: if not port: misperrors['error'] = 'The virustotal_proxy_host config is set, ' \ 'please also set the virustotal_proxy_port.' raise KeyError parsed = urlparse(host) if 'http' in parsed.scheme: scheme = 'http' else: scheme = parsed.scheme netloc = parsed.netloc host = f'{netloc}:{port}' if username: if not password: misperrors['error'] = 'The virustotal_proxy_username config is set, ' \ 'please also set the virustotal_proxy_password.' raise KeyError auth = f'{username}:{password}' host = auth + '@' + host proxies = { 'http': f'{scheme}://{host}', 'https': f'{scheme}://{host}' } self.proxies = proxies return True
def add_event(self): try: event = MISPEvent() event.distribution = 0 # ATD Threat mapping to MISP Threat Level atd_threat_level = self.query['Summary']['Verdict']['Severity'] if not atd_threat_level: pass else: if atd_threat_level == '3': event.threat_level_id = 1 elif atd_threat_level == '4': event.threat_level_id = 2 elif atd_threat_level == '5': event.threat_level_id = 3 else: event.threat_level_id = 0 event.analysis = 0 # initial event.info = "ATD Analysis Report - {0}".format(self.mainfile) event.attributes = self.attributes event.Tag = 'ATD:Report' event = self.misp.add_event(event, pythonify=True) self.evenid = event.id print('SUCCESS: New MISP Event got created with ID: {}'.format(str(event.id))) except Exception as e: exc_type, exc_obj, exc_tb = sys.exc_info() print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}" .format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno, error=str(e)))
samp_details=json.loads(details_request.text) return samp_details.get('data')[0] if (samp_details.get('query_status') == 'ok') else None except: print(details_request.content) return None # a dict of samples in the Event, keyed with <k>sha256 containing <v>MISP-Object Files samples={} # a cache of all the attributes in the event attributes={} #misp setup pm = PyMISP(misp_url, misp_key, ssl=misp_check_cert) misp_event=MISPEvent() search=pm.search(controller='events', eventinfo=misp_event_name) if ( len(search) == 1): misp_event.load(search[0]) #load 'samples' dictionary from misp_event for obj in misp_event.get('Object'): if (obj.name == "file"): existing_hash = obj.get_attributes_by_relation("sha256")[0]['value'] samples.update({existing_hash : obj}) #reload 'attributes' dictionary from misp_event for attr in misp_event.get('Attribute'): attributes.update({attr.value : attr}) else:
urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30" # Window can be 14, 30, and 90 Days headers = {'Authorization': "Basic " + proofpoint_key} responseVap = requests.request("GET", urlVap, headers=headers) jsonDataVap = json.loads(responseVap.text) for alert in jsonDataVap["users"]: orgc = MISPOrganisation() orgc.name = 'Proofpoint' orgc.id = '#{ORGC.ID}' # organisation id orgc.uuid = '#{ORGC.UUID}' # organisation uuid # initialize and set MISPEvent() event = MISPEvent() event.Orgc = orgc event.info = 'Very Attacked Person ' + jsonDataVap["interval"] event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config event.threat_level_id = 2 # setting this to 0 breaks the integration event.analysis = 0 # Optional, defaults to 0 (initial analysis) totalVapUsers = event.add_attribute('counter', jsonDataVap["totalVapUsers"], comment="Total VAP Users") averageAttackIndex = event.add_attribute('counter', jsonDataVap["averageAttackIndex"], comment="Average Attack Count") vapAttackIndexThreshold = event.add_attribute(
class APIVoidParser(): def __init__(self, attribute): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&' def get_results(self): if hasattr(self, 'result'): return self.result event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def parse_domain(self, apikey): feature = 'dnslookup' if requests.get(f'{self.url.format(feature, apikey)}stats').json( )['credits_remained'] < 0.13: self.result = { 'error': 'You do not have enough APIVoid credits to proceed your request.' } return mapping = { 'A': 'resolution-of', 'MX': 'mail-server-of', 'NS': 'server-name-of' } dnslookup = requests.get( f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}' ).json() for item in dnslookup['data']['records']['items']: record_type = item['type'] try: relationship = mapping[record_type] except KeyError: continue self._handle_dns_record(item, record_type, relationship) ssl = requests.get( f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}' ).json() self._parse_ssl_certificate(ssl['data']['certificate']) def _handle_dns_record(self, item, record_type, relationship): dns_record = MISPObject('dns-record') dns_record.add_attribute('queried-domain', type='domain', value=item['host']) attribute_type, feature = ('ip-dst', 'ip') if record_type == 'A' else ('domain', 'target') dns_record.add_attribute(f'{record_type.lower()}-record', type=attribute_type, value=item[feature]) dns_record.add_reference(self.attribute.uuid, relationship) self.misp_event.add_object(**dns_record) def _parse_ssl_certificate(self, certificate): x509 = MISPObject('x509') fingerprint = 'x509-fingerprint-sha1' x509.add_attribute(fingerprint, type=fingerprint, value=certificate['fingerprint']) x509_mapping = { 'subject': { 'name': ('text', 'subject') }, 'issuer': { 'common_name': ('text', 'issuer') }, 'signature': { 'serial': ('text', 'serial-number') }, 'validity': { 'valid_from': ('datetime', 'validity-not-before'), 'valid_to': ('datetime', 'validity-not-after') } } certificate = certificate['details'] for feature, subfeatures in x509_mapping.items(): for subfeature, mapping in subfeatures.items(): attribute_type, relation = mapping x509.add_attribute(relation, type=attribute_type, value=certificate[feature][subfeature]) x509.add_reference(self.attribute.uuid, 'seen-by') self.misp_event.add_object(**x509)
class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename'])) self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1) self.mispevent.add_object(name='url', strict=True) self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' self.mispevent.objects[0].add_reference('b', 'baz', comment='foo') self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes(self): self.init_event() self.mispevent.add_proposal(type='filename', value='baz.jpg') self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0]) self.mispevent.objects[1].add_attribute('filename', value='baz') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com') self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') def test_userdefined_object(self): self.init_event() self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json() if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: # Python2 bullshit self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') self.mispevent.objects[0].add_attribute('member3', value='foo') with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf self.mispevent.to_json() self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') self.mispevent.objects[0].add_attribute('member1', value='bar') self.mispevent.objects[0].add_attribute('member1', value='baz') with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple self.mispevent.to_json() self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
def __init__(self, attribute): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'
class TestPDFExport(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() if not manual_testing: self.root = "tests/" else: self.root = "" self.test_folder = self.root + "reportlab_testfiles/" self.test_batch_folder = self.root + "OSINT_output/" self.storage_folder_OSINT = self.root + "OSINT_PDF/" self.test_image_folder = self.root + "image_json/" self.storage_folder = self.root + "reportlab_testoutputs/" self.storage_image_folder = self.root + "reportlab_test_image_outputs/" self.moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description", "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"] def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def check_python_2(self): if sys.version_info.major < 3: # we want Python2 test to pass return True def test_basic_event(self): if self.check_python_2(): self.assertTrue(True) else: self.init_event() reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_folder + "basic_event.pdf") def test_event(self): if self.check_python_2(): self.assertTrue(True) else: self.init_event() self.mispevent.load_file(self.test_folder + 'to_delete1.json') reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_folder + "normal_event.pdf") def test_HTML_json(self): if self.check_python_2(): self.assertTrue(True) else: self.init_event() self.mispevent.load_file(self.test_folder + 'HTML_event.json') reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_folder + "HTML_event.pdf") def test_long_json(self): if self.check_python_2(): self.assertTrue(True) else: self.init_event() self.mispevent.load_file(self.test_folder + 'long_event.json') reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_folder + "long_event.pdf") # Issue report : "We are not smart enough" : https://pairlist2.pair.net/pipermail/reportlab-users/2010-May/009529.html # Not nice but working solution exposed there: https://pairlist2.pair.net/pipermail/reportlab-users/2016-March/011525.html def test_very_long_json(self): if self.check_python_2(): self.assertTrue(True) else: self.init_event() self.mispevent.load_file(self.test_folder + 'very_long_event.json') reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_folder + "very_long_event.pdf") def test_full_config_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" self.init_event() self.mispevent.load_file(self.test_folder + 'very_long_event.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "config_complete_event.pdf") def test_partial_0_config_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" self.init_event() self.mispevent.load_file(self.test_folder + 'very_long_event.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "config_partial_0_event.pdf") def test_partial_1_config_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[1]] = "My Wonderful CERT" self.init_event() self.mispevent.load_file(self.test_folder + 'very_long_event.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "config_partial_1_event.pdf") def test_image_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" self.init_event() self.mispevent.load_file(self.test_folder + 'image_event.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "image_event.pdf") def test_objects_1_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" self.init_event() self.mispevent.load_file(self.test_folder + 'mainly_objects_1.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "mainly_objects_1.pdf") def test_objects_2_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" self.init_event() self.mispevent.load_file(self.test_folder + 'mainly_objects_2.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "mainly_objects_2.pdf") def test_sightings_1_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" self.init_event() self.mispevent.load_file(self.test_folder + 'sighting_1.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "sighting_1.pdf") def test_sightings_2_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" self.init_event() self.mispevent.load_file(self.test_folder + 'sighting_2.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "sighting_2.pdf") def test_textual_json(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True self.init_event() self.mispevent.load_file(self.test_folder + 'very_long_event.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "textual.pdf") def test_galaxy_1(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True self.init_event() self.mispevent.load_file(self.test_folder + 'galaxy_1.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "galaxy_1.pdf") def test_related_events(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True config[self.moduleconfig[4]] = True self.init_event() self.mispevent.load_file(self.test_folder + 'galaxy_1.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "related_events.pdf") def test_related_events_too_simple(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True config[self.moduleconfig[4]] = True self.init_event() self.mispevent.load_file(self.test_folder + 'to_delete1.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "related_events_no_related.pdf") def test_utf(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True config[self.moduleconfig[4]] = True config[self.moduleconfig[5]] = True self.init_event() self.mispevent.load_file(self.test_folder + 'japanese_test.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "japanese_test.pdf") def test_utf_heavy(self): if self.check_python_2(): self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True config[self.moduleconfig[4]] = True config[self.moduleconfig[5]] = True self.init_event() self.mispevent.load_file(self.test_folder + 'japanese_test_heavy.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "japanese_test_heavy.pdf") def test_utf_ArialUNI_custompath(self): if self.check_python_2(): self.assertTrue(True) elif not manual_testing: self.assertTrue(True) else: config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True config[self.moduleconfig[4]] = True config[self.moduleconfig[5]] = True config[self.moduleconfig[6]] = "/home/user/Desktop/PyMISP/pymisp/tools/pdf_fonts/arial-unicode-ms/ARIALUNI.TTF" self.init_event() self.mispevent.load_file(self.test_folder + 'japanese_test_heavy.json') reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder + "custom_path.pdf") def test_batch_image_events(self): # Test case ONLY for manual testing. Needs to download a full list of image events ! if self.check_python_2(): self.assertTrue(True) elif not manual_testing: self.assertTrue(True) else: self.init_event() file_nb = str(len(os.listdir(self.test_image_folder))) i = 0 t = time.time() for curr_file in os.listdir(self.test_image_folder): self.mispevent = MISPEvent() file_path = self.test_image_folder + curr_file print("Current file : " + file_path + " " + str(i) + " over " + file_nb) i += 1 self.mispevent.load_file(file_path) reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_image_folder + curr_file + ".pdf") print("Elapsed time : " + str(time.time() - t)) # Local run : 73.061s for 102 files def test_batch_OSINT_events(self): # Test case ONLY for manual testing. Needs to download a full list of OSINT events ! if self.check_python_2(): self.assertTrue(True) elif not manual_testing: self.assertTrue(True) else: self.init_event() file_nb = str(len(os.listdir(self.test_batch_folder))) i = 0 t = time.time() for curr_file in os.listdir(self.test_batch_folder): self.mispevent = MISPEvent() file_path = self.test_batch_folder + curr_file print("Current file : " + file_path + " " + str(i) + " over " + file_nb) i += 1 self.mispevent.load_file(file_path) reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent), self.storage_folder_OSINT + curr_file + ".pdf") print("Elapsed time : " + str(time.time() - t)) # Local run : 1958.930s for 1064 files def test_batch_OSINT_with_config_events(self): # Test case ONLY for manual testing. Needs to download a full list of OSINT events ! if self.check_python_2(): self.assertTrue(True) elif not manual_testing: self.assertTrue(True) else: self.init_event() config = {} config[self.moduleconfig[0]] = "http://localhost:8080" config[self.moduleconfig[1]] = "My Wonderful CERT" config[self.moduleconfig[2]] = True config[self.moduleconfig[3]] = True config[self.moduleconfig[4]] = True config[self.moduleconfig[5]] = True file_nb = str(len(os.listdir(self.test_batch_folder))) i = 0 t = time.time() for curr_file in os.listdir(self.test_batch_folder): self.mispevent = MISPEvent() file_path = self.test_batch_folder + curr_file print("Current file : " + file_path + " " + str(i) + " over " + file_nb) i += 1 self.mispevent.load_file(file_path) reportlab_generator.register_value_to_file( reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config), self.storage_folder_OSINT + curr_file + ".pdf") print("Elapsed time : " + str(time.time() - t))
def __init__(self): super(MISP, self).__init__() self.cur_path = __project__.get_path() self.parser.add_argument("--url", help='URL of the MISP instance') self.parser.add_argument( "--off", action='store_true', help='Use offline (can only work on pre-downloaded events)') self.parser.add_argument("--on", action='store_true', help='Switch to online mode') self.parser.add_argument("-k", "--key", help='Your key on the MISP instance') self.parser.add_argument( "-v", "--verify", action='store_false', help='Disable certificate verification (for self-signed)') subparsers = self.parser.add_subparsers(dest='subname') # ##### Upload sample to MISP ##### parser_up = subparsers.add_parser( 'upload', help='Send malware sample to MISP.', formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(''' Distribution levels: * 0: Your organisation only * 1: This community only * 2: Connected communities * 3: All communities * 5: Inherit Sample categories: * 0: Payload delivery * 1: Artifacts dropped * 2: Payload installation * 3: External analysis Analysis levels: * 0: Initial * 1: Ongoing * 2: Completed Threat levels: * 0: High * 1: Medium * 2: Low * 3: Undefined ''')) parser_up.add_argument( "-e", "--event", type=int, help= "Event ID to update. If None, and you're not connected to a MISP event a new one is created." ) parser_up.add_argument( "-d", "--distrib", type=int, choices=[0, 1, 2, 3, 5], help="Distribution of the attributes for the new event.") parser_up.add_argument( "-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.") parser_up.add_argument( "-ids", action='store_true', help="Is eligible for automatically creating IDS signatures.") parser_up.add_argument("-c", "--categ", type=int, choices=[0, 1, 2, 3], default=1, help="Category of the samples.") parser_up.add_argument("-i", "--info", nargs='+', help="Event info field of a new event.") parser_up.add_argument("-o", "--comment", nargs='+', help="Comment associated to the sample.") parser_up.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.") parser_up.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.") # ##### Download samples from event ##### parser_down = subparsers.add_parser( 'download', help='Download malware samples from MISP.') group = parser_down.add_mutually_exclusive_group() group.add_argument( "-e", "--event", type=int, help="Download all the samples related to this event ID.") group.add_argument( "-l", "--list", nargs='*', help= "Download all the samples related to a list of events. Empty list to download all the samples of all the events stored in the current project." ) # noqa group.add_argument( "--hash", help="Download the sample related to this hash (only MD5).") # ##### Search in MISP ##### parser_search = subparsers.add_parser( 'search', help='Search in all the attributes.') parser_search.add_argument( "query", nargs='*', help= "String to search (if empty, search the hashes of the current file)." ) # ##### Check hashes on VT ##### parser_checkhashes = subparsers.add_parser( 'check_hashes', help='Crosscheck hashes on VT.') parser_checkhashes.add_argument( "event", nargs='?', default=None, type=int, help="Lookup all the hashes of an event on VT.") parser_checkhashes.add_argument( "-p", "--populate", action='store_true', help="Automatically populate event with hashes found on VT.") # ##### Download Yara rules ##### parser_checkhashes = subparsers.add_parser( 'yara', help='Get YARA rules of an event.') parser_checkhashes.add_argument( "event", nargs='?', default=None, type=int, help="Download the yara rules of that event.") # ##### Get Events ##### parser_pull = subparsers.add_parser( 'pull', help='Initialize the session with an existing MISP event.') parser_pull.add_argument("event", nargs='+', type=int, help="(List of) Event(s) ID.") # ##### Create an Event ##### parser_create_event = subparsers.add_parser( 'create_event', help= 'Create a new event on MISP and initialize the session with it.', formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent(''' Distribution levels: * 0: Your organisation only * 1: This community only * 2: Connected communities * 3: All communities * 4: Sharing group Sharing Group: * #: ID of sharing group Analysis levels: * 0: Initial * 1: Ongoing * 2: Completed Threat levels: * 0: High * 1: Medium * 2: Low * 3: Undefined ''')) parser_create_event.add_argument( "-d", "--distrib", type=int, choices=[0, 1, 2, 3, 4], help="Distribution of the attributes for the new event.") parser_create_event.add_argument( "-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.") parser_create_event.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.") parser_create_event.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.") parser_create_event.add_argument( "-i", "--info", required=True, nargs='+', help="Event info field of a new event.") parser_create_event.add_argument( "--date", help="Date of the event. (Default: today).") # ##### Add Hashes ##### h = subparsers.add_parser( "add_hashes", help="If no parameters, add all the hashes of the current session." ) h.add_argument("-f", "--filename", help="Filename") h.add_argument("-m", "--md5", help="MD5") h.add_argument("-s", "--sha1", help="SHA1") h.add_argument("-a", "--sha256", help="SHA256") # ##### Add attributes ##### parser_add = subparsers.add_parser( 'add', help='Add attributes to an existing MISP event.') subparsers_add = parser_add.add_subparsers(dest='add') # Hashes # Generic add temp_me = MISPEvent() for t in sorted(temp_me.types): sp = subparsers_add.add_parser( t, help="Add {} to the event.".format(t)) sp.add_argument(t, nargs='+') # ##### Show attributes ##### subparsers.add_parser( 'show', help='Show attributes to an existing MISP event.') # ##### Open file ##### o = subparsers.add_parser( 'open', help='Open a sample from the temp directory.') ox = o.add_mutually_exclusive_group(required=True) ox.add_argument("-l", "--list", action='store_true', help="List available files") ox.add_argument( "-d", "--delete", help= "Delete temporary files (use 'all' to remove all the local samples or an Event ID to only remove the associated samples)" ) ox.add_argument("sid", nargs='?', type=int, help='Sample ID to open (from the list option).') # ##### Publish an event ##### subparsers.add_parser('publish', help='Publish an existing MISP event.') # ##### Show version ##### subparsers.add_parser('version', help='Returns the version of the MISP instance.') # Store s = subparsers.add_parser( 'store', help='Store the current MISP event in the current project.') s.add_argument("-l", "--list", action='store_true', help="List stored MISP events") s.add_argument("-u", "--update", action='store_true', help="Update all stored MISP events") s.add_argument( "-s", "--sync", action='store_true', help="Sync all MISP Events with the remote MISP instance") s.add_argument("-d", "--delete", type=int, help="Delete a stored MISP event") s.add_argument("-o", "--open", help="Open a stored MISP event") # Tags s = subparsers.add_parser('tag', help='Tag managment using MISP taxonomies.') s.add_argument("-l", "--list", action='store_true', help="List Existing taxonomies.") s.add_argument("-d", "--details", help="Display all values of a taxonomy.") s.add_argument("-s", "--search", help="Search all tags matching a value.") s.add_argument("-e", "--event", help="Add tag to the current event.") s.add_argument( "-a", "--attribute", nargs='+', help= "Add tag to an attribute of the current event. Syntax: <identifier for the attribute> <machinetag>" ) # Admin s = subparsers.add_parser('admin', help='Administration options.') admin_parser = s.add_subparsers(dest='admin') # Organisation org = admin_parser.add_parser('org', help="Organisation managment.") subparsers_org = org.add_subparsers(dest='org') # Get display = subparsers_org.add_parser('display', help="Display an organisation.") display.add_argument( 'id', help= 'ID of the organisation to display. Use "local" to display all local organisations, "external" for all remote organisations, and "all", for both.' ) # Search search = subparsers_org.add_parser( 'search', help="Search an organisation by name.") search.add_argument('name', help='(Partial) name of the organisation.') search.add_argument( '-t', '--type', default='local', choices=['local', 'external', 'all'], help= 'Use "local" to search in all local organisations, "external" for remote organisations, and "all", for both.' ) # Add add_org = subparsers_org.add_parser('add', help="Add an organisation.") add_org.add_argument('name', help='Organisation name.') add_org.add_argument('-u', '--uuid', default=None, help='UUID of the organisation.') add_org.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.') add_org.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.') add_org.add_argument('-n', '--nationality', default=None, help='Nationality of the organisation.') add_org.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.') add_org.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.') add_org.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.') # Delete delete = subparsers_org.add_parser('delete', help="Delete an organisation.") delete.add_argument('id', help='ID of the organisation to delete.') # Edit edit = subparsers_org.add_parser('edit', help="Edit an organisation.") edit.add_argument('id', help='ID of the organisation to edit.') edit.add_argument('-n', '--name', help='Organisation name.') edit.add_argument('-u', '--uuid', help='UUID of the organisation.') edit.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.') edit.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.') edit.add_argument('--nationality', help='Nationality of the organisation.') edit.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.') edit.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.') edit.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.') # User user = admin_parser.add_parser('user', help="User managment.") subparsers_user = user.add_subparsers(dest='user') # Get display = subparsers_user.add_parser('display', help="Display a user.") display.add_argument( 'id', help='ID of the user to display. Use "all" to display all users.') # Search search = subparsers_user.add_parser('search', help="Search a user by email.") search.add_argument('name', help='(Partial) email of the user.') # Add add_usr = subparsers_user.add_parser('add', help="Add a user.") add_usr.add_argument('email', help='User email address.') add_usr.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.') add_usr.add_argument('-r', '--role-id', default=None, help='Role of the user') add_usr.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export') add_usr.add_argument( '-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login') add_usr.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted') add_usr.add_argument('-p', '--password', default=None, help='Set a new password') add_usr.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account') # Delete delete = subparsers_user.add_parser('delete', help="Delete a user.") delete.add_argument('id', help='ID of the user to delete.') # Edit edit = subparsers_user.add_parser('edit', help="Edit a user.") edit.add_argument('id', help='ID of the user to edit.') edit.add_argument('-e', '--email', help='User email address.') edit.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.') edit.add_argument('-r', '--role-id', default=None, help='Role of the user') edit.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export') edit.add_argument('-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login') edit.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted') edit.add_argument('-p', '--password', default=None, help='Set a new password') edit.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account') # Role role = admin_parser.add_parser('role', help="Role managment.") subparsers_role = role.add_subparsers(dest='role') # Get display = subparsers_role.add_parser('display', help="Display all the roles.") # Search search = subparsers_role.add_parser('search', help="Search a role by name.") search.add_argument('name', help='(Partial) name of the role.') # Tags t = admin_parser.add_parser('tag', help="Tag managment.") subparsers_tag = t.add_subparsers(dest='tag') # Get display = subparsers_tag.add_parser('display', help="Display all the tags.") # Search search = subparsers_tag.add_parser('search', help="Search a tag by name.") search.add_argument('name', help='(Partial) name of the tag.') self.categories = { 0: 'Payload delivery', 1: 'Artifacts dropped', 2: 'Payload installation', 3: 'External analysis' }
WHERE rel_cnt > 5 MATCH (m)-[r:has]->(n) RETURN m, n LIMIT 200; """ if __name__ == '__main__': parser = argparse.ArgumentParser(description='Get all the events matching a value.') parser.add_argument("-s", "--search", required=True, help="String to search.") parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.") parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.") parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.") parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database") args = parser.parse_args() neo4j = Neo4j(args.host, args.user, args.password) if args.deleteall: neo4j.del_all() misp = PyMISP(misp_url, misp_key) result = misp.search_all(args.search) for json_event in result['response']: if not json_event['Event']: print(json_event) continue print('Importing', json_event['Event']['info'], json_event['Event']['id']) try: misp_event = MISPEvent() misp_event.load(json_event) neo4j.import_event(misp_event) except: print('broken')
class StixParser(): def __init__(self): self.misp_event = MISPEvent() self.event = defaultdict(dict) self.misp_event['Galaxy'] = [] def loadEvent(self, args): filename = os.path.join(os.path.dirname(args[0]), args[1]) with open(filename, 'r', encoding='utf-8') as f: event = json.loads(f.read()) self.filename = filename self.stix_version = 'stix {}'.format(event.get('spec_version')) for o in event.get('objects'): parsed_object = stix2.parse(o, allow_custom=True) try: object_type = parsed_object._type except AttributeError: object_type = parsed_object['type'] object_uuid = parsed_object['id'].split('--')[1] if object_type.startswith('x-misp-object'): object_type = 'x-misp-object' self.event[object_type][object_uuid] = parsed_object if not self.event: print( json.dumps({ 'success': 0, 'message': 'There is no valid STIX object to import' })) sys.exit(1) try: event_distribution = args[2] if not isinstance(event_distribution, int): event_distribution = int( event_distribution) if event_distribution.isdigit() else 5 except IndexError: event_distribution = 5 try: attribute_distribution = args[3] if attribute_distribution != 'event' and not isinstance( attribute_distribution, int): attribute_distribution = int( attribute_distribution) if attribute_distribution.isdigit( ) else 5 except IndexError: attribute_distribution = 5 self.misp_event.distribution = event_distribution self.__attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution self.load_mapping() def load_mapping(self): self.objects_mapping = { 'asn': { 'observable': observable_asn, 'pattern': pattern_asn }, 'domain-ip': { 'observable': observable_domain_ip, 'pattern': pattern_domain_ip }, 'email': { 'observable': self.observable_email, 'pattern': self.pattern_email }, 'file': { 'observable': observable_file, 'pattern': self.pattern_file }, 'ip-port': { 'observable': observable_ip_port, 'pattern': pattern_ip_port }, 'network-socket': { 'observable': observable_socket, 'pattern': pattern_socket }, 'process': { 'observable': observable_process, 'pattern': pattern_process }, 'registry-key': { 'observable': observable_regkey, 'pattern': pattern_regkey }, 'url': { 'observable': observable_url, 'pattern': pattern_url }, 'WindowsPEBinaryFile': { 'observable': self.observable_pe, 'pattern': self.pattern_pe }, 'x509': { 'observable': observable_x509, 'pattern': pattern_x509 } } self.object_from_refs = { 'course-of-action': self.parse_course_of_action, 'vulnerability': self.parse_vulnerability, 'x-misp-object': self.parse_custom } self.object_from_refs.update( dict.fromkeys(list(galaxy_types.keys()), self.parse_galaxy)) self.object_from_refs.update( dict.fromkeys(['indicator', 'observed-data'], self.parse_usual_object)) def handler(self): self.outputname = '{}.stix2'.format(self.filename) if self.from_misp(): self.buildMispDict() else: self.version_attribute = { 'type': 'text', 'object_relation': 'version', 'value': self.stix_version } self.buildExternalDict() self.set_distribution() def from_misp(self): for _, o in self.event['report'].items(): if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get( 'labels'): return True return False def buildMispDict(self): report_attributes = defaultdict(list) orgs = [] for _, report in self.event['report'].items(): org_uuid = report['created_by_ref'].split('--')[1] if org_uuid not in orgs: orgs.append(org_uuid) report_name = report['name'] if report_name not in orgs: report_attributes['name'].append(report_name) if report.get('published'): report_attributes['published'].append(report['published']) if hasattr(report, 'labels'): for l in report['labels']: if l not in report_attributes['labels']: report_attributes['labels'].append(l) if hasattr(report, 'external_references'): for e in report['external_references']: self.add_link(e) for ref in report['object_refs']: object_type, uuid = ref.split('--') if object_type == 'relationship': continue object2parse = self.event[object_type][uuid] labels = object2parse.get('labels') self.object_from_refs[object_type](object2parse, labels) if len(orgs) == 1: identity = self.event['identity'][orgs[0]] self.misp_event['Org'] = {'name': identity['name']} if len(report_attributes['published']) == 1: self.misp_event.publish_timestamp = self.getTimestampfromDate( report_attributes['published'][0]) if len(report_attributes['name']) == 1: self.misp_event.info = report_attributes['name'][0] else: self.misp_event.info = "Imported from MISP import for STIX 2.0 script." for l in report_attributes['labels']: self.misp_event.add_tag(l) def add_link(self, e): link = {"type": "link"} comment = e.get('source_name') try: comment = comment.split('url - ')[1] except IndexError: pass if comment: link['comment'] = comment link['value'] = e.get('url') self.misp_event.add_attribute(**link) def parse_usual_object(self, o, labels): if 'from_object' in labels: self.parse_object(o, labels) else: self.parse_attribute(o, labels) def parse_galaxy(self, o, labels): galaxy_type = self.get_misp_type(labels) tag = labels[1] value = tag.split(':')[1].split('=')[1] galaxy_description, cluster_description = o.get('description').split( '|') _, uuid = o.get('id').split('--') galaxy = { 'type': galaxy_type, 'name': o.get('name'), 'description': galaxy_description, 'GalaxyCluster': [{ 'type': galaxy_type, 'value': value, 'tag_name': tag, 'description': cluster_description, 'uuid': uuid }] } self.misp_event['Galaxy'].append(galaxy) def parse_course_of_action(self, o, _): misp_object = MISPObject('course-of-action') if 'name' in o: attribute = { 'type': 'text', 'object_relation': 'name', 'value': o.get('name') } misp_object.add_attribute(**attribute) else: return if 'description' in o: attribute = { 'type': 'text', 'object_relation': 'description', 'value': o.get('description') } misp_object.add_attribute(**attribute) self.misp_event.add_object(**misp_object) def parse_custom(self, o, labels): if 'from_object' in labels: self.parse_custom_object(o) else: self.parse_custom_attribute(o, labels) def parse_custom_object(self, o): name = o.get('type').split('x-misp-object-')[1] timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp')) category = o.get('category') attributes = [] values = o.get('x_misp_values') for v in values: attribute_type, object_relation = v.split('_') attribute = { 'type': attribute_type, 'value': values.get(v), 'object_relation': object_relation } attributes.append(attribute) misp_object = { 'name': name, 'timestamp': timestamp, 'meta-category': category, 'Attribute': attributes } self.misp_event.add_object(**misp_object) def parse_custom_attribute(self, o, labels): attribute_type = o.get('type').split('x-misp-object-')[1] if attribute_type not in misp_types: attribute_type = attribute_type.replace('-', '|') timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp')) to_ids = bool(labels[1].split('=')[1]) value = o.get('x_misp_value') category = self.get_misp_category(labels) attribute = { 'type': attribute_type, 'timestamp': timestamp, 'to_ids': to_ids, 'value': value, 'category': category } self.misp_event.add_attribute(**attribute) def parse_object(self, o, labels): object_type = self.get_misp_type(labels) name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type object_category = self.get_misp_category(labels) stix_type = o._type misp_object = MISPObject(name) misp_object['meta-category'] = object_category if stix_type == 'indicator': pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ') pattern[0] = pattern[0][1:] pattern[-1] = pattern[-1][:-1] attributes = self.objects_mapping[object_type]['pattern'](pattern) if stix_type == 'observed-data': observable = o.get('objects') attributes = self.objects_mapping[object_type]['observable']( observable) if isinstance(attributes, tuple): attributes, pe_uuid = attributes misp_object.add_reference(pe_uuid, 'included-in') for attribute in attributes: misp_object.add_attribute(**attribute) misp_object.to_ids = bool(labels[1].split('=')[1]) self.misp_event.add_object(**misp_object) def parse_attribute(self, o, labels): attribute_type = self.get_misp_type(labels) attribute_category = self.get_misp_category(labels) attribute = {'type': attribute_type, 'category': attribute_category} tags = [{'name': label} for label in labels[3:]] if tags: attribute['Tag'] = tags stix_type = o._type if stix_type == 'vulnerability': value = o.get('name') else: if stix_type == 'indicator': o_date = o.get('valid_from') pattern = o.get('pattern').replace('\\\\', '\\') value = self.parse_pattern_with_data( pattern) if attribute_type in ( 'malware-sample', 'attachment') else self.parse_pattern(pattern) attribute['to_ids'] = True else: o_date = o.get('first_observed') observable = o.get('objects') try: value = self.parse_observable(observable, attribute_type) except: print('{}: {}'.format(attribute_type, observable)) attribute['to_ids'] = False attribute['timestamp'] = self.getTimestampfromDate(o_date) if 'description' in o: attribute['comment'] = o.get('description') if isinstance(value, tuple): value, data = value attribute['data'] = io.BytesIO(data.encode()) attribute['value'] = value self.misp_event.add_attribute(**attribute) def parse_vulnerability(self, o, labels): if len(labels) > 2: self.parse_usual_object(o, labels) else: self.parse_galaxy(o, labels) @staticmethod def observable_email(observable): attributes = [] addresses = {} files = {} for o_key, o_dict in observable.items(): part_type = o_dict._type if part_type == 'email-addr': addresses[o_key] = o_dict.get('value') elif part_type == 'file': files[o_key] = o_dict.get('name') else: message = dict(o_dict) attributes.append({ 'type': 'email-src', 'object_relation': 'from', 'value': addresses[message.pop('from_ref')], 'to_ids': False }) for ref in ('to_refs', 'cc_refs'): if ref in message: for item in message.pop(ref): mapping = email_mapping[ref] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': addresses[item], 'to_ids': False }) if 'body_multipart' in message: for f in message.pop('body_multipart'): attributes.append({ 'type': 'email-attachment', 'object_relation': 'attachment', 'value': files[f.get('body_raw_ref')], 'to_ids': False }) for m_key, m_value in message.items(): if m_key == 'additional_header_fields': for field_key, field_value in m_value.items(): mapping = email_mapping[field_key] if field_key == 'Reply-To': for rt in field_value: attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': rt, 'to_ids': False }) else: attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': field_value, 'to_ids': False }) else: try: mapping = email_mapping[m_key] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': m_value, 'to_ids': False }) except KeyError: if m_key.startswith("x_misp_attachment_"): attribute_type, relation = m_key.split( "x_misp_")[1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'to_ids': False, 'value': m_value['value'], 'data': io.BytesIO(m_value['data'].encode()) }) elif "x_misp_" in m_key: attribute_type, relation = m_key.split( "x_misp_")[1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': m_value, 'to_ids': False }) return attributes @staticmethod def pattern_email(pattern): attributes = [] attachments = defaultdict(dict) for p in pattern: p_type, p_value = p.split(' = ') try: mapping = email_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) except KeyError: if p_type.startswith("email-message:'x_misp_attachment_"): relation, field = p_type.split('.') relation = relation.split(':')[1][1:-1] attachments[relation][field] = p_value[1:-1] elif "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value[1:-1], 'to_ids': True }) for a_key, a_dict in attachments.items(): _, _, attribute_type, relation = a_key.split('_') attributes.append({ 'type': attribute_type, 'object_relation': relation, 'to_ids': True, 'value': a_dict['value'], 'data': io.BytesIO(a_dict['data'].encode()) }) return attributes @staticmethod def pattern_file(pattern): attributes = [] malware_sample = {} for p in pattern: p_type, p_value = p.split(' = ') if p_type == 'artifact:payload_bin': malware_sample['data'] = p_value elif p_type in ("file:name", "file:hashes.'md5'"): try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) malware_sample['filename'] = p_value[1:-1] except KeyError: attributes.append({ 'type': 'md5', 'object_relation': 'md5', 'value': p_value[1:-1], 'to_ids': True }) malware_sample['md5'] = p_value[1:-1] elif 'file:hashes.' in p_type: _, h = p_type.split('.') h = h[1:-1] attributes.append({ 'type': h, 'object_relation': h, 'value': p_value[1:-1] }) else: try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) except KeyError: if "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value[1:-1], 'to_ids': True }) if 'data' in malware_sample: value = "{}|{}".format(malware_sample['filename'], malware_sample['md5']) attributes.append({ 'type': 'malware-sample', 'object_relation': 'malware-sample', 'value': value, 'to_ids': True, 'data': io.BytesIO(malware_sample['data'].encode()) }) return attributes def observable_pe(self, observable): extension = observable['1']['extensions']['windows-pebinary-ext'] sections = extension['sections'] pe = MISPObject('pe') pe_uuid = str(uuid.uuid4()) pe.uuid = pe_uuid self.fill_object_attributes_observable(pe, pe_mapping, extension) for section in sections: pe_section = MISPObject('pe-section') if 'hashes' in section: for h_type, h_value in section['hashes'].items(): h_type = h_type.lower().replace('-', '') pe_section.add_attribute( **{ 'type': h_type, 'object_relation': h_type, 'value': h_value, 'to_ids': False }) self.fill_object_attributes_observable(pe_section, pe_section_mapping, section) section_uuid = str(uuid.uuid4()) pe_section.uuid = section_uuid pe.add_reference(section_uuid, 'included-in') self.misp_event.add_object(**pe_section) self.misp_event.add_object(**pe) return observable_file(observable), pe_uuid @staticmethod def fill_object_attributes_observable(misp_object, mapping_dict, stix_object): for stix_type, value in stix_object.items(): try: mapping = mapping_dict[stix_type] misp_object.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': value, 'to_ids': False }) except KeyError: if stix_type.startswith("x_misp_"): attribute_type, relation = parse_custom_property(stix_type) misp_object.add_attribute( **{ 'type': attribute_type, 'object_relation': relation[:-1], 'value': value, 'to_ids': False }) def pattern_pe(self, pattern): attributes = [] sections = defaultdict(dict) pe = MISPObject('pe') pe_uuid = str(uuid.uuid4()) pe.uuid = pe_uuid for p in pattern: p_type, p_value = p.split(' = ') p_value = p_value[1:-1] if ':extensions.' in p_type: if '.sections[' in p_type: p_type_list = p_type.split('.') stix_type = "hashes.{}".format( p_type_list[4] [1:-1]) if '.hashes.' in p_type else p_type_list[3] sections[p_type_list[2]][stix_type] = p_value else: stix_type = p_type.split('.')[-1] try: mapping = pe_mapping[stix_type] pe.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value, 'to_ids': True }) except KeyError: if stix_type.startswith("x_misp_"): attribute_type, relation = parse_custom_property( stix_type) pe.add_attribute( **{ 'type': attribute_type, 'object_relation': relation[:-2], 'value': p_value, 'to_ids': False }) else: if 'file:hashes.' in p_type: _, h = p_type.split('.') h = h[1:-1] attributes.append({ 'type': h, 'object_relation': h, 'value': p_value, 'to_ids': True }) else: try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value, 'to_ids': True }) except KeyError: if "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value, 'to_ids': True }) for _, section in sections.items(): pe_section = MISPObject('pe-section') for stix_type, value in section.items(): if 'hashes.' in stix_type: h_type = stix_type.split('.')[1] pe_section.add_attribute( **{ 'type': h_type, 'object_relation': h_type, 'value': value, 'to_ids': True }) else: try: mapping = pe_section_mapping[stix_type] pe_section.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': value, 'to_ids': True }) except KeyError: if "x_misp_" in stix_type: attribute_type, relation = stix_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': value, 'to_ids': True }) section_uuid = str(uuid.uuid4()) pe_section.uuid = pe_uuid pe.add_reference(section_uuid, 'included-in') self.misp_event.add_object(**pe_section) self.misp_event.add_object(**pe) return attributes, pe_uuid def buildExternalDict(self): self.fetch_report() for o in self.event: object_type = o._type if object_type in ('relationship', 'report'): continue if object_type in galaxy_types: self.parse_external_galaxy(o) elif object_type == 'vulnerability': attribute = {'type': 'vulnerability', 'value': o.get('name')} if 'description' in o: attribute['comment'] = o.get('description') self.misp_event.add_attribute(**attribute) elif object_type == 'course-of-action': self.parse_course_of_action(o) elif object_type == 'indicator': pattern = o.get('pattern') self.parse_external_pattern(pattern) attribute = { 'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': pattern } misp_object = { 'name': 'stix2-pattern', 'meta-category': 'stix2-pattern', 'Attribute': [self.version_attribute, attribute] } self.misp_event.add_object(**misp_object) def fetch_report(self): reports = [] for o in self.event: if o._type == 'report': reports.append(o) if len(reports) == 1: self.report = reports[0] self.parse_report() def parse_external_galaxy(self, o): galaxy = {'name': galaxy_types[o._type]} if 'kill_chain_phases' in o: galaxy['type'] = o['kill_chain_phases'][0].get('phase_name') cluster = defaultdict(dict) cluster['value'] = o.get('name') cluster['description'] = o.get('description') if 'aliases' in o: aliases = [] for a in o.get('aliases'): aliases.append(a) cluster['meta']['synonyms'] = aliases galaxy['GalaxyCluster'] = [cluster] self.misp_event['Galaxy'].append(galaxy) def parse_external_pattern(self, pattern): if ' OR ' in pattern and ' AND ' not in pattern: pattern = pattern.split('OR') for p in pattern: attribute = self.attribute_from_external_pattern(p) self.misp_event.add_attribute(**attribute) elif ' OR ' not in pattern and ' LIKE ' not in pattern: pattern = pattern.split('AND') if len(pattern) == 1: attribute = self.attribute_from_external_pattern(pattern[0]) self.misp_event.add_attribute(**attribute) @staticmethod def attribute_from_external_pattern(pattern): pattern_type, pattern_value = pattern.split(' = ') pattern_type, pattern_value = pattern_type[1:].strip( ), pattern_value[1:-2].strip() stix_type, value_type = pattern_type.split(':') if 'hashes' in value_type and 'x509' not in stix_type: h_type = value_type.split('.')[1] return {'type': h_type, 'value': pattern_value} # Might cause some issues, need more examples to test return { 'type': external_pattern_mapping[stix_type][value_type].get('type'), 'value': pattern_value } def set_distribution(self): for attribute in self.misp_event.attributes: attribute.distribution = self.__attribute_distribution for misp_object in self.misp_event.objects: misp_object.distribution = self.__attribute_distribution for attribute in misp_object.attributes: attribute.distribution = self.__attribute_distribution def saveFile(self): eventDict = self.misp_event.to_json() outputfile = '{}.stix2'.format(self.filename) with open(outputfile, 'w') as f: f.write(eventDict) @staticmethod def getTimestampfromDate(stix_date): try: return int(stix_date.timestamp()) except AttributeError: return int( time.mktime( time.strptime( stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S"))) @staticmethod def get_misp_type(labels): return labels[0].split('=')[1][1:-1] @staticmethod def get_misp_category(labels): return labels[1].split('=')[1][1:-1] @staticmethod def parse_observable(observable, attribute_type): return misp_types_mapping[attribute_type](observable, attribute_type) @staticmethod def parse_pattern(pattern): if ' AND ' in pattern: pattern_parts = pattern.split(' AND ') if len(pattern_parts) == 3: _, value1 = pattern_parts[2].split(' = ') _, value2 = pattern_parts[0].split(' = ') return '{}|{}'.format(value1[1:-2], value2[1:-1]) else: _, value1 = pattern_parts[0].split(' = ') _, value2 = pattern_parts[1].split(' = ') if value1 in ("'ipv4-addr'", "'ipv6-addr'"): return value2[1:-2] return '{}|{}'.format(value1[1:-1], value2[1:-2]) else: return pattern.split(' = ')[1][1:-2] def parse_pattern_with_data(self, pattern): if 'artifact:payload_bin' not in pattern: return self.parse_pattern(pattern) pattern_parts = pattern.split(' AND ') if len(pattern_parts) == 3: filename = pattern_parts[0].split(' = ')[1] md5 = pattern_parts[1].split(' = ')[1] return "{}|{}".format( filename[1:-1], md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2] return pattern_parts[0].split(' = ')[1][1:-1], pattern_parts[1].split( ' = ')[1][1:-2]
def check_hashes(self): if self.offline_mode: self.log('error', 'Offline mode, unable to query VirusTotal') return event_id = self._get_eventid() if event_id is None: return event = self.misp.get(event_id) if self._has_error_message(event): return misp_event = MISPEvent() misp_event.load(event) event_hashes = [] sample_hashes = [] base_new_attributes = {} for a in misp_event.attributes: h = None if a.type in ('md5', 'sha1', 'sha256'): h = a.value event_hashes.append(h) elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256', 'malware-sample'): h = a.value.split('|')[1] event_hashes.append(h) if h is not None: base_new_attributes[h] = {"category": a.category, "comment": '{} - Xchecked via VT: {}'.format(a.comment, h), "to_ids": a.to_ids, "Tag": a.Tag, "distribution": a.distribution} unk_vt_hashes = [] vt_request = {'apikey': cfg.virustotal.virustotal_key} # Make sure to start getting reports for the longest possible hashes (reduce risks of collisions) hashes_to_check = sorted(event_hashes, key=len) original_attributes = len(misp_event.attributes) if cfg.virustotal.virustotal_has_private_key is False: quota = 4 timeout = datetime.datetime.now() + datetime.timedelta(minutes=1) while len(hashes_to_check) > 0: vt_request['resource'] = hashes_to_check.pop() try: response = requests.post(cfg.misp.misp_vturl, data=vt_request, proxies=cfg.virustotal.proxies) except requests.ConnectionError: self.log('error', 'Failed to connect to VT for {}'.format(vt_request['resource'])) return if response.status_code == 403: self.log('error', 'This command requires virustotal API key') self.log('error', 'Please check that your key have the right permissions') return try: result = response.json() except: self.log('error', 'Unable to get the report of {}'.format(vt_request['resource'])) continue if result['response_code'] == 1: md5 = result['md5'] sha1 = result['sha1'] sha256 = result['sha256'] hashes_to_check = [eh for eh in hashes_to_check if eh not in (md5, sha1, sha256)] link = [False, result['permalink']] # Do not re-add a link for a in misp_event.attributes: if a.value == link[1]: link[0] = True if md5 in sample_hashes: self.log('success', 'Sample available in MISP:') else: self.log('success', 'Sample available in VT:') if self.args.populate: misp_event = self._prepare_attributes(md5, sha1, sha256, link, base_new_attributes, event_hashes, sample_hashes, misp_event) self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(link[1], md5, sha1, sha256)) if cfg.virustotal.virustotal_has_private_key is False: if quota > 0: quota -= 1 else: waiting_time = (timeout - datetime.datetime.now()).seconds if waiting_time > 0: self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time)) time.sleep(waiting_time) quota = 4 timeout = datetime.datetime.now() + datetime.timedelta(minutes=1) else: unk_vt_hashes.append(vt_request['resource']) if self.args.populate: self.__populate(misp_event, original_attributes) if len(unk_vt_hashes) > 0: self.log('error', 'Unknown on VT:') for h in unk_vt_hashes: self.log('item', '{}'.format(h))
def misp_send(self, strMISPEventID, strInput, strInfo, strUsername): # Establish communication with MISP # event = MISPEvent() # event.info = 'Test event' # event.analysis = 0 # event.distribution = 3 # event.threat_level_id = 2 # event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41') # #<add more attributes> # self.misp.add_event(event) # exit() try: objects = [] #get comments and tags from string input str_comment, tags = self.get_comm_and_tags(strInput) print(tags) if tags == None: self.misp_logger.info('Irate not in Tags: %s equals None' % tags) response = None return response #setup misp objects mispobj_email = MISPObject(name="email") mispobj_file = MISPObject(name="file") mispobj_files = {} mispobj_domainip = MISPObject(name="domain-ip") url_no = 0 file_no = 0 mispobj_urls = {} #process input for line in strInput.splitlines(): if ("domain:" in line.lower() ): #Catch domain and add to domain/IP object mispobj_domainip = MISPObject(name="domain-ip") vals = line.split(":", 1) mispobj_domainip.add_attribute("domain", value=vals[1].strip(), comment=str_comment) objects.append(mispobj_domainip) elif ("ip:" in line.lower()) or ("ip-dst:" in line.lower( )) or ("ip-src:" in line.lower()): #Catch IP and add to domain/IP object if "domain:" in strInput.splitlines(): mispobj_domainip = MISPObject(name="domain-ip") vals = line.split(":", 1) mispobj_domainip.add_attribute("ip", value=vals[1].strip(), comment=str_comment) objects.append(mispobj_domainip) else: mispobj_network_connection = MISPObject( name="network-connection") vals = line.split(":", 1) if ("ip:" in line.lower()) or ("ip-dst:" in line.lower()): mispobj_network_connection.add_attribute( "ip-dst", type="ip-dst", value=vals[1].strip(), comment=str_comment) else: mispobj_network_connection.add_attribute( "ip-src", type="ip-src", value=vals[1].strip(), comment=str_comment) objects.append(mispobj_network_connection) elif ("source-email:" in line.lower()) or ("email-source" in line.lower()) or ( "from:" in line.lower() ): #Catch email and add to email object vals = line.split(":", 1) mispobj_email.add_attribute("from", value=vals[1].strip(), comment=str_comment) elif ("url:" in line.lower()) or ( ('kit:' in line.lower() or ('creds:' in line.lower())) and (('hxxp' in line.lower()) or ('http' in line.lower())) ): #Catch URL and add to URL object vals = line.split(":", 1) url = vals[1].strip() url = refang(url) parsed = urlparse(url) mispobj_url = MISPObject(name="url") mispobj_url.add_attribute("url", value=parsed.geturl(), category="Payload delivery", comment=str_comment) if parsed.hostname: mispobj_url.add_attribute("host", value=parsed.hostname, comment=str_comment) if parsed.scheme: mispobj_url.add_attribute("scheme", value=parsed.scheme, comment=str_comment) if parsed.port: mispobj_url.add_attribute("port", value=parsed.port, comment=str_comment) mispobj_urls[url_no] = mispobj_url url_no += 1 #Catch different hashes and add to file object elif ("sha1:" in line.lower()) or ("SHA1:" in line): vals = line.split(":", 1) mispobj_file.add_attribute("sha1", value=vals[1].strip(), comment=str_comment) elif ("sha256:" in line.lower()) or ("SHA256:" in line): vals = line.split(":", 1) mispobj_file.add_attribute("sha256", value=vals[1].strip(), comment=str_comment) elif ("md5:" in line.lower()) or ("MD5:" in line): vals = line.split(":", 1) mispobj_file.add_attribute("md5", value=vals[1].strip(), comment=str_comment) elif ( "subject:" in line.lower() ): #or ("subject:" in line): #Catch subject and add to email object self.misp_logger.info('adding subject') vals = line.split(":", 1) mispobj_email.add_attribute("subject", value=vals[1].strip(), comment=str_comment) elif ("hash|filename:" in line.lower() ): #catch hash|filename pair and add to file object vals = line.split(":", 1) val = vals[1].split("|") l_hash = val[0] l_filename = val[1] l_mispobj_file = MISPObject(name="file") if len(re.findall(r"\b[a-fA-F\d]{32}\b", l_hash)) > 0: l_mispobj_file.add_attribute("md5", value=l_hash.strip(), comment=str_comment) l_mispobj_file.add_attribute("filename", value=l_filename.strip(), comment=str_comment) mispobj_files[file_no] = l_mispobj_file elif len(re.findall(r'\b[0-9a-f]{40}\b', l_hash)) > 0: l_mispobj_file.add_attribute("sha1", value=l_hash.strip(), comment=str_comment) l_mispobj_file.add_attribute("filename", value=l_filename.strip(), comment=str_comment) mispobj_files[file_no] = l_mispobj_file elif len(re.findall(r'\b[A-Fa-f0-9]{64}\b', l_hash)) > 0: l_mispobj_file.add_attribute("sha256", value=l_hash.strip(), comment=str_comment) l_mispobj_file.add_attribute("filename", value=l_filename.strip(), comment=str_comment) mispobj_files[file_no] = l_mispobj_file file_no += 1 #add all misp objects to List to be processed and submitted to MISP server as one. if len(mispobj_file.attributes) > 0: objects.append(mispobj_file) if len(mispobj_email.attributes) > 0: objects.append(mispobj_email) for u_key, u_value in mispobj_urls.items(): if len(u_value.attributes) > 0: objects.append(u_value) for f_key, f_value in mispobj_files.items(): if len(f_value.attributes) > 0: objects.append(f_value) # Update timestamp and event except Exception as e: error = traceback.format_exc() response = "Error occured when converting string to misp objects:\n %s" % error self.misp_logger.error(response) return response if self.check_object_length(objects) != True: self.misp_logger.error( 'Input from %s did not contain accepted tags.\n Input: \n%s' % (strUsername, strInput)) return "Error in the tags you entered. Please see the guide for accepted tags." try: # self.misp_logger.error(dir(self.misp)) misp_event = MISPEvent() misp_event.info = strInfo misp_event.distribution = 0 misp_event.analysis = 2 misp_event.threat_level_id = 3 # event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41') #event = self.misp.new_event(info=strInfo, distribution='0', analysis='2', threat_level_id='3', published=False) #misp_event = MISPEvent() #misp_event.load(event) add = self.misp.add_event(misp_event) self.misp_logger.info("Added event %s" % add) a, b = self.submit_to_misp(self.misp, misp_event, objects) for tag in tags: self.misp.tag(misp_event.uuid, tag) #self.misp.add_internal_comment(misp_event.id, reference="Author: " + strUsername, comment=str_comment) ccc = self.misp.publish(misp_event, alert=False) self.misp_logger.info(ccc) misp_event = self.misp.get_event(misp_event) response = misp_event #for response in misp_event: if ('errors' in response and response['errors'] != None): return ("Submission error: " + repr(response['errors'])) else: if response['Event']['RelatedEvent']: e_related = "" for each in response['Event']['RelatedEvent']: e_related = e_related + each['Event']['id'] + ", " return "Created ID: " + str( response['Event'] ['id']) + "\nRelated Events: " + ''.join(e_related) else: return "Created ID: " + str(response['Event']['id']) except Exception as e: error = traceback.format_exc() response = "Error occured when submitting to misp:\n %s" % error self.misp_logger.error(response) return response
class ReportGenerator(): def __init__(self, profile="daily_report"): self.taxonomies = Taxonomies() self.report = '' profile_name = "profiles.{}".format(profile) self.template = importlib.import_module(name=profile_name) def from_remote(self, event_id): from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert misp = PyMISP(misp_url, misp_key, misp_verifycert) result = misp.get(event_id) self.misp_event = MISPEvent() self.misp_event.load(result) def from_file(self, path): self.misp_event = MISPEvent() self.misp_event.load_file(path) def attributes(self): if not self.misp_event.attributes: return '' list_attributes = [] for attribute in self.misp_event.attributes: if attribute.type in self.template.types_to_attach: list_attributes.append("* {}".format(defang(attribute.value))) for obj in self.misp_event.Object: if obj.name in self.template.objects_to_attach: for attribute in obj.Attribute: if attribute.type in self.template.types_to_attach: list_attributes.append("* {}".format(defang(attribute.value))) return self.template.attributes.format(list_attributes="\n".join(list_attributes)) def _get_tag_info(self, machinetag): return self.taxonomies.revert_machinetag(machinetag) def report_headers(self): content = {'org_name': 'name', 'date': date.today().isoformat()} self.report += self.template.headers.format(**content) def event_level_tags(self): if not self.misp_event.Tag: return '' for tag in self.misp_event.Tag: # Only look for TLP for now if tag['name'].startswith('tlp'): tax, predicate = self._get_tag_info(tag['name']) return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) def title(self): internal_id = '' summary = '' # Get internal refs for report for obj in self.misp_event.Object: if obj.name != 'report': continue for a in obj.Attribute: if a.object_relation == 'case-number': internal_id = a.value if a.object_relation == 'summary': summary = a.value return self.template.title.format(internal_id=internal_id, title=self.misp_event.info, summary=summary) def asciidoc(self, lang='en'): self.report += self.title() self.report += self.event_level_tags() self.report += self.attributes()