def generate_MISP_Event(deduplicated_observations, conf, tags, attr_tags): dt = datetime.now() event = MISPEvent() event.info = dt.strftime("%Y%m%d ") + 'TIE' event.publish_timestamp = dt.strftime("%s") event.timestamp = dt.strftime("%s") event['timestamp'] = dt.strftime("%s") event.analysis = 2 event.published = conf.event_published orgc = MISPOrganisation() orgc.from_json(json.dumps({'name': conf.org_name, 'uuid': conf.org_uuid})) event.orgc = orgc event.threat_level_id = conf.event_base_thread_level event.date = dt event['uuid'] = str(uuid.uuid1()) if len(tags) > 0: event['Tag'] = tags attr_hashes = [] for key, attr in deduplicated_observations.items(): misp_attr = MISPAttribute() misp_attr.timestamp = dt.strftime("%s") misp_attr['timestamp'] = dt.strftime("%s") misp_attr.type = get_Attribute_Type(attr) misp_attr.value = get_MISP_Fitted_Value(attr["value"], misp_attr.type) if 'c2-server' in attr['categories'] and attr_tags.c2tags: misp_attr['Tag'] = attr_tags.c2tags if 'malware' in attr['categories'] and attr_tags.malwaretags: misp_attr['Tag'] = attr_tags.malwaretags if 'espionage' in attr['categories'] and attr_tags.espionagetags: misp_attr['Tag'] = attr_tags.espionagetags if 'bot' in attr['categories'] and attr_tags.bottags: misp_attr['Tag'] = attr_tags.bottags if 'whitelist' in attr['categories'] and attr_tags.whitelisttags: misp_attr['Tag'] = attr_tags.whitelisttags if 'cybercrime' in attr['categories'] and attr_tags.cybercrimetags: misp_attr['Tag'] = attr_tags.cybercrimetags if 'phishing' in attr['categories'] and attr_tags.phishingtags: misp_attr['Tag'] = attr_tags.phishingtags misp_attr.category = get_Attribute_Category(attr) if conf.attr_to_ids and attr[ 'min_confidence'] >= conf.attr_to_ids_threshold: misp_attr.to_ids = True else: misp_attr.to_ids = False misp_attr['comment'] = 'categories: ' + str(attr['categories']) + ' actors: ' + str(attr['actors']) + \ ' families: ' + str(attr['families']) + ' sources: ' + str(attr['sources']) + \ ' severity: ' + str(attr['max_severity']) + \ ' confidence: ' + str(attr['max_confidence']) misp_attr.edited = False event.add_attribute(**(misp_attr.to_dict())) attr_hashes.append([ hashlib.md5(attr['value'].encode("utf-8")).hexdigest(), event['uuid'] ]) event.edited = False return event, attr_hashes
def submit_tf_update(misp: ExpandedPyMISP, attributes: list) -> MISPEvent: """ create/update abuse.ch MISP-Event and append the new attributes """ eventinfo = event_info_template.format( datetime.now().strftime(info_dateformat)) # logging.debug(eventinfo) events = misp.search(controller='events', eventinfo=eventinfo, org=1, pythonify=True) if events: # current event exists already event = events[0] else: # create a new event event = MISPEvent() event.distribution = event_distribution event.threat_level_id = event_threat_level event.analysis = 2 event.info = eventinfo for tag in tagging: event.add_tag(tag) event = misp.add_event(event, pythonify=True) for att in attributes: event.add_attribute(**att) event.published = autopublish return misp.update_event(event)
class PassiveDNSParser(): def __init__(self, attribute, authentication): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.pdns = pypdns.PyPDNS(basic_auth=authentication) def get_results(self): if hasattr(self, 'result'): return self.result event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def parse(self, value): try: results = self.pdns.query(self.attribute.value) except Exception: self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} return mapping = {'count': 'counter', 'origin': 'text', 'time_first': 'datetime', 'rrtype': 'text', 'rrname': 'text', 'rdata': 'text', 'time_last': 'datetime'} for result in results: pdns_object = MISPObject('passive-dns') for relation, attribute_type in mapping.items(): pdns_object.add_attribute(relation, type=attribute_type, value=result[relation]) pdns_object.add_reference(self.attribute.uuid, 'associated-to') self.misp_event.add_object(**pdns_object)
def process(self, task: Task) -> None: # type: ignore config = task.get_payload("config") family = task.headers["family"] dhash = config_dhash(config) # Parse the config using iocextract library iocs = parse(family, config) if not iocs: # Nothing actionable found - skip the config return # Upload structured data to MISP event = MISPEvent() event.uuid = str(uuid5(self.CONFIG_NAMESPACE, dhash)) event.add_tag(f"mwdb:family:{family}") event.info = f"Malware configuration ({family})" if self.mwdb_url is not None: event.add_attribute("link", f"{self.mwdb_url}/config/{dhash}") for o in iocs.to_misp(): event.add_object(o) misp = ExpandedPyMISP(self.misp_url, self.misp_key, self.misp_verifycert) misp.add_event(event)
def import_all(self, stations_short_names, interval, data_type): object_creator = getattr(self, f'{interval}_flask_{data_type}') if data_type == 'co2': base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/' elif data_type in ['c13', 'o18']: base_url = 'http://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/' for station in stations_short_names: url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv' infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations' filepath = self.fetch(url) if not filepath: continue update = True event = self.get_existing_event_to_update(infofield) if event: location = event.get_objects_by_name('geolocation')[0] if not event: event = MISPEvent() event.info = infofield event.add_tag(getattr(self, f'tag_{station}')()) location = getattr(self, f'geolocation_{station}')() event.add_object(location) event.add_attribute('link', f'http://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}') update = False object_creator(event, location, filepath, update) if update: self.misp.update_event(event) else: self.misp.add_event(event)
def load_openioc(openioc): # Takes a opened file, or a string if not has_bs4: raise Exception('You need to install BeautifulSoup: pip install bs4') misp_event = MISPEvent() iocreport = BeautifulSoup(openioc, "html.parser") # Set event fields info = extract_field(iocreport, 'short_description') if info: misp_event.info = info date = extract_field(iocreport, 'authored_date') if date: misp_event.set_date(date) # Set special attributes description = extract_field(iocreport, 'description') if description: if not misp_event.info: misp_event.info = description else: misp_event.add_attribute('comment', description) if not misp_event.info: misp_event.info = 'OpenIOC import' author = extract_field(iocreport, 'authored_by') if author: misp_event.add_attribute('comment', author) misp_event = set_all_attributes(iocreport, misp_event) return misp_event
def create_new_event(): me = MISPEvent() me.info = "Fail2Ban blocking" me.add_tag(args.tag) start = datetime.now() me.add_attribute('datetime', start.isoformat(), comment='Start Time') return me
def test_sync_all_communities(self): '''Simple event, all communities, enable automatic push on two sub-instances''' event = MISPEvent() event.info = 'Event created on first instance - test_sync_all_communities' event.distribution = Distribution.all_communities event.add_attribute('ip-src', '1.1.1.1') try: source = self.instances[0] server = source.site_admin_connector.update_server( {'push': True}, source.sync_servers[0].id) self.assertTrue(server.push) middle = self.instances[1] middle.site_admin_connector.update_server( {'push': True}, middle.sync_servers[1].id ) # Enable automatic push to 3rd instance last = self.instances[2] event = source.user_connector.add_event(event) source.org_admin_connector.publish(event) source.site_admin_connector.server_push(source.sync_servers[0]) time.sleep(30) middle_event = middle.user_connector.get_event(event.uuid) self.assertEqual(event.attributes[0].value, middle_event.attributes[0].value) last_event = last.user_connector.get_event(event.uuid) self.assertEqual(event.attributes[0].value, last_event.attributes[0].value) finally: source.org_admin_connector.delete_event(event) middle.site_admin_connector.delete_event(middle_event) last.site_admin_connector.delete_event(last_event) source.site_admin_connector.update_server( {'push': False}, source.sync_servers[0].id) middle.site_admin_connector.update_server( {'push': False}, middle.sync_servers[1].id)
class PassiveSSLParser(): def __init__(self, attribute, authentication): self.misp_event = MISPEvent() self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.misp_event.add_attribute(**self.attribute) self.pssl = pypssl.PyPSSL(basic_auth=authentication) self.cert_hash = 'x509-fingerprint-sha1' self.cert_type = 'pem' self.mapping = {'issuer': ('text', 'issuer'), 'keylength': ('text', 'pubkey-info-size'), 'not_after': ('datetime', 'validity-not-after'), 'not_before': ('datetime', 'validity-not-before'), 'subject': ('text', 'subject')} def get_results(self): if hasattr(self, 'result'): return self.result event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def parse(self): value = self.attribute.value.split('|')[0] if '|' in self.attribute.type else self.attribute.value try: results = self.pssl.query(value) except Exception: self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'} return if not results: self.result = {'error': 'Not found'} return for ip_address, certificates in results.items(): ip_uuid = self._handle_ip_attribute(ip_address) for certificate in certificates['certificates']: self._handle_certificate(certificate, ip_uuid) def _handle_certificate(self, certificate, ip_uuid): x509 = MISPObject('x509') x509.add_attribute(self.cert_hash, type=self.cert_hash, value=certificate) cert_details = self.pssl.fetch_cert(certificate) info = cert_details['info'] for feature, mapping in self.mapping.items(): attribute_type, object_relation = mapping x509.add_attribute(object_relation, type=attribute_type, value=info[feature]) x509.add_attribute(self.cert_type, type='text', value=self.cert_type) x509.add_reference(ip_uuid, 'seen-by') self.misp_event.add_object(**x509) def _handle_ip_attribute(self, ip_address): if ip_address == self.attribute.value: return self.attribute.uuid ip_attribute = MISPAttribute() ip_attribute.from_dict(**{'type': self.attribute.type, 'value': ip_address}) self.misp_event.add_attribute(**ip_attribute) return ip_attribute.uuid
def create_simple_event(): event = MISPEvent() event.info = 'This is a super simple test' event.distribution = Distribution.your_organisation_only event.threat_level_id = ThreatLevel.low event.analysis = Analysis.completed event.add_attribute('text', str(uuid.uuid4())) return event
def create_simple_event(self, force_timestamps=False): mispevent = MISPEvent(force_timestamps=force_timestamps) mispevent.info = 'This is a super simple test' mispevent.distribution = Distribution.your_organisation_only mispevent.threat_level_id = ThreatLevel.low mispevent.analysis = Analysis.completed mispevent.add_attribute('text', str(uuid4())) return mispevent
class VirusTotalParser(): def __init__(self): super(VirusTotalParser, self).__init__() self.misp_event = MISPEvent() def declare_variables(self, apikey, attribute): self.attribute = MISPAttribute() self.attribute.from_dict(**attribute) self.apikey = apikey def get_result(self): event = json.loads(self.misp_event.to_json())['Event'] results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} def parse_urls(self, query_result): for feature in ('detected_urls', 'undetected_urls'): if feature in query_result: for url in query_result[feature]: value = url['url'] if isinstance(url, dict) else url[0] self.misp_event.add_attribute('url', value) def parse_resolutions(self, resolutions, subdomains=None, uuids=None): domain_ip_object = MISPObject('domain-ip') if self.attribute.type == 'domain': domain_ip_object.add_attribute('domain', type='domain', value=self.attribute.value) attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address') else: domain_ip_object.add_attribute('ip', type='ip-dst', value=self.attribute.value) attribute_type, relation, key = ('domain', 'domain', 'hostname') for resolution in resolutions: domain_ip_object.add_attribute(relation, type=attribute_type, value=resolution[key]) if subdomains: for subdomain in subdomains: attribute = MISPAttribute() attribute.from_dict(**dict(type='domain', value=subdomain)) self.misp_event.add_attribute(**attribute) domain_ip_object.add_reference(attribute.uuid, 'subdomain') if uuids: for uuid in uuids: domain_ip_object.add_reference(uuid, 'sibling-of') self.misp_event.add_object(**domain_ip_object) def parse_vt_object(self, query_result): vt_object = MISPObject('virustotal-report') vt_object.add_attribute('permalink', type='link', value=query_result['permalink']) detection_ratio = '{}/{}'.format(query_result['positives'], query_result['total']) vt_object.add_attribute('detection-ratio', type='text', value=detection_ratio) self.misp_event.add_object(**vt_object) def get_query_result(self, query_type): params = {query_type: self.attribute.value, 'apikey': self.apikey} return requests.get(self.base_url, params=params)
def convertTEtoMISP(self, teevent): """ Convert a ThreatExchange entry to MISP entry """ # Create empty event mispevt = MISPEvent() mispevt.info = "[Facebook ThreatExchange]" mispevt.distribution = 0 mispevt.sharing_group_id = self.privacy_levels[teevent["privacy_type"]] # Check if event is to be kept if "status" in teevent.keys() and teevent["status"] in self.score.keys() and self.score[teevent["status"]] < self.badness_threshold : print("IGNORE EVENT %s due to status (%s)" % (teevent, teevent["status"])) return None # Add indicator to event if "raw_indicator" in teevent.keys(): if "type" in teevent.keys(): if teevent["type"] in self.type_map.keys(): indicator = teevent["raw_indicator"].replace("\\", "") mispevt.add_attribute(self.type_map[teevent["type"]] , indicator) # not to brutal?? else: print("WARNING: TYPE %s SHOULD BE ADDED TO MAPPING" % teevent["type"]) else: print("WARNING, event %s does not contains any indicator :(" % teevent) return None # don't create event without content! # Add a category mispevt.category = "Network activity" # Enrich description if "description" in teevent.keys(): mispevt.info = mispevt.info + " - %s" % teevent["description"] if "owner" in teevent.keys() and "name" in teevent["owner"].keys(): owner = teevent["owner"]["name"] if("email" in teevent["owner"].keys()): email = teevent["owner"]["email"].replace("\\u0040", "@") else: email = "" mispevt.info = mispevt.info + " - by %s (%s)" % (owner, email) # Add sharing indicators (tags) if "share_level" in teevent.keys(): if teevent["share_level"] in self.share_levels.keys(): mispevt.Tag.append(self.share_levels[teevent["share_level"]]) else: print("WARNING: SHARING LEVEL %s SHOULD BE ADDED TO MAPPING" % teevent["share_level"]) if self.extra_tag is not None: mispevt.Tag.append(self.extra_tag) evtid = teevent["id"] return [evtid, mispevt]
def create_response(original_attribute: dict, software: str, signature: Optional[str] = None) -> dict: misp_event = MISPEvent() if signature: misp_event.add_attribute(**original_attribute) av_signature_object = MISPObject("av-signature") av_signature_object.add_attribute("signature", signature) av_signature_object.add_attribute("software", software) av_signature_object.add_reference(original_attribute["uuid"], "belongs-to") misp_event.add_object(av_signature_object) event = json.loads(misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {"results": results}
def test_first_last_seen(self): me = MISPEvent() me.info = 'Test First and Last Seen' me.date = '2020.01.12' self.assertEqual(me.date.day, 12) me.add_attribute('ip-dst', '8.8.8.8', first_seen='06-21-1998', last_seen=1580213607.469571) self.assertEqual(me.attributes[0].first_seen.year, 1998) self.assertEqual(me.attributes[0].last_seen.year, 2020) now = datetime.now().astimezone() me.attributes[0].last_seen = now today = date.today() me.attributes[0].first_seen = today self.assertEqual(me.attributes[0].first_seen.year, today.year) self.assertEqual(me.attributes[0].last_seen, now)
def parse_result(attribute, values): event = MISPEvent() initial_attribute = MISPAttribute() initial_attribute.from_dict(**attribute) event.add_attribute(**initial_attribute) mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')} print(values) for last_seen, response in values['response'].items(): asn = MISPObject('asn') asn.add_attribute('last-seen', **{'type': 'datetime', 'value': last_seen}) for feature, attribute_fields in mapping.items(): attribute_type, object_relation = attribute_fields asn.add_attribute(object_relation, **{'type': attribute_type, 'value': response[feature]}) asn.add_reference(initial_attribute.uuid, 'related-to') event.add_object(**asn) event = json.loads(event.to_json()) return {key: event[key] for key in ('Attribute', 'Object')}
def test_feed(self): me = MISPEvent() me.info = 'Test feed' org = MISPOrganisation() org.name = 'TestOrg' org.uuid = '123478' me.Orgc = org me.add_attribute('ip-dst', '8.8.8.8') obj = me.add_object(name='file') obj.add_attributes('filename', *['foo.exe', 'bar.exe']) h = hashlib.new('md5') h.update(b'8.8.8.8') hash_attr_val = h.hexdigest() feed = me.to_feed(with_meta=True) self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val) self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed') self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
def test_sync_community(self): '''Simple event, this community only, pull from member of the community''' event = MISPEvent() event.info = 'Event created on first instance - test_sync_community' event.distribution = Distribution.this_community_only event.add_attribute('ip-src', '1.1.1.1') try: source = self.instances[0] dest = self.instances[1] event = source.org_admin_connector.add_event(event) source.org_admin_connector.publish(event) dest.site_admin_connector.server_pull(dest.sync_servers[0]) time.sleep(10) dest_event = dest.org_admin_connector.get_event(event.uuid) self.assertEqual(dest_event.distribution, 0) finally: source.org_admin_connector.delete_event(event) dest.site_admin_connector.delete_event(dest_event)
def test_simple_sync(self): '''Test simple event, push to one server''' event = MISPEvent() event.info = 'Event created on first instance - test_simple_sync' event.distribution = Distribution.all_communities event.add_attribute('ip-src', '1.1.1.1') try: source = self.instances[0] dest = self.instances[1] event = source.org_admin_connector.add_event(event) source.org_admin_connector.publish(event) source.site_admin_connector.server_push(source.sync_servers[0], event) time.sleep(10) dest_event = dest.org_admin_connector.get_event(event.uuid) self.assertEqual(event.attributes[0].value, dest_event.attributes[0].value) finally: source.org_admin_connector.delete_event(event) dest.site_admin_connector.delete_event(dest_event)
class HashlookupParser(): def __init__(self, attribute, hashlookupresult, api_url): self.attribute = attribute self.hashlookupresult = hashlookupresult self.api_url = api_url self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.references = defaultdict(list) def get_result(self): if self.references: self.__build_references() event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object') if (key in event and event[key])} return {'results': results} def parse_hashlookup_information(self): hashlookup_object = MISPObject('hashlookup') if 'source' in self.hashlookupresult: hashlookup_object.add_attribute('source', **{'type': 'text', 'value': self.hashlookupresult['source']}) if 'KnownMalicious' in self.hashlookupresult: hashlookup_object.add_attribute('KnownMalicious', **{'type': 'text', 'value': self.hashlookupresult['KnownMalicious']}) hashlookup_object.add_attribute('MD5', **{'type': 'md5', 'value': self.hashlookupresult['MD5']}) hashlookup_object.add_attribute('SHA-1', **{'type': 'sha1', 'value': self.hashlookupresult['SHA-1']}) if 'SSDEEP' in self.hashlookupresult: hashlookup_object.add_attribute('SSDEEP', **{'type': 'ssdeep', 'value': self.hashlookupresult['SSDEEP']}) if 'TLSH' in self.hashlookupresult: hashlookup_object.add_attribute('TLSH', **{'type': 'tlsh', 'value': self.hashlookupresult['TLSH']}) if 'FileName' in self.hashlookupresult: hashlookup_object.add_attribute('FileName', **{'type': 'filename', 'value': self.hashlookupresult['FileName']}) if 'FileSize' in self.hashlookupresult: hashlookup_object.add_attribute('FileSize', **{'type': 'size-in-bytes', 'value': self.hashlookupresult['FileSize']}) hashlookup_object.add_reference(self.attribute['uuid'], 'related-to') self.misp_event.add_object(hashlookup_object) def __build_references(self): for object_uuid, references in self.references.items(): for misp_object in self.misp_event.objects: if misp_object.uuid == object_uuid: for reference in references: misp_object.add_reference(**reference) break
class FarsightDnsdbParser(): def __init__(self, attribute): self.attribute = attribute self.misp_event = MISPEvent() self.misp_event.add_attribute(**attribute) self.passivedns_mapping = { 'bailiwick': {'type': 'text', 'object_relation': 'bailiwick'}, 'count': {'type': 'counter', 'object_relation': 'count'}, 'raw_rdata': {'type': 'text', 'object_relation': 'raw_rdata'}, 'rdata': {'type': 'text', 'object_relation': 'rdata'}, 'rrname': {'type': 'text', 'object_relation': 'rrname'}, 'rrtype': {'type': 'text', 'object_relation': 'rrtype'}, 'time_first': {'type': 'datetime', 'object_relation': 'time_first'}, 'time_last': {'type': 'datetime', 'object_relation': 'time_last'}, 'zone_time_first': {'type': 'datetime', 'object_relation': 'zone_time_first'}, 'zone_time_last': {'type': 'datetime', 'object_relation': 'zone_time_last'} } self.comment = 'Result from an %s lookup on DNSDB about the %s: %s' def parse_passivedns_results(self, query_response): for query_type, results in query_response.items(): comment = self.comment % (query_type, TYPE_TO_FEATURE[self.attribute['type']], self.attribute['value']) for result in results: passivedns_object = MISPObject('passive-dns') if result.get('rdata') and isinstance(result['rdata'], list): for rdata in result.pop('rdata'): passivedns_object.add_attribute(**self._parse_attribute(comment, 'rdata', rdata)) for feature, value in result.items(): passivedns_object.add_attribute(**self._parse_attribute(comment, feature, value)) passivedns_object.add_reference(self.attribute['uuid'], 'related-to') self.misp_event.add_object(passivedns_object) def get_results(self): event = json.loads(self.misp_event.to_json()) results = {key: event[key] for key in ('Attribute', 'Object')} return {'results': results} def _parse_attribute(self, comment, feature, value): attribute = {'value': value, 'comment': comment} attribute.update(self.passivedns_mapping[feature]) return attribute
def create_complex_event(self): event = MISPEvent() event.info = 'Complex Event' event.distribution = Distribution.all_communities event.add_tag('tlp:white') event.add_attribute('ip-src', '8.8.8.8') event.add_attribute('ip-dst', '8.8.8.9') event.add_attribute('domain', 'google.com') event.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd43') event.attributes[0].distribution = Distribution.your_organisation_only event.attributes[1].distribution = Distribution.this_community_only event.attributes[2].distribution = Distribution.connected_communities event.attributes[0].add_tag('tlp:red') event.attributes[1].add_tag('tlp:amber') event.attributes[2].add_tag('tlp:green') obj = MISPObject('file') obj.distribution = Distribution.connected_communities obj.add_attribute('filename', 'testfile') obj.add_attribute('md5', '3c656da41f4645f77e3ec3281b63dd44') obj.attributes[0].distribution = Distribution.your_organisation_only event.add_object(obj) return event
def load_openioc(openioc): if not has_bs4: raise Exception('You need to install BeautifulSoup: pip install bs4') misp_event = MISPEvent() with open(openioc, "r") as ioc_file: iocreport = BeautifulSoup(ioc_file, "lxml") # Set event fields info = extract_field(iocreport, 'short_description') if info: misp_event.info = info date = extract_field(iocreport, 'authored_date') if date: misp_event.set_date(date) # Set special attributes description = extract_field(iocreport, 'description') if description: misp_event.add_attribute('comment', description) author = extract_field(iocreport, 'authored_by') if author: misp_event.add_attribute('comment', author) misp_event = set_all_attributes(iocreport, misp_event) return misp_event
def inserta_misp(nombre_evento, full_tweet, fverbose): #Instancio evento MISP event = MISPEvent() #Nombre del evento. Se cambiara por cada tweet recibido event.info = nombre_evento # Required #Valores por defecto event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config event.threat_level_id = 2 # Optional, defaults to MISP.default_event_threat_level in MISP config event.analysis = 1 # Optional, defaults to 0 (initial analysis) #Inserto el tweet completo #event.add_attribute('External analysis', full_tweet) event.add_attribute('text', full_tweet) event.add_tag('tlp:white') add_tweet_atributes(event, full_tweet, fverbose) #Inserto el evento MISP event = misp.add_event(event, pythonify=True)
def import_all(self, stations_short_names, interval, data_type): object_creator = getattr(self, f'{interval}_flask_{data_type}') if data_type == 'co2': base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/' elif data_type in ['c13', 'o18']: base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/' for station in stations_short_names: url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv' infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations' filepath = self.fetch(url) if not filepath: continue if infofield in self.scrippts_meta: event = MISPEvent() event.load_file(str(self.output_dir / self.scrippts_meta[infofield])) location = event.get_objects_by_name('geolocation')[0] update = True else: event = MISPEvent() event.uuid = str(uuid4()) event.info = infofield event.Orgc = self.misp_org event.add_tag(getattr(self, f'tag_{station}')()) location = getattr(self, f'geolocation_{station}')() event.add_object(location) event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}') update = False with self.scrippts_meta_file.open('a') as f: writer = csv.writer(f) writer.writerow([infofield, f'{event.uuid}.json']) object_creator(event, location, filepath, update) if update: # Bump the publish timestamp event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now()) feed_output = event.to_feed(with_meta=False) with (self.output_dir / f'{event.uuid}.json').open('w') as f: # json.dump(feed_output, f, indent=2, sort_keys=True) # For testing json.dump(feed_output, f)
def handle_hash(self, event: MISPEvent, hash_: Hash) -> MISPEvent: """Handle a single hash.""" if hash_.hash_type() == MD5: event.add_attribute("md5", str(hash_)) elif hash_.hash_type() == SHA1: event.add_attribute("sha1", str(hash_)) elif hash_.hash_type() == SHA256: event.add_attribute("sha256", str(hash_)) return event
def handle_hash(self, hash_, event: pymisp.MISPEvent): """Handle a single hash.""" if hash_.hash_type() == hash_.MD5: event.add_attribute("md5", str(hash_)) elif hash_.hash_type() == hash_.SHA1: event.add_attribute("sha1", str(hash_)) elif hash_.hash_type() == hash_.SHA256: event.add_attribute("sha256", str(hash_)) return event
def export(self, cache: 'CaptureCache', is_public_instance: bool=False) -> MISPEvent: '''Export a capture in MISP format. You can POST the return of this method directly to a MISP instance and it will create an event.''' public_domain = get_config('generic', 'public_domain') event = MISPEvent() event.info = f'Lookyloo Capture ({cache.url})' lookyloo_link: MISPAttribute = event.add_attribute('link', f'https://{public_domain}/tree/{cache.uuid}') # type: ignore if not is_public_instance: lookyloo_link.distribution = 0 initial_url = URLObject(cache.url) initial_url.comment = 'Submitted URL' self.__misp_add_ips_to_URLObject(initial_url, cache.tree.root_hartree.hostname_tree) redirects: List[URLObject] = [] for nb, url in enumerate(cache.redirects): if url == cache.url: continue obj = URLObject(url) obj.comment = f'Redirect {nb}' self.__misp_add_ips_to_URLObject(obj, cache.tree.root_hartree.hostname_tree) redirects.append(obj) if redirects: redirects[-1].comment = f'Last redirect ({nb})' if redirects: prec_object = initial_url for u_object in redirects: prec_object.add_reference(u_object, 'redirects-to') prec_object = u_object initial_obj = event.add_object(initial_url) initial_obj.add_reference(lookyloo_link, 'captured-by', 'Capture on lookyloo') for u_object in redirects: event.add_object(u_object) final_redirect = event.objects[-1] try: fo = FileObject(pseudofile=cache.tree.root_hartree.rendered_node.body, filename=cache.tree.root_hartree.rendered_node.filename) fo.comment = 'Content received for the final redirect (before rendering)' fo.add_reference(final_redirect, 'loaded-by', 'URL loading that content') event.add_object(fo) except Har2TreeError: pass except AttributeError: # No `body` in rendered node pass return event
def _create_event(self, artifact: Type[Artifact]) -> MISPEvent: """Create an event in MISP, return an Event object.""" event = MISPEvent() event.info = self.event_info.format(source_name=artifact.source_name) # Add tags. for tag in self.tags: event.add_tag(tag) # Add references. if artifact.reference_link != "": event.add_attribute("link", artifact.reference_link) if artifact.reference_text != "": event.add_attribute("text", artifact.reference_text) if artifact.source_name != "" and self.include_artifact_source_name: event.add_attribute("other", f"source:{artifact.source_name}") return event
class StixParser(): def __init__(self): self.misp_event = MISPEvent() self.event = [] self.misp_event['Galaxy'] = [] def loadEvent(self, args): try: filename = os.path.join(os.path.dirname(args[0]), args[1]) tempFile = open(filename, 'r', encoding='utf-8') self.filename = filename event = json.loads(tempFile.read()) self.stix_version = 'stix {}'.format(event.get('spec_version')) for o in event.get('objects'): try: try: self.event.append(stix2.parse(o, allow_custom=True)) except: self.parse_custom(o) except: pass if not self.event: print( json.dumps({ 'success': 0, 'message': 'There is no valid STIX object to import' })) sys.exit(1) try: event_distribution = args[2] if not isinstance(event_distribution, int): event_distribution = int( event_distribution) if event_distribution.isdigit( ) else 5 except: event_distribution = 5 try: attribute_distribution = args[3] if attribute_distribution != 'event' and not isinstance( attribute_distribution, int): attribute_distribution = int( attribute_distribution ) if attribute_distribution.isdigit() else 5 except: attribute_distribution = 5 self.misp_event.distribution = event_distribution self.__attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution self.load_mapping() except: print( json.dumps({ 'success': 0, 'message': 'The STIX file could not be read' })) sys.exit(1) def parse_custom(self, obj): custom_object_type = obj.pop('type') labels = obj['labels'] try: @stix2.CustomObject( custom_object_type, [('id', stix2.properties.StringProperty(required=True)), ('x_misp_timestamp', stix2.properties.StringProperty(required=True)), ('labels', stix2.properties.ListProperty(labels, required=True)), ('x_misp_value', stix2.properties.StringProperty(required=True)), ('created_by_ref', stix2.properties.StringProperty(required=True)), ('x_misp_comment', stix2.properties.StringProperty()), ('x_misp_category', stix2.properties.StringProperty())]) class Custom(object): def __init__(self, **kwargs): return custom = Custom(**obj) except: @stix2.CustomObject( custom_object_type, [('id', stix2.properties.StringProperty(required=True)), ('x_misp_timestamp', stix2.properties.StringProperty(required=True)), ('labels', stix2.properties.ListProperty(labels, required=True)), ('x_misp_values', stix2.properties.DictionaryProperty(required=True)), ('created_by_ref', stix2.properties.StringProperty(required=True)), ('x_misp_comment', stix2.properties.StringProperty()), ('x_misp_category', stix2.properties.StringProperty())]) class Custom(object): def __init__(self, **kwargs): return custom = Custom(**obj) self.event.append(stix2.parse(custom)) def load_mapping(self): self.objects_mapping = { 'asn': { 'observable': observable_asn, 'pattern': pattern_asn }, 'domain-ip': { 'observable': observable_domain_ip, 'pattern': pattern_domain_ip }, 'email': { 'observable': self.observable_email, 'pattern': self.pattern_email }, 'file': { 'observable': observable_file, 'pattern': self.pattern_file }, 'ip-port': { 'observable': observable_ip_port, 'pattern': pattern_ip_port }, 'network-socket': { 'observable': observable_socket, 'pattern': pattern_socket }, 'process': { 'observable': observable_process, 'pattern': pattern_process }, 'registry-key': { 'observable': observable_regkey, 'pattern': pattern_regkey }, 'url': { 'observable': observable_url, 'pattern': pattern_url }, 'WindowsPEBinaryFile': { 'observable': self.observable_pe, 'pattern': self.pattern_pe }, 'x509': { 'observable': observable_x509, 'pattern': pattern_x509 } } def handler(self): self.outputname = '{}.stix2'.format(self.filename) if self.from_misp(): self.buildMispDict() else: self.version_attribute = { 'type': 'text', 'object_relation': 'version', 'value': self.stix_version } self.buildExternalDict() self.set_distribution() def from_misp(self): for o in self.event: if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get( 'labels'): index = self.event.index(o) self.report = self.event.pop(index) return True return False def buildMispDict(self): self.parse_identity() self.parse_report() for o in self.event: try: object_type = o._type except: object_type = o['type'] labels = o.get('labels') if object_type in galaxy_types: self.parse_galaxy(o, labels) elif object_type == 'course-of-action': self.parse_course_of_action(o) elif 'x-misp-object' in object_type: if 'from_object' in labels: self.parse_custom_object(o) else: self.parse_custom_attribute(o, labels) else: if 'from_object' in labels: self.parse_object(o, labels) else: self.parse_attribute(o, labels) def parse_identity(self): identity = self.event.pop(0) org = {'name': identity.get('name')} self.misp_event['Org'] = org def parse_report(self): report = self.report self.misp_event.info = report.get('name') if report.get('published'): self.misp_event.publish_timestamp = self.getTimestampfromDate( report.get('published')) if hasattr(report, 'labels'): labels = report['labels'] for l in labels: self.misp_event.add_tag(l) if hasattr(report, 'external_references'): ext_refs = report['external_references'] for e in ext_refs: link = {"type": "link"} comment = e.get('source_name') try: comment = comment.split('url - ')[1] except: pass if comment: link['comment'] = comment link['value'] = e.get('url') self.misp_event.add_attribute(**link) def parse_galaxy(self, o, labels): galaxy_type = self.get_misp_type(labels) tag = labels[1] value = tag.split(':')[1].split('=')[1] galaxy_description, cluster_description = o.get('description').split( '|') galaxy = { 'type': galaxy_type, 'name': o.get('name'), 'description': galaxy_description, 'GalaxyCluster': [{ 'type': galaxy_type, 'value': value, 'tag_name': tag, 'description': cluster_description }] } self.misp_event['Galaxy'].append(galaxy) def parse_course_of_action(self, o): misp_object = MISPObject('course-of-action') if 'name' in o: attribute = { 'type': 'text', 'object_relation': 'name', 'value': o.get('name') } misp_object.add_attribute(**attribute) else: return if 'description' in o: attribute = { 'type': 'text', 'object_relation': 'description', 'value': o.get('description') } misp_object.add_attribute(**attribute) self.misp_event.add_object(**misp_object) def parse_custom_object(self, o): name = o.get('type').split('x-misp-object-')[1] timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp')) category = o.get('category') attributes = [] values = o.get('x_misp_values') for v in values: attribute_type, object_relation = v.split('_') attribute = { 'type': attribute_type, 'value': values.get(v), 'object_relation': object_relation } attributes.append(attribute) misp_object = { 'name': name, 'timestamp': timestamp, 'meta-category': category, 'Attribute': attributes } self.misp_event.add_object(**misp_object) def parse_custom_attribute(self, o, labels): attribute_type = o.get('type').split('x-misp-object-')[1] if attribute_type not in misp_types: attribute_type = attribute_type.replace('-', '|') timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp')) to_ids = bool(labels[1].split('=')[1]) value = o.get('x_misp_value') category = self.get_misp_category(labels) attribute = { 'type': attribute_type, 'timestamp': timestamp, 'to_ids': to_ids, 'value': value, 'category': category } self.misp_event.add_attribute(**attribute) def parse_object(self, o, labels): object_type = self.get_misp_type(labels) name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type object_category = self.get_misp_category(labels) stix_type = o._type misp_object = MISPObject(name) misp_object['meta-category'] = object_category if stix_type == 'indicator': pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ') pattern[0] = pattern[0][1:] pattern[-1] = pattern[-1][:-1] attributes = self.objects_mapping[object_type]['pattern'](pattern) if stix_type == 'observed-data': observable = o.get('objects') attributes = self.objects_mapping[object_type]['observable']( observable) if isinstance(attributes, tuple): attributes, pe_uuid = attributes misp_object.add_reference(pe_uuid, 'included-in') for attribute in attributes: misp_object.add_attribute(**attribute) misp_object.to_ids = bool(labels[1].split('=')[1]) self.misp_event.add_object(**misp_object) def parse_attribute(self, o, labels): attribute_type = self.get_misp_type(labels) attribute_category = self.get_misp_category(labels) attribute = {'type': attribute_type, 'category': attribute_category} stix_type = o._type if stix_type == 'vulnerability': value = o.get('name') else: if stix_type == 'indicator': o_date = o.get('valid_from') pattern = o.get('pattern').replace('\\\\', '\\') value = self.parse_pattern_with_data( pattern) if attribute_type in ( 'malware-sample', 'attachment') else self.parse_pattern(pattern) attribute['to_ids'] = True else: o_date = o.get('first_observed') observable = o.get('objects') try: value = self.parse_observable(observable, attribute_type) except: print('{}: {}'.format(attribute_type, observable)) attribute['to_ids'] = False attribute['timestamp'] = self.getTimestampfromDate(o_date) if 'description' in o: attribute['comment'] = o.get('description') if isinstance(value, tuple): value, data = value attribute['data'] = io.BytesIO(data.encode()) attribute['value'] = value self.misp_event.add_attribute(**attribute) @staticmethod def observable_email(observable): attributes = [] addresses = {} files = {} for o_key, o_dict in observable.items(): part_type = o_dict._type if part_type == 'email-addr': addresses[o_key] = o_dict.get('value') elif part_type == 'file': files[o_key] = o_dict.get('name') else: message = dict(o_dict) attributes.append({ 'type': 'email-src', 'object_relation': 'from', 'value': addresses[message.pop('from_ref')], 'to_ids': False }) for ref in ('to_refs', 'cc_refs'): if ref in message: for item in message.pop(ref): mapping = email_mapping[ref] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': addresses[item], 'to_ids': False }) if 'body_multipart' in message: for f in message.pop('body_multipart'): attributes.append({ 'type': 'email-attachment', 'object_relation': 'attachment', 'value': files[f.get('body_raw_ref')], 'to_ids': False }) for m_key, m_value in message.items(): if m_key == 'additional_header_fields': for field_key, field_value in m_value.items(): mapping = email_mapping[field_key] if field_key == 'Reply-To': for rt in field_value: attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': rt, 'to_ids': False }) else: attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': field_value, 'to_ids': False }) else: try: mapping = email_mapping[m_key] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': m_value, 'to_ids': False }) except: if m_key.startswith("x_misp_attachment_"): attribute_type, relation = m_key.split( "x_misp_")[1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'to_ids': False, 'value': m_value['value'], 'data': io.BytesIO(m_value['data'].encode()) }) elif "x_misp_" in m_key: attribute_type, relation = m_key.split( "x_misp_")[1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': m_value, 'to_ids': False }) return attributes @staticmethod def pattern_email(pattern): attributes = [] attachments = defaultdict(dict) for p in pattern: p_type, p_value = p.split(' = ') try: mapping = email_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) except KeyError: if p_type.startswith("email-message:'x_misp_attachment_"): relation, field = p_type.split('.') relation = relation.split(':')[1][1:-1] attachments[relation][field] = p_value[1:-1] elif "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value[1:-1], 'to_ids': True }) for a_key, a_dict in attachments.items(): _, _, attribute_type, relation = a_key.split('_') attributes.append({ 'type': attribute_type, 'object_relation': relation, 'to_ids': True, 'value': a_dict['value'], 'data': io.BytesIO(a_dict['data'].encode()) }) return attributes @staticmethod def pattern_file(pattern): attributes = [] malware_sample = {} for p in pattern: p_type, p_value = p.split(' = ') if p_type == 'artifact:payload_bin': malware_sample['data'] = p_value elif p_type in ("file:name", "file:hashes.'md5'"): try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) malware_sample['filename'] = p_value[1:-1] except KeyError: attributes.append({ 'type': 'md5', 'object_relation': 'md5', 'value': p_value[1:-1], 'to_ids': True }) malware_sample['md5'] = p_value[1:-1] elif 'file:hashes.' in p_type: _, h = p_type.split('.') h = h[1:-1] attributes.append({ 'type': h, 'object_relation': h, 'value': p_value[1:-1] }) else: try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value[1:-1], 'to_ids': True }) except KeyError: if "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value[1:-1], 'to_ids': True }) if 'data' in malware_sample: value = "{}|{}".format(malware_sample['filename'], malware_sample['md5']) attributes.append({ 'type': 'malware-sample', 'object_relation': 'malware-sample', 'value': value, 'to_ids': True, 'data': io.BytesIO(malware_sample['data'].encode()) }) return attributes def observable_pe(self, observable): extension = observable['1']['extensions']['windows-pebinary-ext'] sections = extension['sections'] pe = MISPObject('pe') pe_uuid = str(uuid.uuid4()) pe.uuid = pe_uuid self.fill_object_attributes_observable(pe, pe_mapping, extension) for section in sections: pe_section = MISPObject('pe-section') if 'hashes' in section: for h_type, h_value in section['hashes'].items(): h_type = h_type.lower().replace('-', '') pe_section.add_attribute( **{ 'type': h_type, 'object_relation': h_type, 'value': h_value, 'to_ids': False }) self.fill_object_attributes_observable(pe_section, pe_section_mapping, section) section_uuid = str(uuid.uuid4()) pe_section.uuid = section_uuid pe.add_reference(section_uuid, 'included-in') self.misp_event.add_object(**pe_section) self.misp_event.add_object(**pe) return observable_file(observable), pe_uuid @staticmethod def fill_object_attributes_observable(misp_object, mapping_dict, stix_object): for stix_type, value in stix_object.items(): try: mapping = mapping_dict[stix_type] misp_object.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': value, 'to_ids': False }) except KeyError: if stix_type.startswith("x_misp_"): attribute_type, relation = parse_custom_property(stix_type) misp_object.add_attribute( **{ 'type': attribute_type, 'object_relation': relation[:-1], 'value': value, 'to_ids': False }) def pattern_pe(self, pattern): attributes = [] sections = defaultdict(dict) pe = MISPObject('pe') pe_uuid = str(uuid.uuid4()) pe.uuid = pe_uuid for p in pattern: p_type, p_value = p.split(' = ') p_value = p_value[1:-1] if ':extensions.' in p_type: if '.sections[' in p_type: p_type_list = p_type.split('.') stix_type = "hashes.{}".format( p_type_list[4] [1:-1]) if '.hashes.' in p_type else p_type_list[3] sections[p_type_list[2]][stix_type] = p_value else: stix_type = p_type.split('.')[-1] try: mapping = pe_mapping[stix_type] pe.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value, 'to_ids': True }) except KeyError: if stix_type.startswith("x_misp_"): attribute_type, relation = parse_custom_property( stix_type) pe.add_attribute( **{ 'type': attribute_type, 'object_relation': relation[:-2], 'value': p_value, 'to_ids': False }) else: if 'file:hashes.' in p_type: _, h = p_type.split('.') h = h[1:-1] attributes.append({ 'type': h, 'object_relation': h, 'value': p_value, 'to_ids': True }) else: try: mapping = file_mapping[p_type] attributes.append({ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': p_value, 'to_ids': True }) except KeyError: if "x_misp_" in p_type: attribute_type, relation = p_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': p_value, 'to_ids': True }) for _, section in sections.items(): pe_section = MISPObject('pe-section') for stix_type, value in section.items(): if 'hashes.' in stix_type: h_type = stix_type.split('.')[1] pe_section.add_attribute( **{ 'type': h_type, 'object_relation': h_type, 'value': value, 'to_ids': True }) else: try: mapping = pe_section_mapping[stix_type] pe_section.add_attribute( **{ 'type': mapping['type'], 'object_relation': mapping['relation'], 'value': value, 'to_ids': True }) except KeyError: if "x_misp_" in stix_type: attribute_type, relation = stix_type.split( "x_misp_")[1][:-1].split("_") attributes.append({ 'type': attribute_type, 'object_relation': relation, 'value': value, 'to_ids': True }) section_uuid = str(uuid.uuid4()) pe_section.uuid = pe_uuid pe.add_reference(section_uuid, 'included-in') self.misp_event.add_object(**pe_section) self.misp_event.add_object(**pe) return attributes, pe_uuid def buildExternalDict(self): self.fetch_report() for o in self.event: object_type = o._type if object_type in ('relationship', 'report'): continue if object_type in galaxy_types: self.parse_external_galaxy(o) elif object_type == 'vulnerability': attribute = {'type': 'vulnerability', 'value': o.get('name')} if 'description' in o: attribute['comment'] = o.get('description') self.misp_event.add_attribute(**attribute) elif object_type == 'course-of-action': self.parse_course_of_action(o) elif object_type == 'indicator': pattern = o.get('pattern') self.parse_external_pattern(pattern) attribute = { 'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': pattern } misp_object = { 'name': 'stix2-pattern', 'meta-category': 'stix2-pattern', 'Attribute': [self.version_attribute, attribute] } self.misp_event.add_object(**misp_object) def fetch_report(self): reports = [] for o in self.event: if o._type == 'report': reports.append(o) if len(reports) == 1: self.report = reports[0] self.parse_report() def parse_external_galaxy(self, o): galaxy = {'name': galaxy_types[o._type]} if 'kill_chain_phases' in o: galaxy['type'] = o['kill_chain_phases'][0].get('phase_name') cluster = defaultdict(dict) cluster['value'] = o.get('name') cluster['description'] = o.get('description') if 'aliases' in o: aliases = [] for a in o.get('aliases'): aliases.append(a) cluster['meta']['synonyms'] = aliases galaxy['GalaxyCluster'] = [cluster] self.misp_event['Galaxy'].append(galaxy) def parse_external_pattern(self, pattern): if ' OR ' in pattern and ' AND ' not in pattern: pattern = pattern.split('OR') for p in pattern: attribute = self.attribute_from_external_pattern(p) self.misp_event.add_attribute(**attribute) elif ' OR ' not in pattern and ' LIKE ' not in pattern: pattern = pattern.split('AND') if len(pattern) == 1: attribute = self.attribute_from_external_pattern(pattern[0]) self.misp_event.add_attribute(**attribute) @staticmethod def attribute_from_external_pattern(pattern): pattern_type, pattern_value = pattern.split(' = ') pattern_type, pattern_value = pattern_type[1:].strip( ), pattern_value[1:-2].strip() stix_type, value_type = pattern_type.split(':') if 'hashes' in value_type and 'x509' not in stix_type: h_type = value_type.split('.')[1] return {'type': h_type, 'value': pattern_value} else: # Might cause some issues, need more examples to test return { 'type': external_pattern_mapping[stix_type][value_type].get('type'), 'value': pattern_value } def set_distribution(self): for attribute in self.misp_event.attributes: attribute.distribution = self.__attribute_distribution for misp_object in self.misp_event.objects: misp_object.distribution = self.__attribute_distribution for attribute in misp_object.attributes: attribute.distribution = self.__attribute_distribution def saveFile(self): eventDict = self.misp_event.to_json() outputfile = '{}.stix2'.format(self.filename) with open(outputfile, 'w') as f: f.write(eventDict) @staticmethod def getTimestampfromDate(stix_date): try: return int(stix_date.timestamp()) except: return int( time.mktime( time.strptime( stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S"))) @staticmethod def get_misp_type(labels): return labels[0].split('=')[1][1:-1] @staticmethod def get_misp_category(labels): return labels[1].split('=')[1][1:-1] @staticmethod def parse_observable(observable, attribute_type): return misp_types_mapping[attribute_type](observable, attribute_type) @staticmethod def parse_pattern(pattern): if ' AND ' in pattern: pattern_parts = pattern.split(' AND ') if len(pattern_parts) == 3: _, value1 = pattern_parts[2].split(' = ') _, value2 = pattern_parts[0].split(' = ') return '{}|{}'.format(value1[1:-2], value2[1:-1]) else: _, value1 = pattern_parts[0].split(' = ') _, value2 = pattern_parts[1].split(' = ') if value1 in ("'ipv4-addr'", "'ipv6-addr'"): return value2[1:-2] return '{}|{}'.format(value1[1:-1], value2[1:-2]) else: return pattern.split(' = ')[1][1:-2] def parse_pattern_with_data(self, pattern): if 'artifact:payload_bin' not in pattern: return self.parse_pattern(pattern) pattern_parts = pattern.split(' AND ') if len(pattern_parts) == 3: filename = pattern_parts[0].split(' = ')[1] md5 = pattern_parts[1].split(' = ')[1] return "{}|{}".format( filename[1:-1], md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2] else: return pattern_parts[0].split( ' = ')[1][1:-1], pattern_parts[1].split(' = ')[1][1:-2]
class TestMISPEvent(unittest.TestCase): def setUp(self): self.maxDiff = None self.mispevent = MISPEvent() def init_event(self): self.mispevent.info = 'This is a test' self.mispevent.distribution = 1 self.mispevent.threat_level_id = 1 self.mispevent.analysis = 1 self.mispevent.set_date("2017-12-31") # test the set date method def test_simple(self): with open('tests/mispevent_testfiles/simple.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event(self): self.init_event() self.mispevent.publish() with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_loadfile(self): self.mispevent.load_file('tests/mispevent_testfiles/event.json') with open('tests/mispevent_testfiles/event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_tag(self): self.init_event() self.mispevent.add_tag('bar') self.mispevent.add_tag(name='baz') new_tag = MISPTag() new_tag.from_dict(name='foo') self.mispevent.add_tag(new_tag) with open('tests/mispevent_testfiles/event_tags.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_attribute(self): self.init_event() self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.add_attribute_tag('osint', 'bar.exe') attr_tags = self.mispevent.get_attribute_tag('bar.exe') self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint') self.assertEqual(attr_tags[0].name, 'osint') with open('tests/mispevent_testfiles/attribute.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) # Fake setting an attribute ID for testing self.mispevent.attributes[0].id = 42 self.mispevent.delete_attribute(42) with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_object_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah') self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename'])) self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1) self.mispevent.add_object(name='url', strict=True) self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' self.mispevent.objects[0].add_reference('b', 'baz', comment='foo') self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz') with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168") def test_object_level_tag(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar') self.mispevent.objects[0].add_tag('osint') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_malware(self): with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.init_event() self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile) attribute = self.mispevent.attributes[0] self.assertEqual(attribute.malware_binary, pseudofile) with open('tests/mispevent_testfiles/malware.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_malware(self): self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json') with open('tests/mispevent_testfiles/simple.json', 'rb') as f: pseudofile = BytesIO(f.read()) self.assertEqual( self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(), pseudofile.read()) def test_sighting(self): sighting = MISPSighting() sighting.from_dict(value='1', type='bar', timestamp=11111111) with open('tests/mispevent_testfiles/sighting.json', 'r') as f: ref_json = json.load(f) self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_existing_event(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') with open('tests/mispevent_testfiles/existing_event.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes_existing(self): self.mispevent.load_file('tests/mispevent_testfiles/shadow.json') with open('tests/mispevent_testfiles/shadow.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_shadow_attributes(self): self.init_event() self.mispevent.add_proposal(type='filename', value='baz.jpg') self.mispevent.add_attribute('filename', 'bar.exe') self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf') with open('tests/mispevent_testfiles/proposals.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_default_attributes(self): self.mispevent.add_object(name='file', strict=True) self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}]) self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0]) self.mispevent.objects[1].add_attribute('filename', value='baz') self.mispevent.objects[0].uuid = 'a' self.mispevent.objects[1].uuid = 'b' with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_default_values(self): self.init_event() self.mispevent.add_object(name='whois', strict=True) self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com') self.mispevent.objects[0].add_attribute('domain', value='domain.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com') self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis') self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/def_param.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_event_not_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) def test_event_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.info = 'blah' self.assertTrue(self.mispevent.edited) def test_event_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.add_tag('foo') self.assertTrue(self.mispevent.edited) def test_event_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.mispevent.attributes[0].value = 'blah' self.assertTrue(self.mispevent.attributes[0].edited) self.assertFalse(self.mispevent.attributes[1].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].tags[0].name = 'blah' self.assertTrue(self.mispevent.attributes[0].tags[0].edited) self.assertFalse(self.mispevent.attributes[0].tags[1].edited) self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_attribute_tag_edited_second(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.attributes[0].add_tag(name='blah') self.assertTrue(self.mispevent.attributes[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].edited) self.assertFalse(self.mispevent.objects[1].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].comment = 'blah' self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) def test_event_object_attribute_edited_tag(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') self.assertFalse(self.mispevent.edited) self.mispevent.objects[0].attributes[0].add_tag('blah') self.assertTrue(self.mispevent.objects[0].attributes[0].edited) self.assertTrue(self.mispevent.objects[0].edited) self.assertTrue(self.mispevent.edited) with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2)) def test_obj_by_id(self): self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json') misp_obj = self.mispevent.get_object_by_id(1556) self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f') def test_userdefined_object(self): self.init_event() self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles') with self.assertRaises(InvalidMISPObject) as e: # Fail on required self.mispevent.to_json() if sys.version_info >= (3, ): self.assertEqual(e.exception.message, '{\'member3\'} are required.') else: # Python2 bullshit self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.') self.mispevent.objects[0].add_attribute('member3', value='foo') with self.assertRaises(InvalidMISPObject) as e: # Fail on requiredOneOf self.mispevent.to_json() self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2') self.mispevent.objects[0].add_attribute('member1', value='bar') self.mispevent.objects[0].add_attribute('member1', value='baz') with self.assertRaises(InvalidMISPObject) as e: # member1 is not a multiple self.mispevent.to_json() self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed') self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2] self.mispevent.objects[0].uuid = 'a' with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f: ref_json = json.load(f) self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))