def processEvent(self, event): mevent = MISPEvent() mevent.from_dict(Event=event) changed = False for attr in mevent.attributes: if (attr["type"] == "ip-dst" or attr["type"] == "ip-src") and attr["to_ids"]: print("Removing IDS flag in event '{}' on attr '{}'".format( mevent.id, attr["value"])) changed = True attr["to_ids"] = False self.misp.update_attribute(attr) for obj in mevent.objects: for attr in obj.Attribute: if (attr["type"] == "ip-dst" or attr["type"] == "ip-src") and attr["to_ids"]: print( "Removing IDS flag in event '{}' on attr '{}'".format( mevent.id, attr["value"])) changed = True attr["to_ids"] = False self.misp.update_attribute(attr) self.misp.tag(mevent, self.expiredTag, True) if changed: self.misp.update_event(mevent.id, mevent) self.misp.publish(mevent)
def _get_event_from_id(self, event_uuid): with open(os.path.join(settings.outputdir, '%s.json' % event_uuid), 'r') as f: event_dict = json.load(f)['Event'] event = MISPEvent() event.from_dict(**event_dict) return event
def create_daily_event(self): new_uuid = gen_uuid() today = str(datetime.date.today()) event_dict = { 'uuid': new_uuid, 'id': len(self.manifest)+1, 'Tag': settings.Tag, 'info': self.daily_event_name.format(today), 'analysis': settings.analysis, # [0-2] 'threat_level_id': settings.threat_level_id, # [1-4] 'published': settings.published, 'date': today } event = MISPEvent() event.from_dict(**event_dict) # reference org org_dict = {} org_dict['name'] = settings.org_name org_dict['uui'] = settings.org_uuid event['Orgc'] = org_dict # save event on disk self.flush_event(new_event=event) # add event to manifest self.manifest[event['uuid']] = self._addEventToManifest(event) self.save_manifest() return event
def create_event(self): if self.args.threat is not None: # Dirty trick to keep consistency in the module: the threat level in the upload # API can go from 0 import to 3 but it is 1 to 4 in the event mgmt API. # It will be fixed in a near future, in the meantime, we do that: self.args.threat += 1 if not self.args.info: self.log('error', 'Info field is required for a new event') info = ' '.join(self.args.info) # Check if the following arguments have been set (and correctly set). If not, take the config values self.args.distrib = self.distribution if self.args.distrib is None else self.args.distrib self.args.sharing = self.sharinggroup if self.args.sharing is None else self.args.sharing if self.args.sharing and self.args.distrib != 4: self.args.sharing = None self.log('info', "Sharing group can only be set if distribution is 4. Clearing set value") misp_event = MISPEvent() misp_event.from_dict(info=info, distribution=self.args.distrib, sharing_group_id=self.args.sharing, threat_level_id=self.args.threat, analysis=self.args.analysis, date=self.args.date) self._search_local_hashes(misp_event) if self.offline_mode: # New event created locally, no ID __sessions__.current.misp_event.current_dump_file = self._dump() __sessions__.current.misp_event.offline() else: misp_event = self.misp.add_event(misp_event) if self._has_error_message(misp_event): return __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode)) self._dump()
def create_daily_event(self): new_uuid = gen_uuid() today = str(datetime.date.today()) event_dict = { 'uuid': new_uuid, 'id': len(self.manifest) + 1, 'Tag': self.tag, 'info': self.daily_event_name.format(today), 'analysis': self.confp['FEEDGENERATOR']['ANALYSIS'], # [0-2] 'threat_level_id': self.confp['FEEDGENERATOR']['THREAT_LEVEL_ID'], # [1-4] 'published': self.confp['FEEDGENERATOR']['PUBLISHED'], 'date': today } event = MISPEvent() event.from_dict(**event_dict) # reference org org_dict = {} org_dict['name'] = self.confp['FEEDGENERATOR']['ORG_NAME'] org_dict['uuid'] = self.confp['FEEDGENERATOR']['ORG_UUID'] event['Orgc'] = org_dict # save event on disk self.flush_event(new_event=event) # add event to manifest self.manifest[event['uuid']] = self._addEventToManifest(event) self.save_manifest() return event
def create_daily_event(self): today = str(datetime.date.today()) event_dict = { 'id': len(self.manifest) + 1, 'Tag': settings.Tag, 'info': self.daily_event_name.format(today), 'analysis': settings.analysis, # [0-2] 'threat_level_id': settings.threat_level_id, # [1-4] 'published': settings.published, 'date': today } event = MISPEvent() event.from_dict(**event_dict) # reference org org = MISPOrganisation() org.name = settings.org_name org.uuid = settings.org_uuid event.Orgc = org # save event on disk self.flush_event(new_event=event) # add event to manifest self.manifest.update(event.manifest) self.save_manifest() return event
def dropped_files(self, results, event, whitelist): if self.options.get("dropped", False) and "dropped" in results: for entry in results["dropped"]: if entry["md5"] and entry["md5"] not in whitelist: self.misper["iocs"].append({"md5": entry["md5"]}) self.misper["iocs"].append({"sha1": entry["sha1"]}) self.misper["iocs"].append({"sha256": entry["sha256"]}) """ Add all the dropped files as MISP attributes. """ from pymisp import MISPEvent # Upload all the dropped files at once filepaths = [r.get("path") for r in results.get("dropped", [])] if not filepaths: return try: self.misp.upload_samplelist( filepaths=filepaths, event_id=event["Event"]["id"], category="Artifacts dropped", comment="Dropped file", ) except: log.error( "Couldn't upload the dropped file, maybe " "the max upload size has been reached." ) return False # Load the event from MISP (we cannot use event as it # does not contain the sample uploaded above, nor it is # a MISPEvent but a simple dict) e = MISPEvent() e.from_dict(Event=self.misp.get_event(event["Event"]["id"])["Event"]) dropped_files = { f.get_attributes_by_relation("sha1")[0].value: f for f in e.objects if f["name"] == "file" } # Add further details on the dropped files for entry in results.get("dropped", []): # Find the corresponding object sha1 = entry.get("sha1") obj = dropped_files[sha1] # Add the real location of the dropped file (during the analysis) real_filepath = entry.get("guest_paths") obj.add_attribute("fullpath", real_filepath[0]) # Add Yara matches if any for match in entry.get("yara", []): desc = match["meta"]["description"] obj.add_attribute("text", value=desc, comment="Yara match") # Update the event self.misp.update_event(event_id=event["Event"]["id"], event=e)
def _get_event_from_id(self, event_uuid): with open( os.path.join(self.confp['FEEDGENERATOR']['OUTPUT'], '%s.json' % event_uuid), 'r') as f: event_dict = json.load(f) event = MISPEvent() event.from_dict(**event_dict) return event
def processEvent(self, event): mevent = MISPEvent() mevent.from_dict(Event=event) changed = False for attr in mevent.attributes: if (attr["type"] == "ip-dst" or attr["type"] == "ip-src") and attr["to_ids"]: print("Removing IDS flag in event '{}' on attr '{}'".format( mevent.id, attr["value"])) changed = True attr["to_ids"] = False if changed: mevent.add_tag(self.expiredTag) res = self.misp.update_event(mevent.id, mevent)
def register_misp(misp: ExpandedPyMISP, misp_event_dict: dict) -> None: """ MISPイベントデータを受け取り、MISPに登録する """ misp_event = MISPEvent() misp_event.from_dict(**misp_event_dict) threat_level_id = misp_event.get('threat_level_id') threat_level = f'anyrun:threat_level:{threat_level_id}' misp_event.add_tag('anyrun') misp_event.add_tag(threat_level) retry_count = 0 while True: try: # pymispをインスタンス化してイベント登録 event_data = misp.add_event(misp_event) if event_data.get('errors'): raise Exception(event_data['errors']) # MISPに登録されるイベントIDを取得し、出力 event_id = event_data['Event']['id'] print(f'新規に登録されたEvent_ID: {event_id}') return except: except_return = traceback.format_exc() # インポート済みの場合 if const.DUPLICATE_EVENT_CONFIRM_WORD in except_return: print('Importを行おうとしたイベントは既にMISPに登録されています') return # リトライ回数チェック retry_count += 1 if retry_count >= const.RETRY_MAXIMUM_LIMIT: raise # インターバル処理 print('MISPへのイベントインポートをリトライします') time.sleep(const.COMMAND_INTERVAL_TIME)
now = int(time.time()) for line in f: event_changed = 0 fields = line.split("|") ASN = fields.pop(0) ORG = fields.pop(0) CNPJ = fields.pop(0) EVENT_NAME = str(ORG + " " + CNPJ + " " + ASN) result_uuid = find_event( old_manifest, ASN) #search substring for ASN, this detects changed event names if result_uuid: # if event exists, get from feed event = MISPEvent() event.uuid = result_uuid event.from_dict(**loadEvent(result_uuid)) print("Checking for updated values in Event: " + event.uuid + " " + event.info) if event.info != EVENT_NAME: # information on event info is the same? print("Event name changed, updading... old:" + event.info + " new:" + EVENT_NAME) event.info = EVENT_NAME event_changed = 1 event.timestamp = now else: # event does not exist, generate a new one event = MISPEvent() event.info = EVENT_NAME event.analysis = 0 # events are created with Threat level Undefined and Analysis Initial event.threat_level_id = 4 event.published = 0 # events are created unpublished event.orgc = MISPOrganisation()
class FeedGenerator: """Helper object to create MISP feed. Configuration taken from the file settings.py""" def __init__(self, settings, tags): """This object can be use to easily create a MISP-feed of events. It handles the events creation, manifest file and cache file (hashes.csv). """ self.tag = tags self.confp = settings self.daily = bool(settings['FEEDGENERATOR']['DAILY']) self.sys_templates = get_system_templates() self.constructor_dict = dict(settings._sections['CONSTRUCTOR_DICT']) # TODO remove periodic flushing self.flushing_interval = int( settings['FEEDGENERATOR']['FLUSHING_INTERVAL']) self.flushing_next = time.time() + self.flushing_interval self.manifest = {} self.attributeHashes = [] self.daily_event_name = settings['FEEDGENERATOR'][ 'DAILY_EVENT_NAME'] + ' {}' self.event_name = settings['FEEDGENERATOR']['EVENT_NAME'] + ' {}' event_date_str, self.current_event_uuid, self.event_name = self.get_last_event_from_manifest( ) temp = [int(x) for x in event_date_str.split('-')] self.current_event_date = datetime.date(temp[0], temp[1], temp[2]) self.current_event = self._get_event_from_id(self.current_event_uuid) def add_sighting_on_attribute(self, sight_type, attr_uuid, **data): """Add a sighting on an attribute. Not supported for the moment.""" if self.daily: self.update_daily_event_id() self._after_addition() return False def add_attribute_to_event(self, attr_type, attr_value, **attr_data): """Add an attribute to the daily event""" if self.daily: self.update_daily_event_id() self.current_event.add_attribute(attr_type, attr_value, **attr_data) self._add_hash(attr_type, attr_value) self._after_addition() return True def add_object_to_event(self, obj_name, relationship=None, **data): """Add an object to the daily event""" if self.daily: self.update_daily_event_id() if obj_name not in self.sys_templates: print('Unkown object template') return False # Get MISP object constructor obj_constr = self.constructor_dict.get(obj_name, None) obj_constr = globals()[obj_constr] # Constructor not known, using the generic one if obj_constr is None: obj_constr = self.constructor_dict.get('generic') misp_object = globals()[obj_constr] # Fill generic object for k, v in data.items(): # attribute is not in the object template definition if k not in self.sys_templates[obj_name]['attributes']: # add it with type text misp_object.add_attribute(k, **{ 'value': v, 'type': 'text' }) else: misp_object.add_attribute(k, **{'value': v}) else: misp_object = obj_constr(data) if relationship is not None: misp_object.add_reference(relationship['uuid'], relationship['type']) self.current_event.add_object(misp_object) for attr_type, attr_value in data.items(): self._add_hash(attr_type, attr_value) if self.daily: self._after_addition() return misp_object.uuid def _after_addition(self): """Write event on disk""" now = time.time() if self.flushing_next <= now: self.flush_event() self.flushing_next = now + self.flushing_interval # Cache def _add_hash(self, attr_type, attr_value): if ('|' in attr_type or attr_type == 'malware-sample'): split = attr_value.split('|') self.attributeHashes.append([ hashlib.md5(str(split[0]).encode("utf-8")).hexdigest(), self.current_event_uuid ]) self.attributeHashes.append([ hashlib.md5(str(split[1]).encode("utf-8")).hexdigest(), self.current_event_uuid ]) else: self.attributeHashes.append([ hashlib.md5(str(attr_value).encode("utf-8")).hexdigest(), self.current_event_uuid ]) # Manifest def _init_manifest(self): # create an empty manifest with open( os.path.join(self.confp['FEEDGENERATOR']['OUTPUT'], 'manifest.json'), 'w'): pass # create new event and save manifest if self.daily: self.create_daily_event() else: self.create_event() def flush_event(self, new_event=None): print('Writting event on disk' + ' ' * 50) if new_event is not None: event_uuid = new_event['uuid'] event = new_event else: event_uuid = self.current_event_uuid event = self.current_event eventFile = open( os.path.join(self.confp['FEEDGENERATOR']['OUTPUT'], event_uuid + '.json'), 'w') eventSupport = "{{\"Event\": {}}}".format(event.to_json()) eventFile.write(eventSupport) eventFile.close() self.save_hashes() def save_manifest(self): try: manifestFile = open( os.path.join(self.confp['FEEDGENERATOR']['OUTPUT'], 'manifest.json'), 'w') manifestFile.write(json.dumps(self.manifest)) manifestFile.close() print('Manifest saved') except Exception as e: print(e) sys.exit('Could not create the manifest file.') def save_hashes(self): if len(self.attributeHashes) == 0: return False try: hashFile = open( os.path.join(self.confp['FEEDGENERATOR']['OUTPUT'], 'hashes.csv'), 'a') for element in self.attributeHashes: hashFile.write('{},{}\n'.format(element[0], element[1])) hashFile.close() self.attributeHashes = [] print('Hash saved' + ' ' * 30) except Exception as e: print(e) sys.exit('Could not create the quick hash lookup file.') def _addEventToManifest(self, event): event_dict = event.to_dict() tags = [] for eventTag in event_dict.get('EventTag', []): tags.append({ 'name': eventTag['Tag']['name'], 'colour': eventTag['Tag']['colour'] }) return { 'Orgc': event_dict.get('Orgc', []), 'Tag': tags, 'info': event_dict['info'], 'date': event_dict['date'], 'analysis': event_dict['analysis'], 'threat_level_id': event_dict['threat_level_id'], 'timestamp': event_dict.get('timestamp', int(time.time())) } def get_last_event_from_manifest(self): """Retreive last event from the manifest. If the manifest doesn't exists or if it is empty, initialize it. """ try: manifest_path = os.path.join(self.confp['FEEDGENERATOR']['OUTPUT'], 'manifest.json') with open(manifest_path, 'r') as f: man = json.load(f) dated_events = [] for event_uuid, event_json in man.items(): # add events to manifest self.manifest[event_uuid] = event_json dated_events.append( [event_json['date'], event_uuid, event_json['info']]) # Sort by date then by event name dated_events.sort(key=lambda k: (k[0], k[2]), reverse=True) return dated_events[0] except (FileNotFoundError, json.decoder.JSONDecodeError) as e: print('Manifest not found, generating a fresh one') self._init_manifest() return self.get_last_event_from_manifest() # DAILY def update_daily_event_id(self): if self.current_event_date != datetime.date.today( ): # create new event # save current event on disk self.flush_event() self.current_event = self.create_daily_event() self.current_event_date = datetime.date.today() self.current_event_uuid = self.current_event.get('uuid') self.event_name = self.current_event.info def _get_event_from_id(self, event_uuid): with open( os.path.join(self.confp['FEEDGENERATOR']['OUTPUT'], '%s.json' % event_uuid), 'r') as f: event_dict = json.load(f) event = MISPEvent() event.from_dict(**event_dict) return event def create_daily_event(self): new_uuid = gen_uuid() today = str(datetime.date.today()) event_dict = { 'uuid': new_uuid, 'id': len(self.manifest) + 1, 'Tag': self.tag, 'info': self.daily_event_name.format(today), 'analysis': self.confp['FEEDGENERATOR']['ANALYSIS'], # [0-2] 'threat_level_id': self.confp['FEEDGENERATOR']['THREAT_LEVEL_ID'], # [1-4] 'published': self.confp['FEEDGENERATOR']['PUBLISHED'], 'date': today } event = MISPEvent() event.from_dict(**event_dict) # reference org org_dict = {} org_dict['name'] = self.confp['FEEDGENERATOR']['ORG_NAME'] org_dict['uuid'] = self.confp['FEEDGENERATOR']['ORG_UUID'] event['Orgc'] = org_dict # save event on disk self.flush_event(new_event=event) # add event to manifest self.manifest[event['uuid']] = self._addEventToManifest(event) self.save_manifest() return event def create_event(self, h): new_uuid = gen_uuid() today = str(datetime.date.today()) event_dict = { 'uuid': new_uuid, 'id': len(self.manifest) + 1, 'Tag': self.tag, 'info': self.event_name.format(h), 'analysis': self.confp['FEEDGENERATOR']['ANALYSIS'], # [0-2] 'threat_level_id': self.confp['FEEDGENERATOR']['THREAT_LEVEL_ID'], # [1-4] 'published': self.confp['FEEDGENERATOR']['PUBLISHED'], 'date': today } self.current_event = MISPEvent() self.current_event.from_dict(**event_dict) # reference org org_dict = {} org_dict['name'] = self.confp['FEEDGENERATOR']['ORG_NAME'] org_dict['uuid'] = self.confp['FEEDGENERATOR']['ORG_UUID'] self.current_event['Orgc'] = org_dict # TODO return self.current_event def save_event(self): # save event on disk self.flush_event(new_event=self.current_event) # add event to manifest self.manifest[self.current_event['uuid']] = self._addEventToManifest( self.current_event) self.save_manifest()
def create_objects(vt_results, event_dict, comments, forced): event = MISPEvent() event.from_dict(**event_dict) vt_data = '' print("- Creating object(s)") if (vt_results['response_code'] == 1): # Add VT Object for obj_loop in vt_results['scans']: if (vt_results['scans'][obj_loop]['detected'] == True): vt_data += "%s (%s) Detection: %s\r\n" % ( obj_loop, vt_results['scans'][obj_loop]['version'], vt_results['scans'][obj_loop]['result']) else: vt_data += "%s (%s) Detection: No detection\r\n" % ( obj_loop, vt_results['scans'][obj_loop]['version']) detection = "%s/%s" % (vt_results['positives'], vt_results['total']) vt_comment = "File %s" % (vt_results['md5']) misp_object = event.add_object(name='virustotal-report', comment=vt_comment, distribution=5, standalone=False) obj_attr = misp_object.add_attribute('permalink', value=vt_results['permalink'], distribution=5) misp_object.add_attribute('detection-ratio', value=detection, distribution=5) if (args.verbose): misp_object.add_attribute('comment', value=vt_data, disable_correlation=True, distribution=5) misp_object.add_attribute('last-submission', value=vt_results['scan_date'], disable_correlation=True, distribution=5) vt_obj_uuid = misp_object.uuid print("\t* Permalink: " + vt_results['permalink']) print("\t* Detection: " + detection) print("\t* Last scan: " + vt_results['scan_date'] + "\r\n") # Add File Object misp_object = event.add_object(name='file', comment=comments, standalone=False) obj_attr = [] try: misp_object.add_attribute('md5', value=vt_results['md5'], distribution=5) except KeyError: vt_results['md5'] = None try: misp_object.add_attribute('sha1', value=vt_results['sha1'], distribution=5) except KeyError: vt_results['sha1'] = None try: misp_object.add_attribute('sha256', value=vt_results['sha256'], distribution=5) except KeyError: vt_results['sha256'] = None # Adding object to object relation if (vt_results['response_code'] == 1): misp_object.add_reference(referenced_uuid=vt_obj_uuid, relationship_type='analysed-with', comment='Expanded with virustotal data') if not (vt_results['md5'] == None): print("\t* MD5: " + vt_results['md5']) if not (vt_results['sha1'] == None): print("\t* SHA1: " + vt_results['sha1']) if not (vt_results['sha256'] == None): print("\t* SHA256: " + vt_results['sha256']) if (vt_results['response_code'] == 1): print("\t------------") print("\t* VirusTotal detections: ") vt_detects = vt_data.split('\n') for vt_detect in vt_detects: print("\t\t" + vt_detect) print("\t------------") try: # Submit the File and VT Objects to MISP if sys.version_info >= (3, 6, 0): misp.update_event(event) else: misp.update(event) except (KeyError, RuntimeError, TypeError, NameError): print("An error occoured when updating the event") sys.exit() print( "- The MISP objects seems to have been added correctly to the event.... \r\n\r\n" )
def _add_event(self, value): for tag in value['event_tags']: self._check_tag(tag) for attribute in value['attributes']: for tag in attribute['tags']: self._check_tag(tag) cnt = 0 while True: try: if self.misp == None: self._connect() tmp = MISPEvent() tmp.from_dict( distribution=self.misp_param['distribution'], threat_level_id=self.misp_param['threat_level_id'], analysis=self.misp_param['analysis'], info=value['title'], date=value['date'], published=False) response = self.misp.add_event(tmp) if response.get('errors'): raise Exception(str(response['errors'])) event = MISPEvent() event.load(response) break except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('add new event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return None self.debug_print(event.id) for tag in value['event_tags']: event.add_tag(tag) for attribute in value['attributes']: attribute_tags = [] event.add_attribute(type=attribute['type'], value=attribute['value'], category=attribute['category'], comment=attribute.get('comment', ''), distribution=self.misp_param['distribution'], Tag=self._create_tags(attribute['tags'])) event.published = True if self._update_event(event): self.debug_print('completed') return event else: self.debug_print('add failed') return None
def ProcessMISP( self, toParse, source: Union[Tuple[str], None] = None, url: Union[str, None] = None, ): try: event = MISPEvent() event.from_dict(**toParse) if source is None: source = [] if event.Orgc is not None: source.append(event.Orgc.name) description = event.info if (event.Tag is not None) and (len(event.Tag) > 0): tags = [ x.name for x in event.Tag if not x.name.startswith('osint:certainty') and not x.name. startswith('type:') and not x.name.startswith('source:') ] # TODO: 'slice' object is not subscriptable source.extend([ x.name[7:] for x in event.Tag if x.name.startswith('source:') ]) certaintyTags = [ x.name.replace('"', '') for x in event.Tag if x.name.startswith('osint:certainty') ] try: certainty = float(certaintyTags[0].split('=') [-1]) if len(certaintyTags) > 0 else None except ValueError as ve: certainty = None else: tags = [] certainty = None for attribute in event.attributes: # map event attribute to Zeek value(s) if (((not hasattr(attribute, 'deleted')) or (attribute.deleted == False)) and ((self.since is None) or (event.timestamp >= self.since) or (attribute.timestamp >= self.since)) and (vals := map_misp_attribute_to_zeek( attribute=attribute, source=source, url=url, description= f"{description}{'. '+attribute.comment if attribute.comment else ''}", tags=tags, confidence=certainty, logger=self.logger, ))): for val in vals: self.PrintHeader() with self.lock: # print the intelligence item fields according to the columns in 'fields' print('\t'.join([val[key] for key in self.fields]), file=self.outFile) except Exception as e: if self.logger is not None: self.logger.warning(e, exc_info=True)
for line in lines: if "BTC Address" in line: attrs = line.split(",") if lasthash != attrs[3]: lasthash = attrs[3] event_obj = MISPEvent() event_obj.distribution = 1 event_obj.threat_level_id = 3 event_obj.analysis = 1 event_obj.info = "RansomCoin Ransomware Survey " + attrs[3] event = misp.add_event(event_obj) event_id = event["Event"]["id"] print("Creating Event id: %s" % event_id) event_dict = misp.get(event_id)['Event'] event = MISPEvent() event.from_dict(**event_dict) event.add_attribute(type='md5', category='Artifacts dropped', value=attrs[1], disable_correlation=False, to_ids=False, proposal=False, distribution=5) event.add_attribute(type='sha1', category='Artifacts dropped', value=attrs[2], disable_correlation=False, to_ids=False, proposal=False, distribution=5) event.add_attribute(type='sha256',