def main(): parser = argparse.ArgumentParser( description="Test of adding event to MISP") parser.add_argument("mwdb_user", help="Mwdb username") parser.add_argument("mwdb_pass", help="Mwdb password") parser.add_argument("config", help="Config") parser.add_argument("misp_url", help="Misp url") parser.add_argument("misp_key", help="Misp key") args = parser.parse_args() mwdb = Malwarecage() mwdb.login(args.mwdb_user, args.mwdb_pass) try: cfg = mwdb.query_config(args.config) iocs = parse(cfg.family, cfg.cfg) except FamilyNotSupportedYetError: logging.info("Family %s not supported yet...", cfg.family) return if not iocs: # Nothing actionable found - skip the config return event = MISPEvent() event.add_tag(f"mwdb:family:{cfg.family}") event.info = f"Malware configuration ({cfg.family})" for o in iocs.to_misp(): event.add_object(o) misp = PyMISP(args.misp_url, args.misp_key, False) misp.add_event(event)
class MISP(): def __init__(self, config: Dict[str, Any]): self.logger = logging.getLogger(f'{self.__class__.__name__}') self.logger.setLevel(get_config('generic', 'loglevel')) if not config.get('apikey'): self.available = False self.logger.info('Module not enabled.') return self.available = True self.enable_lookup = False self.enable_push = False try: self.client = PyMISP(url=config['url'], key=config['apikey'], ssl=config['verify_tls_cert']) except Exception as e: self.available = False self.logger.warning(f'Unable to connect to MISP: {e}') return if config.get('enable_lookup'): self.enable_lookup = True if config.get('enable_push'): self.enable_push = True self.storage_dir_misp = get_homedir() / 'misp' self.storage_dir_misp.mkdir(parents=True, exist_ok=True) def push(self, event: MISPEvent) -> Union[MISPEvent, Dict]: if self.available and self.enable_push: return self.client.add_event(event, pythonify=True) else: return {'error': 'Module not available or push not enabled.'}
class MISPApi(object): def __init__(self, config: dict): self.pymisp = PyMISP(config['MISP_URL'], config['MISP_KEY'], config['MISP_VERIFYCERT'], 'json') def search(self, controller='attributes', **kwargs): return self.pymisp.search(controller, **kwargs) def org_name_id_mapping(self): pass def domain_name_lookup(self, domain_name: str) -> List[Any]: result = self.search(type='domain', value=domain_name) return result['response'].get('Attribute', []) def add_event(self, domain_names: list, info: str, tags: List, comment: str, to_ids: bool, ts: Optional[int] = None, published: Optional[bool] = False): attrs = [] for name in domain_names: attr = MISPAttribute() attr.from_dict(type='domain', category='Network activity', to_ids=to_ids, value=name, comment=comment, timestamp=ts) attrs.append(attr) event = MISPEvent() event.from_dict(info=info, Attribute=attrs, Tag=tags, date=datetime.date.today(), published=published) logger.debug(event) return self.pymisp.add_event(event)
def _misp_create_event_function(self, event, *args, **kwargs): """Function: create a MISP event from an incident """ try: def get_config_option(option_name, optional=False): """Given option_name, checks if it is in app.config. Raises ValueError if a mandatory option is missing""" option = self.options.get(option_name) if option is None and optional is False: err = "'{0}' is mandatory and is not set in ~/.resilient/app.config file. You must set this value to run this function".format( option_name) raise ValueError(err) else: return option API_KEY = get_config_option("misp_key") URL = get_config_option("misp_url") VERIFY_CERT = True if get_config_option( "verify_cert").lower() == "true" else False # Get the function parameters: misp_event_name = kwargs.get("misp_event_name") # text misp_distribution = kwargs.get("misp_distribution") # number misp_analysis_level = kwargs.get("misp_analysis_level") # number misp_threat_level = kwargs.get("misp_threat_level") # number log = logging.getLogger(__name__) log.info("misp_event_name: %s", misp_event_name) log.info("misp_distribution: %s", misp_distribution) log.info("misp_analysis_level: %s", misp_analysis_level) log.info("misp_threat_level: %s", misp_threat_level) yield StatusMessage("Setting up connection to MISP") misp_client = PyMISP(URL, API_KEY, VERIFY_CERT, 'json') eventJson = { "Event": { "info": misp_event_name, "analysis": misp_analysis_level, "distribution": misp_distribution, "threat_level_id": misp_threat_level } } event = misp_client.add_event(eventJson) log.info(event) yield StatusMessage("Event has been created") results = {"success": True, "content": event} # Produce a FunctionResult with the results yield FunctionResult(results) except Exception: yield FunctionError()
def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None): if event_info: event.info = event_info event.distribution = sanitize_event_distribution(distribution) event.threat_level_id = sanitize_event_threat_level_id(threat_level_id) event.analysis = sanitize_event_analysis(analysis) if publish: event.publish() # # TODO: handle multiple MISP instance misp = PyMISP(misp_url, misp_key, misp_verifycert) #print(event.to_json()) misp_event = misp.add_event(event, pythonify=True) # # TODO: handle error event_metadata = extract_event_metadata(misp_event) return event_metadata
def pushtoMISP(ioc_value): # Variable: MISP URL, Cert, Key misp_url = os.getenv("MISP_URL") misp_verifycert = False misp_key = os.getenv("MISP_KEY") # MISP connection misp = PyMISP(misp_url, misp_key, misp_verifycert, 'json') #date = "2018-02-28" i = datetime.now() date = i.strftime('%Y-%m-%d') # Set the event name Event_name = "ModSecurity - "+date+" - Honeypot" # Prepare MISP attribute misp_attributes = [] ioc_datetime = str(convertToEpoch(date)) comments = " " misp_attribute = { 'category': ioc_category, 'type': 'ip-dst', 'value': ioc_value, 'distribution': '0', 'to_ids': True, 'comment': comments , 'timestamp': ioc_datetime } misp_attributes.append(misp_attribute) # Prepare MISP event misp_event = { 'Event': { 'info': Event_name, 'date': date, 'distribution': '0', 'threat_level_id': '1', 'analysis': '2', 'Tag': [ {'name': 'ModSecurity'}, {'name': 'AUTO'} ], 'Attribute': misp_attributes } } # Insert Event + Attribute to MISP misp_event_added = misp.add_event(misp_event) # Publish (/!\ Bring in production) misp.publish(misp_event_added)
class TestBasic(unittest.TestCase): def setUp(self): self.maxDiff = None self.misp = PyMISP(url, key, True, 'json', True) def _clean_event(self, event): event['Event'].pop('orgc_id', None) event['Event'].pop('uuid', None) event['Event'].pop('sharing_group_id', None) event['Event'].pop('timestamp', None) event['Event'].pop('org_id', None) event['Event'].pop('date', None) event['Event'].pop('RelatedEvent', None) event['Event'].pop('publish_timestamp', None) if event['Event'].get('Attribute'): for a in event['Event'].get('Attribute'): a.pop('uuid', None) a.pop('event_id', None) a.pop('id', None) a.pop('timestamp', None) if event['Event'].get('Orgc'): event['Event']['Orgc'].pop('uuid', None) event['Event']['Orgc'].pop('id', None) if event['Event'].get('Org'): event['Event']['Org'].pop('uuid', None) event['Event']['Org'].pop('id', None) return event['Event'].pop('id', None) def new_event(self): event = self.misp.new_event(0, 1, 0, "This is a test") event_id = self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': None, 'disable_correlation': False, u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False, u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, u'Galaxy': [], u'threat_level_id': u'1'}} print(event) self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) def add_hashes(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, u'Galaxy': [], u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at adding hashes') def publish(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.publish(event) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, u'Galaxy': [], u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at publishing event') def delete(self, eventid): event = self.misp.delete_event(eventid) print(event) def delete_attr(self, attrid): event = self.misp.delete_attribute(attrid) print(event) def get(self, eventid): event = self.misp.get_event(eventid) print(event) def get_stix(self, **kwargs): event = self.misp.get_stix(kwargs) print(event) def add(self): event = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} event = self.misp.add_event(event) print(event) def test_create_event(self): eventid = self.new_event() time.sleep(1) self.delete(eventid) def test_get_event(self): eventid = self.new_event() time.sleep(1) self.get(eventid) time.sleep(1) self.delete(eventid) def test_add_event(self): self.add() time.sleep(1) self.delete(1) def test_del_attr(self): eventid = self.new_event() time.sleep(1) self.delete_attr(1) time.sleep(1) self.delete(eventid) def test_one_or_more(self): self.assertEqual(self.misp._one_or_more(1), (1,)) self.assertEqual(self.misp._one_or_more([1]), [1])
if province['deadCount']: obj_province.add_attribute('total-death', province['deadCount']) if province['comment']: obj_province.add_attribute('comment', province['comment']) for city in province['cities']: obj_city = event.add_object(name='covid19-dxy-live-city', standalone=False) obj_city.add_attribute('city', city['cityName']) obj_city.add_attribute('update', d) if city['currentConfirmedCount']: obj_city.add_attribute('current-confirmed', city['currentConfirmedCount']) if city['confirmedCount']: obj_city.add_attribute('total-confirmed', city['confirmedCount']) if city['curedCount']: obj_city.add_attribute('total-cured', city['curedCount']) if city['deadCount']: obj_city.add_attribute('total-death', city['deadCount']) obj_city.add_reference(obj_province, 'part-of') if make_feed: with (Path('output') / f'{event.uuid}.json').open('w') as _w: json.dump(event.to_feed(), _w) else: misp.add_event(event) if make_feed: feed_meta_generator(Path('output'))
with p.open() as f: reader = DictReader(f) for row in reader: if aggregate_by_country: country_aggregate(aggregate, row) else: obj = MISPObject(name='covid19-csse-daily-report') add_detailed_object(obj, row) event.add_object(obj) if aggregate_by_country: for country, values in aggregate.items(): obj = event.add_object(name='covid19-csse-daily-report', standalone=False) obj.add_attribute('country-region', country) obj.add_attribute('update', values['update']) obj.add_attribute('confirmed', values['confirmed']) obj.add_attribute('death', values['death']) obj.add_attribute('recovered', values['recovered']) obj.add_attribute('active', values['active']) if make_feed: with (Path('output') / f'{event.uuid}.json').open('w') as _w: json.dump(event.to_feed(), _w) else: event = misp.add_event(event) misp.publish(event) if make_feed: feed_meta_generator(Path('output'))
csvfile = open("Ransomware.csv") csvreader = csv.DictReader(csvfile) lines = csvfile.readlines() lasthash = '' for line in lines: if "BTC Address" in line: attrs = line.split(",") if lasthash != attrs[3]: lasthash = attrs[3] event_obj = MISPEvent() event_obj.distribution = 1 event_obj.threat_level_id = 3 event_obj.analysis = 1 event_obj.info = "RansomCoin Ransomware Survey " + attrs[3] event = misp.add_event(event_obj) event_id = event["Event"]["id"] print("Creating Event id: %s" % event_id) event_dict = misp.get(event_id)['Event'] event = MISPEvent() event.from_dict(**event_dict) event.add_attribute(type='md5', category='Artifacts dropped', value=attrs[1], disable_correlation=False, to_ids=False, proposal=False, distribution=5) event.add_attribute(type='sha1', category='Artifacts dropped', value=attrs[2],
if args.disable_new: event_id = response['response'][0]['id'] else: last_event_date = parse(response['response'][0]['date']).date() nb_attr = response['response'][0]['attribute_count'] if last_event_date < date.today() or int(nb_attr) > 1000: me = create_new_event() else: event_id = response['response'][0]['id'] else: me = create_new_event() parameters = {'banned-ip': args.banned_ip, 'attack-type': args.attack_type} if args.processing_timestamp: parameters['processing-timestamp'] = args.processing_timestamp if args.failures: parameters['failures'] = args.failures if args.sensor: parameters['sensor'] = args.sensor if args.victim: parameters['victim'] = args.victim if args.logline: parameters['logline'] = b64decode(args.logline).decode() f2b = Fail2BanObject(parameters=parameters, standalone=False) if me: me.add_object(f2b) pymisp.add_event(me) elif event_id: template_id = pymisp.get_object_template_id(f2b.template_uuid) a = pymisp.add_object(event_id, template_id, f2b)
class TestBasic(unittest.TestCase): def setUp(self): self.maxDiff = None self.misp = PyMISP(url, key, True, 'json') def _clean_event(self, event): event['Event'].pop('uuid', None) event['Event'].pop('timestamp', None) event['Event'].pop('date', None) event['Event'].pop('org', None) event['Event'].pop('orgc', None) event['Event'].pop('RelatedEvent', None) event['Event'].pop('publish_timestamp', None) if event['Event'].get('Attribute'): for a in event['Event'].get('Attribute'): a.pop('uuid', None) a.pop('event_id', None) a.pop('id', None) a.pop('timestamp', None) return event['Event'].pop('id', None) def new_event(self): event = self.misp.new_event(0, 1, 0, "This is a test") event_id = self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'0', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False, u'threat_level_id': u'1'}}, self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) def add_hashes(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at adding hashes') def publish(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.publish(event) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at publishing event') def delete(self, eventid): event = self.misp.delete_event(eventid) print event.json() def delete_attr(self, attrid): event = self.misp.delete_attribute(attrid) print event.json() def get(self, eventid): event = self.misp.get_event(eventid) print event.json() def add(self): event = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} event = self.misp.add_event(event) print event.json() def test_create_event(self): eventid = self.new_event() time.sleep(1) self.delete(eventid) def test_get_event(self): eventid = self.new_event() time.sleep(1) self.get(eventid) time.sleep(1) self.delete(eventid) def test_add_event(self): self.add() time.sleep(1) self.delete(1) def test_del_attr(self): eventid = self.new_event() time.sleep(1) self.delete_attr(1) time.sleep(1) self.delete(eventid)
class misp_custom: def __init__(self, misp_url, misp_key, misp_ssl): try: self.misp = PyMISP(misp_url, misp_key, misp_ssl) #, 'json') except Exception as err: sys.exit('Batch Job Terminated: MISP connection error - \n' + repr(err)) self.misp_logger = logging.getLogger('mispattruploader') def submit_to_misp(self, misp, misp_event, misp_objects): ''' Submit a list of MISP objects to a MISP event :misp: PyMISP API object for interfacing with MISP :misp_event: MISPEvent object :misp_objects: List of MISPObject objects. Must be a list ''' # go through round one and only add MISP objects a = [] for misp_object in misp_objects: self.misp_logger.debug(misp_object) if len(misp_object.attributes) > 0: if misp_object.name == 'network-connection': template_id = 'af16764b-f8e5-4603-9de1-de34d272f80b' else: # self.misp_logger.debug(dir(pymisp.api)) # self.misp_logger.debug(dir(self.misp)) # exit() self.misp_logger.debug(misp_object.template_uuid) object_template = self.misp.get_object_template( misp_object.template_uuid) template_id = object_template['ObjectTemplate']['id'] self.misp_logger.debug(template_id) self.misp_logger.debug(dir(misp_event)) self.misp_logger.debug(misp_event) _a = misp.add_object(event=misp_event, misp_object=misp_object) self.misp_logger.debug(_a) a.append(_a) # go through round two and add all the object references for each object b = [] for misp_object in misp_objects: for reference in misp_object.ObjectReference: _b = misp.add_object_reference(reference) b.append(_b) return a, b def check_object_length(self, misp_objects): for misp_object in misp_objects: self.misp_logger.info(misp_object.name) self.misp_logger.info(dir(misp_object)) if len(misp_object.attributes) == 0: self.misp_logger.error('failure to put in correct tags') return False return True def get_comm_and_tags(self, strInput): comment = None str_comment = "" tags = ["tlp:green"] tag_type = None for line in strInput.splitlines(): if ("comment:" in line.lower()): vals = line.split(":", 1) comment = vals[1:] elif ("tag:" in line.lower()): vals = line.split(":", 1) value = vals[1].strip().lower() if "tlp" in value: tags.remove("tlp:green") vals_str = "tlp:" vals_split = vals[1].split(":") vals_str += vals_split[1] tags.append(vals_str) elif ("type:" in line.lower()): vals = line.split(":", 1) value = vals[1].strip().lower() if value == "phish": tag_type = value elif value == "malware": tag_type = value elif value == "bec/spam": tag_type = value elif value == "dump": tag_type = value elif (value == "apt") or (value == "APT"): tag_type = value if tag_type: self.misp_logger.info('Setting tag to ir8: %s' % tag_type) tag = "ir8:" + tag_type tags.append(tag) else: tags = None if comment != None: for c in comment: str_comment += c else: str_comment = comment return str_comment, tags ####### SLACK ENTRY POINT! ###### def misp_send(self, strMISPEventID, strInput, strInfo, strUsername): # Establish communication with MISP # event = MISPEvent() # event.info = 'Test event' # event.analysis = 0 # event.distribution = 3 # event.threat_level_id = 2 # event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41') # #<add more attributes> # self.misp.add_event(event) # exit() try: objects = [] #get comments and tags from string input str_comment, tags = self.get_comm_and_tags(strInput) print(tags) if tags == None: self.misp_logger.info('Irate not in Tags: %s equals None' % tags) response = None return response #setup misp objects mispobj_email = MISPObject(name="email") mispobj_file = MISPObject(name="file") mispobj_files = {} mispobj_domainip = MISPObject(name="domain-ip") url_no = 0 file_no = 0 mispobj_urls = {} #process input for line in strInput.splitlines(): if ("domain:" in line.lower() ): #Catch domain and add to domain/IP object mispobj_domainip = MISPObject(name="domain-ip") vals = line.split(":", 1) mispobj_domainip.add_attribute("domain", value=vals[1].strip(), comment=str_comment) objects.append(mispobj_domainip) elif ("ip:" in line.lower()) or ("ip-dst:" in line.lower( )) or ("ip-src:" in line.lower()): #Catch IP and add to domain/IP object if "domain:" in strInput.splitlines(): mispobj_domainip = MISPObject(name="domain-ip") vals = line.split(":", 1) mispobj_domainip.add_attribute("ip", value=vals[1].strip(), comment=str_comment) objects.append(mispobj_domainip) else: mispobj_network_connection = MISPObject( name="network-connection") vals = line.split(":", 1) if ("ip:" in line.lower()) or ("ip-dst:" in line.lower()): mispobj_network_connection.add_attribute( "ip-dst", type="ip-dst", value=vals[1].strip(), comment=str_comment) else: mispobj_network_connection.add_attribute( "ip-src", type="ip-src", value=vals[1].strip(), comment=str_comment) objects.append(mispobj_network_connection) elif ("source-email:" in line.lower()) or ("email-source" in line.lower()) or ( "from:" in line.lower() ): #Catch email and add to email object vals = line.split(":", 1) mispobj_email.add_attribute("from", value=vals[1].strip(), comment=str_comment) elif ("url:" in line.lower()) or ( ('kit:' in line.lower() or ('creds:' in line.lower())) and (('hxxp' in line.lower()) or ('http' in line.lower())) ): #Catch URL and add to URL object vals = line.split(":", 1) url = vals[1].strip() url = refang(url) parsed = urlparse(url) mispobj_url = MISPObject(name="url") mispobj_url.add_attribute("url", value=parsed.geturl(), category="Payload delivery", comment=str_comment) if parsed.hostname: mispobj_url.add_attribute("host", value=parsed.hostname, comment=str_comment) if parsed.scheme: mispobj_url.add_attribute("scheme", value=parsed.scheme, comment=str_comment) if parsed.port: mispobj_url.add_attribute("port", value=parsed.port, comment=str_comment) mispobj_urls[url_no] = mispobj_url url_no += 1 #Catch different hashes and add to file object elif ("sha1:" in line.lower()) or ("SHA1:" in line): vals = line.split(":", 1) mispobj_file.add_attribute("sha1", value=vals[1].strip(), comment=str_comment) elif ("sha256:" in line.lower()) or ("SHA256:" in line): vals = line.split(":", 1) mispobj_file.add_attribute("sha256", value=vals[1].strip(), comment=str_comment) elif ("md5:" in line.lower()) or ("MD5:" in line): vals = line.split(":", 1) mispobj_file.add_attribute("md5", value=vals[1].strip(), comment=str_comment) elif ( "subject:" in line.lower() ): #or ("subject:" in line): #Catch subject and add to email object self.misp_logger.info('adding subject') vals = line.split(":", 1) mispobj_email.add_attribute("subject", value=vals[1].strip(), comment=str_comment) elif ("hash|filename:" in line.lower() ): #catch hash|filename pair and add to file object vals = line.split(":", 1) val = vals[1].split("|") l_hash = val[0] l_filename = val[1] l_mispobj_file = MISPObject(name="file") if len(re.findall(r"\b[a-fA-F\d]{32}\b", l_hash)) > 0: l_mispobj_file.add_attribute("md5", value=l_hash.strip(), comment=str_comment) l_mispobj_file.add_attribute("filename", value=l_filename.strip(), comment=str_comment) mispobj_files[file_no] = l_mispobj_file elif len(re.findall(r'\b[0-9a-f]{40}\b', l_hash)) > 0: l_mispobj_file.add_attribute("sha1", value=l_hash.strip(), comment=str_comment) l_mispobj_file.add_attribute("filename", value=l_filename.strip(), comment=str_comment) mispobj_files[file_no] = l_mispobj_file elif len(re.findall(r'\b[A-Fa-f0-9]{64}\b', l_hash)) > 0: l_mispobj_file.add_attribute("sha256", value=l_hash.strip(), comment=str_comment) l_mispobj_file.add_attribute("filename", value=l_filename.strip(), comment=str_comment) mispobj_files[file_no] = l_mispobj_file file_no += 1 #add all misp objects to List to be processed and submitted to MISP server as one. if len(mispobj_file.attributes) > 0: objects.append(mispobj_file) if len(mispobj_email.attributes) > 0: objects.append(mispobj_email) for u_key, u_value in mispobj_urls.items(): if len(u_value.attributes) > 0: objects.append(u_value) for f_key, f_value in mispobj_files.items(): if len(f_value.attributes) > 0: objects.append(f_value) # Update timestamp and event except Exception as e: error = traceback.format_exc() response = "Error occured when converting string to misp objects:\n %s" % error self.misp_logger.error(response) return response if self.check_object_length(objects) != True: self.misp_logger.error( 'Input from %s did not contain accepted tags.\n Input: \n%s' % (strUsername, strInput)) return "Error in the tags you entered. Please see the guide for accepted tags." try: # self.misp_logger.error(dir(self.misp)) misp_event = MISPEvent() misp_event.info = strInfo misp_event.distribution = 0 misp_event.analysis = 2 misp_event.threat_level_id = 3 # event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41') #event = self.misp.new_event(info=strInfo, distribution='0', analysis='2', threat_level_id='3', published=False) #misp_event = MISPEvent() #misp_event.load(event) add = self.misp.add_event(misp_event) self.misp_logger.info("Added event %s" % add) a, b = self.submit_to_misp(self.misp, misp_event, objects) for tag in tags: self.misp.tag(misp_event.uuid, tag) #self.misp.add_internal_comment(misp_event.id, reference="Author: " + strUsername, comment=str_comment) ccc = self.misp.publish(misp_event, alert=False) self.misp_logger.info(ccc) misp_event = self.misp.get_event(misp_event) response = misp_event #for response in misp_event: if ('errors' in response and response['errors'] != None): return ("Submission error: " + repr(response['errors'])) else: if response['Event']['RelatedEvent']: e_related = "" for each in response['Event']['RelatedEvent']: e_related = e_related + each['Event']['id'] + ", " return "Created ID: " + str( response['Event'] ['id']) + "\nRelated Events: " + ''.join(e_related) else: return "Created ID: " + str(response['Event']['id']) except Exception as e: error = traceback.format_exc() response = "Error occured when submitting to misp:\n %s" % error self.misp_logger.error(response) return response
class TestBasic(unittest.TestCase): def setUp(self): self.maxDiff = None self.misp = PyMISP(url, key, True, 'json') def _clean_event(self, event): event['Event'].pop('orgc_id', None) event['Event'].pop('uuid', None) event['Event'].pop('sharing_group_id', None) event['Event'].pop('timestamp', None) event['Event'].pop('org_id', None) event['Event'].pop('date', None) event['Event'].pop('RelatedEvent', None) event['Event'].pop('publish_timestamp', None) if event['Event'].get('Attribute'): for a in event['Event'].get('Attribute'): a.pop('uuid', None) a.pop('event_id', None) a.pop('id', None) a.pop('timestamp', None) if event['Event'].get('Orgc'): event['Event']['Orgc'].pop('uuid', None) event['Event']['Orgc'].pop('id', None) if event['Event'].get('Org'): event['Event']['Org'].pop('uuid', None) event['Event']['Org'].pop('id', None) return event['Event'].pop('id', None) def new_event(self): event = self.misp.new_event(0, 1, 0, "This is a test") event_id = self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'0', 'disable_correlation': False, u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Attribute': [], u'proposal_email_lock': False, u'Object': [], u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, u'Galaxy': [], u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) def add_hashes(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, u'Galaxy': [], u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at adding hashes') def publish(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.publish(event) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Org': {u'name': u'ORGNAME'}, u'Orgc': {u'name': u'ORGNAME'}, u'Galaxy': [], u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at publishing event') def delete(self, eventid): event = self.misp.delete_event(eventid) print(event) def delete_attr(self, attrid): event = self.misp.delete_attribute(attrid) print(event) def get(self, eventid): event = self.misp.get_event(eventid) print(event) def get_stix(self, **kwargs): event = self.misp.get_stix(kwargs) print(event) def add(self): event = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} event = self.misp.add_event(event) print(event) def add_user(self): email = '*****@*****.**' role_id = '5' org_id = '1' password = '******' external_auth_required = False external_auth_key = '' enable_password = False nids_sid = '1238717' server_id = '1' gpgkey = '' certif_public = '' autoalert = False contactalert = False disabled = False change_pw = '0' termsaccepted = False newsread = '0' authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' to_check = {'User': {'email': email, 'org_id': org_id, 'role_id': role_id, 'password': password, 'external_auth_required': external_auth_required, 'external_auth_key': external_auth_key, 'enable_password': enable_password, 'nids_sid': nids_sid, 'server_id': server_id, 'gpgkey': gpgkey, 'certif_public': certif_public, 'autoalert': autoalert, 'contactalert': contactalert, 'disabled': disabled, 'change_pw': change_pw, 'termsaccepted': termsaccepted, 'newsread': newsread, 'authkey': authkey}} user = self.misp.add_user(email=email, role_id=role_id, org_id=org_id, password=password, external_auth_required=external_auth_required, external_auth_key=external_auth_key, enable_password=enable_password, nids_sid=nids_sid, server_id=server_id, gpgkey=gpgkey, certif_public=certif_public, autoalert=autoalert, contactalert=contactalert, disabled=disabled, change_pw=change_pw, termsaccepted=termsaccepted, newsread=newsread, authkey=authkey) # delete user to allow reuse of test uid = user.get('User').get('id') self.misp.delete_user(uid) # ---------------------------------- # test interesting keys only (some keys are modified(password) and some keys are added (lastlogin) tested_keys = ['email', 'org_id', 'role_id', 'server_id', 'autoalert', 'authkey', 'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted', 'newsread', 'contactalert', 'disabled'] for k in tested_keys: self.assertEqual(user.get('User').get(k), to_check.get('User').get(k), "Failed to match input with output on key: {}".format(k)) def add_organisation(self): name = 'Organisation tests' description = 'This is a test organisation' orgtype = 'Type is a string' nationality = 'French' sector = 'Bank sector' uuid = '16fd2706-8baf-433b-82eb-8c7fada847da' contacts = 'Text field with no limitations' local = False to_check = {'Organisation': {'name': name, 'description': description, 'type': orgtype, 'nationality': nationality, 'sector': sector, 'uuid': uuid, 'contacts': contacts, 'local': local}} org = self.misp.add_organisation(name=name, description=description, type=orgtype, nationality=nationality, sector=sector, uuid=uuid, contacts=contacts, local=local, ) # delete organisation to allow reuse of test oid = org.get('Organisation').get('id') self.misp.delete_organisation(oid) # ---------------------------------- tested_keys = ['anonymise', 'contacts', 'description', 'local', 'name', 'nationality', 'sector', 'type', 'uuid'] for k in tested_keys: self.assertEqual(org.get('Organisation').get(k), to_check.get('Organisation').get(k), "Failed to match input with output on key: {}".format(k)) def test_create_event(self): eventid = self.new_event() time.sleep(1) self.delete(eventid) def test_get_event(self): eventid = self.new_event() time.sleep(1) self.get(eventid) time.sleep(1) self.delete(eventid) def test_add_event(self): self.add() time.sleep(1) self.delete(1) def test_del_attr(self): eventid = self.new_event() time.sleep(1) self.delete_attr(1) time.sleep(1) self.delete(eventid) def test_one_or_more(self): self.assertEqual(self.misp._one_or_more(1), (1,)) self.assertEqual(self.misp._one_or_more([1]), [1]) def test_create_user(self): self.add_user() def test_create_organisation(self): self.add_organisation()
def export(request): """Export extracteddump to misp""" data = {} if request.method == "POST": extracted_dump = get_object_or_404( ExtractedDump, pk=request.POST.get("selected_exdump")) misp_info = get_object_or_404(Service, name=2) # CREATE GENERIC EVENT misp = PyMISP(misp_info.url, misp_info.key, False, proxies=misp_info.proxy) event = MISPEvent() event.info = "From orochi: {}@{}".format( extracted_dump.result.plugin.name, extracted_dump.result.dump.name) # CREATE FILE OBJ file_obj = FileObject(extracted_dump.path) event.add_object(file_obj) # ADD CLAMAV SIGNATURE if extracted_dump.clamav: clamav_obj = MISPObject("av-signature") clamav_obj.add_attribute("signature", value=extracted_dump.clamav) clamav_obj.add_attribute("software", value="clamav") file_obj.add_reference(clamav_obj.uuid, "attributed-to") event.add_object(clamav_obj) # ADD VT SIGNATURE if extracted_dump.vt_report: vt_obj = MISPObject("virustotal-report") vt_obj.add_attribute("last-submission", value=extracted_dump.vt_report.get( "scan_date", "")) vt_obj.add_attribute( "detection-ratio", value="{}/{}".format( extracted_dump.vt_report.get("positives", 0), extracted_dump.vt_report.get("total", 0), ), ) vt_obj.add_attribute("permalink", value=extracted_dump.vt_report.get( "permalink", "")) file_obj.add_reference(vt_obj.uuid, "attributed-to") event.add_object(vt_obj) misp.add_event(event) return JsonResponse({"success": True}) extracted_dump = get_object_or_404(ExtractedDump, path=urllib.parse.unquote( request.GET.get("path"))) form = MispExportForm( instance=extracted_dump, initial={ "selected_exdump": extracted_dump.pk, "selected_index_name": extracted_dump.result.dump.name, "selected_plugin_name": extracted_dump.result.plugin.name, }, ) context = {"form": form} data["html_form"] = render_to_string( "website/partial_export.html", context, request=request, ) return JsonResponse(data)
for sample in it: raw_report = getFileDetail(apikey=api_key, resource=sample.id) scandate = datetime.datetime.strptime(raw_report["scan_date"], '%Y-%m-%d %H:%M:%S') me = MISPEvent() this_event_name = misp_event_name.format(scandate.strftime("%Y-%m-%d")) search = pm.search(controller='events', eventinfo=this_event_name) if (len(search) == 1): me.load(search[0]) else: me.info = this_event_name pm.add_event(me) vtreport = GenericObjectGenerator('virustotal-report') vtreport.add_attribute("last-submission", value=raw_report["scan_date"]) vtreport.add_attribute("permalink", value=raw_report["permalink"]) ratio = "{}/{}".format(raw_report["positives"], raw_report["total"]) vtreport.add_attribute("detection-ratio", value=ratio) file_object = GenericObjectGenerator('file') file_object.add_attribute("md5", value=raw_report["md5"]) file_object.add_attribute("sha1", value=raw_report["sha1"]) file_object.add_attribute("sha256", value=raw_report["sha256"]) file_object.add_attribute("ssdeep", value=raw_report["ssdeep"]) file_object.add_attribute("authentihash", value=raw_report["authentihash"])
# # Load objects from the CSV file objects = get_object_fields(args.csv, args.delim, args.quotechar, args.strictcsv) if len(objects) == 0: log.critical('No Objects to create! Are they commented out? Run with --verbose (-v) to see what\'s happening!') exit(1) # # Create a new Event if args.info: event = MISPEvent() event.info = args.info if args.distribution: log.debug('Setting distribution level for Event: {}'.format(args.distribution)) event.distribution = args.distribution if not args.dryrun: new_event = pymisp.add_event(event) if 'errors' in new_event.keys(): log.critical('Error creating the new event. {}'.format(new_event['errors'][2])) exit(1) # # Get the ID of the new event for later args.event = new_event['Event']['uuid'] log.info('New event created: {}'.format(args.event)) # # Add Objects to existing Event for i, o in enumerate(objects, 1): misp_object = GenericObjectGenerator(o['object'], misp_objects_path_custom=args.custom_objects_path) try: misp_object.generate_attributes(o['attributes']) except NewAttributeError as e:
class MISPHandler: def __init__(self, config: dict): self.url = config['misp_url'] self.key = config['misp_auth_key'] self.misp = PyMISP(self.url, self.key) self.tag_list = self.create_tag_list() self.logger = logging.getLogger('misp_handler') self.logger.debug("URLhausHandler init done") def create_tag_list(self) -> list: tags = [] for item in self.misp.tags(pythonify=True): tags.append(item.name) return tags def make_sure_tag_exists(self, tag: str) -> bool: if tag in self.tag_list: return True else: self.misp.add_tag({"name": tag}, pythonify=True) self.tag_list = self.create_tag_list() if tag in self.tag_list: return True else: return False def add_tag_to_attribute(self, attr: MISPAttribute, tag: str) -> MISPAttribute: if self.make_sure_tag_exists(tag): attr.add_tag(tag) return attr def create_attr(self, raw_attr: dict) -> MISPAttribute: # Create attribute and assign simple values attr = MISPAttribute() attr.type = 'url' attr.value = raw_attr['url'] attr.disable_correlation = False attr.__setattr__('first_seen', datetime.strptime(raw_attr['dateadded'], '%Y-%m-%d %H:%M:%S')) # Add URLhaus tag self.add_tag_to_attribute(attr, 'URLhaus') # Add other tags if raw_attr['tags']: for tag in raw_attr['tags'].split(','): self.add_tag_to_attribute(attr, tag.strip()) # Add online/offline tag if not pandas.isna(raw_attr['url_status']): if raw_attr['url_status'] == 'online': attr.to_ids = True else: attr.to_ids = False self.add_tag_to_attribute(attr, raw_attr['url_status']) # Add reporter tag if not pandas.isna(raw_attr['reporter']): self.add_tag_to_attribute(attr, raw_attr['reporter']) attr.comment = raw_attr['urlhaus_link'] return attr def create_attr_feodo(self, raw_attr: dict) -> MISPAttribute: attr = MISPAttribute() attr.type = 'ip-dst|port' attr.value = f"{raw_attr['DstIP']}|{raw_attr['DstPort']}" self.add_tag_to_attribute(attr, 'FeodoTracker') self.add_tag_to_attribute(attr, raw_attr['Malware']) attr.comment = 'Feodo tracker DST IP/port' attr.__setattr__('first_seen', datetime.strptime(raw_attr['Firstseen'], '%Y-%m-%d %H:%M:%S')) if not pandas.isna(raw_attr['LastOnline']): last_seen_time = datetime.strptime(str(raw_attr['LastOnline']), '%Y-%m-%d').replace(tzinfo=pytz.utc) first_seen_time = datetime.strptime(str(raw_attr["Firstseen"]), '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.utc) if first_seen_time > last_seen_time: last_seen_time = first_seen_time + timedelta(seconds=1) attr.__setattr__('last_seen', last_seen_time) else: last_seen_time = datetime.strptime(str(raw_attr['Firstseen']), '%Y-%m-%d %H:%M:%S').replace(tzinfo=pytz.utc) attr.__setattr__('last_seen', last_seen_time) attr.to_ids = False attr.disable_correlation = False return attr def create_attr_azorult(self, raw_attr: dict) -> MISPAttribute: attr_list = [] for type in [{'json': 'domain', 'misp': 'domain'}, {'json': 'ip', 'misp': 'ip-dst'}, {'json': 'panel_index', 'misp': 'url'}]: if type['json'] in raw_attr: attr = MISPAttribute() self.add_tag_to_attribute(attr, 'AzorultTracker') self.add_tag_to_attribute(attr, raw_attr['panel_version']) self.add_tag_to_attribute(attr, raw_attr['feeder']) self.add_tag_to_attribute(attr, raw_attr['status']) attr.comment = f'Azorult panel {type["misp"]}' attr.__setattr__('first_seen', datetime.fromtimestamp(raw_attr['first_seen'])) attr.to_ids = False attr.disable_correlation = False attr.type = type['misp'] attr.value = f"{raw_attr[type['json']]}" attr_list.append(attr) return attr_list @staticmethod def get_attribute_tag_list(self, tag_list: list) -> list: tags = [] for item in tag_list: tags.append(item['name']) return tags @staticmethod def create_event(title: str, date_added: datetime) -> MISPEvent: misp_event = MISPEvent() misp_event.info = title if date_added != '': misp_event.date = date_added return misp_event def get_event(self, event_id): return self.misp.get_event(event_id, pythonify=True) def add_attr_to_event(self, event: MISPEvent, attribute: MISPAttribute): event.attributes.append(attribute) return event def update_event(self, event: MISPEvent): return self.misp.update_event(event, pythonify=True) def get_day_event(self, day: str, source: str, date: str): if source in ['URLHaus', 'FeodoTracker']: misp_event = self.misp.search('events', 'json', org=1, eventinfo=f'{source} import day {day}', pythonify=True) elif source in ['AzorultTracker']: misp_event = self.misp.search('events', 'json', org=1, eventinfo=f'{source} import panel {day}', pythonify=True) if len(misp_event) >= 1: return misp_event[0] else: if source in ['URLHaus', 'FeodoTracker']: misp_event = self.create_event(f"{source} import day {day}", date_added=datetime.timestamp( datetime.strptime(date, '%Y-%m-%d %H:%M:%S'))) elif source in ['AzorultTracker']: misp_event = self.create_event(f"{source} import panel {day}", date_added=datetime.fromtimestamp(date)) event_id = self.misp.add_event(misp_event) self.misp.publish(event_id) return self.get_event(event_id) @staticmethod def delete_attribute_by_value(search_value: str, event: MISPEvent): found = False for a in event.attributes: if (hasattr(a, 'value') and a.value == search_value): a.deleted = True return event
class MISP(Report): """MISP Analyzer.""" order = 1 def malpedia(self, results, event, malfamily): if malfamily in name_update_shema: malfamily = name_update_shema[malfamily] if malfamily in malpedia_json: self.misp.tag(event["uuid"], 'misp-galaxy:malpedia="{}"'.format(malfamily)) def signature(self, results, event): for ttp in results.get("ttps", []) or []: for i in ttps_json.get("objects", []) or []: try: if i["external_references"][0]["external_id"] == ttp: self.misp.tag(event, f'misp-galaxy:mitre-attack-pattern="{i["name"]}-{ttp}"') except Exception: pass def sample_hashes(self, results, event): if results.get("target", {}).get("file", {}): f = results["target"]["file"] misp_object = MISPObject("file") misp_object.comment = "File submitted to CAPEv2" misp_object.add_attribute("filename", value=f["name"], category="Payload delivery") misp_object.add_attribute("md5", value=f["md5"], category="Payload delivery") misp_object.add_attribute("sha1", value=f["sha1"], category="Payload delivery") misp_object.add_attribute("sha256", value=f["sha256"], category="Payload delivery") misp_object.add_attribute("ssdeep", value=f["ssdeep"], category="Payload delivery") self.misp.add_object(event, misp_object) def all_network(self, results, event): """All of the accessed URLS as per the PCAP.""" urls = set() if self.options.get("network", False) and "network" in results.keys(): urls = set() for req in results["network"].get("http", []): if "uri" in req and req["uri"] not in whitelist: urls.add(req["uri"]) if "user-agent" in req: event.add_attribute("user-agent", req["user-agent"]) domains, ips = {}, set() for domain in results.get("network", {}).get("domains", []): if domain["domain"] not in whitelist and domain["ip"] not in whitelist: domains[domain["domain"]] = domain["ip"] ips.add(domain["ip"]) for block in results.get("network", {}).get("hosts", []): if block["ip"] not in whitelist: ips.add(block["ip"]) for block in results["network"].get("dns", []): # Added DNS if block.get("request", "") and (block["request"] not in whitelist): if block["request"] not in domains and block["request"] not in whitelist: if block["answers"]: domains[block["request"]] = block["answers"][0]["data"] ips.add(domain[block["answers"][0]["data"]]) # Added CAPE Addresses for section in results.get("CAPE", []) or []: try: if section.get("cape_config", {}).get("address", []) or []: for ip in section["cape_config"]["address"]: if ip not in ips: ips.add(ip.split(":")[0]) except Exception as e: print(e) for url in sorted(list(urls)): event.add_attribute("url", url) for ip in sorted(list(ips)): event.add_attribute("ip-dst", ip) for domain, ips in domains.items(): obj = MISPObject("domain-ip") obj.add_attribute("domain", domain) for ip in ips: obj.add_attribute("ip", ip) event.add_object(obj) self.misp.update_event(event) def dropped_files(self, results, event): """ if self.options.get("dropped", False) and "dropped" in results: for entry in results["dropped"]: if entry["md5"] and entry["md5"] not in whitelist: self.misper["iocs"].append({"md5": entry["md5"]}) self.misper["iocs"].append({"sha1": entry["sha1"]}) self.misper["iocs"].append({"sha256": entry["sha256"]}) """ """ Add all the dropped files as MISP attributes. """ # Upload all the dropped files at once # TODO: Use expanded for r in results.get("dropped", []) or []: with open(r.get("path"), "rb") as f: event.add_attribute("malware-sample", value=os.path.basename(r.get("path")), data=BytesIO(f.read()), expand="binary") event.run_expansions() self.misp.update_event(event) """ # Load the event from MISP (we cannot use event as it # does not contain the sample uploaded above, nor it is # a MISPEvent but a simple dict) e = MISPEvent() e.from_dict(Event=self.misp.get_event(event["Event"]["id"])["Event"]) dropped_files = { f.get_attributes_by_relation("sha1")[0].value: f for f in e.objects if f["name"] == "file" } # Add further details on the dropped files for entry in results.get("dropped", []): # Find the corresponding object sha1 = entry.get("sha1") obj = dropped_files[sha1] # Add the real location of the dropped file (during the analysis) real_filepath = entry.get("guest_paths") obj.add_attribute("fullpath", real_filepath[0]) # Add Yara matches if any for match in entry.get("yara", []): desc = match["meta"]["description"] obj.add_attribute("text", value=desc, comment="Yara match") # Update the event self.misp.update_event(event_id=event["Event"]["id"], event=e) """ def run(self, results): """Run analysis. @return: MISP results dict. """ url = self.options.get("url", "") apikey = self.options.get("apikey", "") if not url or not apikey: log.error("MISP URL or API key not configured.") return self.misp = PyMISP(url, apikey, False, "json") self.threads = self.options.get("threads", "") if not self.threads: self.threads = 5 self.iocs = deque() self.misper = dict() try: if self.options.get("upload_iocs", False) and results.get("malscore", 0) >= self.options.get("min_malscore", 0): distribution = int(self.options.get("distribution", 0)) threat_level_id = int(self.options.get("threat_level_id", 4)) analysis = int(self.options.get("analysis", 0)) tag = self.options.get("tag") or "CAPEv2" info = self.options.get("title", "") upload_sample = self.options.get("upload_sample") malfamily = "" if results.get("detections", ""): malfamily = results["detections"] response = self.misp.search("attributes", value=results["target"]["file"]["sha256"], return_format="json", pythonify=True) if response: event = self.misp.get_event(response[0].event_id, pythonify=True) else: event = MISPEvent() event.distribution = distribution event.threat_level_id = threat_level_id event.analysis = analysis event.info = "{} {} - {}".format(info, malfamily, results.get("info", {}).get("id")) event = self.misp.add_event(event, pythonify=True) # Add a specific tag to flag Cuckoo's event if tag: self.misp.tag(event, tag) # malpedia galaxy if malpedia_json: self.malpedia(results, event, malfamily) # ToDo? self.signature(results, event) self.sample_hashes(results, event) self.all_network(results, event) self.dropped_files(results, event) if upload_sample: target = results.get("target", {}) f = target.get("file", {}) if target.get("category") == "file" and f: with open(f["path"], "rb") as f: event.add_attribute( "malware-sample", value=os.path.basename(f["path"]), data=BytesIO(f.read()), expand="binary", comment="Sample run", ) if results.get("target", {}).get("url", "") and results["target"]["url"] not in whitelist: event.add_attribute("url", results["target"]["url"]) # ToDo migth be outdated! # if self.options.get("ids_files", False) and "suricata" in results.keys(): # for surifile in results["suricata"]["files"]: # if "file_info" in surifile.keys(): # self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]}) # self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]}) # self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]}) if self.options.get("mutexes", False) and "behavior" in results and "summary" in results["behavior"]: if "mutexes" in results.get("behavior", {}).get("summary", {}): for mutex in results["behavior"]["summary"]["mutexes"]: if mutex not in whitelist: event.add_attribute("mutex", mutex) if self.options.get("registry", False) and "behavior" in results and "summary" in results["behavior"]: if "read_keys" in results["behavior"].get("summary", {}): for regkey in results["behavior"]["summary"]["read_keys"]: event.add_attribute("regkey", regkey) event.run_expansions() self.misp.update_event(event) # Make event public if self.options.get("published", True): self.misp.publish(event) except Exception as e: log.error("Failed to generate JSON report: %s" % e, exc_info=True)
nb_attr = response['response'][0]['attribute_count'] if last_event_date < date.today() or int(nb_attr) > 1000: me = create_new_event() else: event_id = response['response'][0]['id'] else: me = create_new_event() parameters = {'banned-ip': args.banned_ip, 'attack-type': args.attack_type} if args.processing_timestamp: parameters['processing-timestamp'] = args.processing_timestamp if args.failures: parameters['failures'] = args.failures if args.sensor: parameters['sensor'] = args.sensor if args.victim: parameters['victim'] = args.victim if args.logline: parameters['logline'] = b64decode(args.logline).decode() if args.logfile: with open(args.logfile, 'rb') as f: parameters['logfile'] = {'value': os.path.basename(args.logfile), 'data': BytesIO(f.read())} f2b = Fail2BanObject(parameters=parameters, standalone=False) if me: me.add_object(f2b) pymisp.add_event(me) elif event_id: template_id = pymisp.get_object_template_id(f2b.template_uuid) a = pymisp.add_object(event_id, template_id, f2b)
misp_event=MISPEvent() search=pm.search(controller='events', eventinfo=misp_event_name) if ( len(search) == 1): misp_event.load(search[0]) #load 'samples' dictionary from misp_event for obj in misp_event.get('Object'): if (obj.name == "file"): existing_hash = obj.get_attributes_by_relation("sha256")[0]['value'] samples.update({existing_hash : obj}) #reload 'attributes' dictionary from misp_event for attr in misp_event.get('Attribute'): attributes.update({attr.value : attr}) else: misp_event.info=misp_event_name pm.add_event(misp_event) #bazaar setup r = requests.post(bazaar_url, data=bazaar_query) bazaardata = json.loads(r.text) # add File objects/samples that don't currently exist for sample in bazaardata.get('data'): hash_id=sample["sha256_hash"] if (hash_id not in samples): addSampleByHash(hash_id, misp_event) pm.update_event(misp_event)
class MISP(): """ Handle manipulation of event into MISP TODO: - use MISPEvent for creating MISP events check https://www.circl.lu/assets/files/misp-training/luxembourg2017/4.1-pymisp.pdf - use PyTaxonomies for Tags """ api = "" url = "" misp = "" proxies = None sslCheck = False # Not recommended debug = False # Enable debug mode score = { # Map Facebook Threat Exchange "status" to a score to determine if event will be made in MISP 0 : "NON MALICIOUS", 1 : "UNKNOWN", 2 : "SUSPICIOUS", 3 : "MALICIOUS" } badness_threshold = 1 # Minimum score in score table to create a MISP event published = False # Default state of MISP created object # ThreatExchange type -> MISP # see IndicatorType object # https://developers.facebook.com/docs/threat-exchange/reference/apis/indicator-type/v2.8 type_map = {} # Map of Facebook Threat Exchange to MISP types share_levels = {} # Map of sharing levels (TLP -> TLP) extra_tag = None # Extra tag to add to all imported events (! no check of consistency !) privacy_levels = {} # Map the privacy_type of Facebook Threat Exchange to Sharing Group ID of MISP # ----------------------------------------------------------------------- # def __init__(self, url, api, proxies=None, feed=False): self.url = url self.api = api self.proxies = proxies if not feed: self.misp = PyMISP(self.url, self.api, ssl=self.sslCheck, out_type='json', debug=self.debug, proxies=self.proxies, cert=None) else: self.misp = None return def convertTEtoMISP(self, teevent): """ Convert a ThreatExchange entry to MISP entry """ # Create empty event mispevt = MISPEvent() mispevt.info = "[Facebook ThreatExchange]" mispevt.distribution = 0 mispevt.sharing_group_id = self.privacy_levels[teevent["privacy_type"]] # Check if event is to be kept if "status" in teevent.keys() and teevent["status"] in self.score.keys() and self.score[teevent["status"]] < self.badness_threshold : print("IGNORE EVENT %s due to status (%s)" % (teevent, teevent["status"])) return None # Add indicator to event if "raw_indicator" in teevent.keys(): if "type" in teevent.keys(): if teevent["type"] in self.type_map.keys(): indicator = teevent["raw_indicator"].replace("\\", "") mispevt.add_attribute(self.type_map[teevent["type"]] , indicator) # not to brutal?? else: print("WARNING: TYPE %s SHOULD BE ADDED TO MAPPING" % teevent["type"]) else: print("WARNING, event %s does not contains any indicator :(" % teevent) return None # don't create event without content! # Add a category mispevt.category = "Network activity" # Enrich description if "description" in teevent.keys(): mispevt.info = mispevt.info + " - %s" % teevent["description"] if "owner" in teevent.keys() and "name" in teevent["owner"].keys(): owner = teevent["owner"]["name"] if("email" in teevent["owner"].keys()): email = teevent["owner"]["email"].replace("\\u0040", "@") else: email = "" mispevt.info = mispevt.info + " - by %s (%s)" % (owner, email) # Add sharing indicators (tags) if "share_level" in teevent.keys(): if teevent["share_level"] in self.share_levels.keys(): mispevt.Tag.append(self.share_levels[teevent["share_level"]]) else: print("WARNING: SHARING LEVEL %s SHOULD BE ADDED TO MAPPING" % teevent["share_level"]) if self.extra_tag is not None: mispevt.Tag.append(self.extra_tag) evtid = teevent["id"] return [evtid, mispevt] def convertTEtoMISPTEST(self, teevents=[]): """ Convert a ThreatExchange entry to MISP entry """ # Create empty event mispevt = MISPEvent() mispevt.info = "[Facebook ThreatExchange]" mispevt.distribution = 0 mispevt.category = "Network activity" share_level = "WHITE" evtids = [] for teevent in teevents: # Set event visiblity to VISIBLE except if stated otherwise in event if(self.privacy_levels[teevent["privacy_type"]] != self.privacy_levels["VISIBLE"]): mispevt.sharing_group_id = self.privacy_levels[teevent["privacy_type"]] else: mispevt.sharing_group_id = self.privacy_levels["VISIBLE"] # Check if event is to be kept if "status" in teevent.keys() and teevent["status"] in self.score.keys() and self.score[teevent["status"]] < self.badness_threshold : print("IGNORE EVENT %s due to status (%s)" % (teevent, teevent["status"])) continue # Add indicator to event if "raw_indicator" in teevent.keys(): if "type" in teevent.keys(): if teevent["type"] in self.type_map.keys(): indicator = teevent["raw_indicator"].replace("\\", "") mispevt.add_attribute(self.type_map[teevent["type"]] , indicator) # not to brutal?? else: print("WARNING: TYPE %s SHOULD BE ADDED TO MAPPING" % teevent["type"]) # Enrich description - last will be kept :-S if "description" in teevent.keys(): mispevt.info = mispevt.info + " - %s" % teevent["description"] # Ownership - last will be kept :-S if "owner" in teevent.keys() and "name" in teevent["owner"].keys(): owner = teevent["owner"]["name"] if("email" in teevent["owner"].keys()): email = teevent["owner"]["email"].replace("\\u0040", "@") else: email = "" mispevt.info = mispevt.info + " - by %s (%s)" % (owner, email) # Add sharing indicators (tags) - keep more strict if "share_level" in teevent.keys(): if teevent["share_level"] in self.share_levels.keys(): # sharing level has to be reduced if int(self.share_levels[share_level]["id"]) > int(self.share_levels[teevent["share_level"]]["id"]): share_level = teevent["share_level"] else: print("WARNING: SHARING LEVEL %s SHOULD BE ADDED TO MAPPING" % teevent["share_level"]) # Add Extra Tags if self.extra_tag is not None: mispevt.Tag.append(self.extra_tag) # Add ID to list of ID making this event evtids.append(teevent["id"]) # Set share level mispevt.Tag.append(self.share_levels[share_level]) # Return new MISP event ready for import return [evtids, mispevt] def createEvent(self, mispevent): """ Create a new event in MISP using a hash table structure describing the event """ if mispevent is None: return None # Not empty event jevent = json.dumps(mispevent, cls=EncodeUpdate) misp_event = self.misp.add_event(jevent) mispid = misp_event["Event"]["id"] return mispid def saveMapping(self, mapfile="./mapping.json"): """ Save internal mapping definition """ mappings = { "Sharing" : self.share_levels, "Type" : self.type_map, "Extra-Tag" : self.extra_tag, "Privacy" : self.privacy_levels } try: fd = open(mapfile, "w") json.dump(mappings, fd, sort_keys=True,indent=4,separators=(',', ': ')) fd.close() except Exception as e: print("IMPOSSIBLE TO SAVE MAPPINGS to %s" % mapfile) print(e) return def loadMapping(self, mapfile="./mapping.json"): """ Restore internal mapping from saved JSON file """ try: fd = open(mapfile, "r") mappings = json.load(fd) if "Sharing" in mappings.keys(): self.share_levels = mappings["Sharing"] if "Type" in mappings.keys(): self.type_map = mappings["Type"] if "Extra-Tag" in mappings.keys(): self.extra_tag = mappings["Extra-Tag"] if "Privacy" in mappings.keys(): self.privacy_levels = mappings["Privacy"] fd.close() except Exception as e: print("IMPOSSIBLE TO LOAD MAPPINGS from %s" % mapfile) print(e) sys.exit(0) return
dom, seen, score = row seendate=datetime.datetime.strptime(seen, '%Y-%m-%d') if (seendate >= filterdate): curEvent=MISPEvent() eventName = '[{}] - Domaintools +70 malscore'.format(seen) if (eventName not in events) : print("searching for: {}".format(eventName)) search=pm.search(controller='events', eventinfo=eventName) if ( len(search) == 1): curEvent.load(search[0]) events.update({eventName : curEvent}) elif ( len(search) == 0): curEvent.info=eventName pm.add_event(curEvent) events.update({eventName : curEvent}) else: curEvent=events[eventName] curEvent=events[eventName] attr=curEvent.add_attribute('domain', dom) curEvent.add_attribute_tag("ifx-vetting:score=\"{}\"".format(score), attr.uuid) curEvent.add_attribute_tag("tlp:white", attr.uuid) curEvent.add_attribute_tag("pandemic:covid-19=\"cyber\"", attr.uuid) if ( int(score) >= 95 ): curEvent.add_attribute_tag("estimative-language:likelihood-probability=\"almost-certain\"", attr.uuid) elif ( int(score) >= 80): curEvent.add_attribute_tag("estimative-language:likelihood-probability=\"very-likely\"", attr.uuid)
assert server_test["status"] == 1 assert server_test["post"] == 1 # Get remote user url = f'servers/getRemoteUser/{remote_server["id"]}' remote_user = pymisp._check_json_response(pymisp._prepare_request('GET', url)) check_response(remote_user) assert remote_user["Sync flag"] == "Yes" assert remote_user["Role name"] == "admin" assert remote_user["User"] == "*****@*****.**" # Create testing event event = MISPEvent() event.load_file(os.path.dirname(os.path.realpath(__file__)) + "/event.json") pymisp.delete_event_blocklist(event) event = pymisp.add_event(event, metadata=True) check_response(event) # Publish that event check_response(pymisp.publish(event)) # Preview event url = f'servers/previewEvent/{remote_server["id"]}/{event.uuid}' event_preview = pymisp._check_json_response(pymisp._prepare_request( 'GET', url)) check_response(event_preview) assert event_preview["Event"]["uuid"] == event.uuid # Test pull url = f'servers/pull/{remote_server["id"]}/disable_background_processing:1' pull_response = pymisp._check_json_response(pymisp._prepare_request(
class TestBasic(unittest.TestCase): def setUp(self): self.maxDiff = None self.misp = PyMISP(url, key, True, 'json', True) def _clean_event(self, event): event['Event'].pop('orgc_id', None) event['Event'].pop('uuid', None) event['Event'].pop('sharing_group_id', None) event['Event'].pop('timestamp', None) event['Event'].pop('org_id', None) event['Event'].pop('date', None) event['Event'].pop('RelatedEvent', None) event['Event'].pop('publish_timestamp', None) if event['Event'].get('Attribute'): for a in event['Event'].get('Attribute'): a.pop('uuid', None) a.pop('event_id', None) a.pop('id', None) a.pop('timestamp', None) if event['Event'].get('Orgc'): event['Event']['Orgc'].pop('uuid', None) event['Event']['Orgc'].pop('id', None) if event['Event'].get('Org'): event['Event']['Org'].pop('uuid', None) event['Event']['Org'].pop('id', None) return event['Event'].pop('id', None) def new_event(self): event = self.misp.new_event(0, 1, 0, "This is a test") event_id = self._clean_event(event) to_check = { u'Event': { u'info': u'This is a test', u'locked': False, u'attribute_count': u'0', 'disable_correlation': False, u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Attribute': [], u'proposal_email_lock': False, u'Org': { u'name': u'ORGNAME' }, u'Orgc': { u'name': u'ORGNAME' }, u'Galaxy': [], u'threat_level_id': u'1' } } self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) def add_hashes(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.add_hashes( event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2) self._clean_event(event) to_check = { u'Event': { u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Org': { u'name': u'ORGNAME' }, u'Orgc': { u'name': u'ORGNAME' }, u'Galaxy': [], u'Attribute': [{ u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5' }, { u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1' }, { u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256' }], u'proposal_email_lock': False, u'threat_level_id': u'1' } } self.assertEqual(event, to_check, 'Failed at adding hashes') def publish(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.publish(event) self._clean_event(event) to_check = { u'Event': { u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Org': { u'name': u'ORGNAME' }, u'Orgc': { u'name': u'ORGNAME' }, u'Galaxy': [], u'Attribute': [{ u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5' }, { u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1' }, { u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256' }], u'proposal_email_lock': False, u'threat_level_id': u'1' } } self.assertEqual(event, to_check, 'Failed at publishing event') def delete(self, eventid): event = self.misp.delete_event(eventid) print(event) def delete_attr(self, attrid): event = self.misp.delete_attribute(attrid) print(event) def get(self, eventid): event = self.misp.get_event(eventid) print(event) def get_stix(self, **kwargs): event = self.misp.get_stix(kwargs) print(event) def add(self): event = { u'Event': { u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Attribute': [{ u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5' }, { u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1' }, { u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256' }], u'proposal_email_lock': False, u'threat_level_id': u'1' } } event = self.misp.add_event(event) print(event) def add_user(self): email = '*****@*****.**' role_id = '5' org_id = '1' password = '******' external_auth_required = False external_auth_key = '' enable_password = False nids_sid = '1238717' server_id = '1' gpgkey = '' certif_public = '' autoalert = False contactalert = False disabled = False change_pw = '0' termsaccepted = False newsread = '0' authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' to_check = { 'User': { 'email': email, 'org_id': org_id, 'role_id': role_id, 'password': password, 'external_auth_required': external_auth_required, 'external_auth_key': external_auth_key, 'enable_password': enable_password, 'nids_sid': nids_sid, 'server_id': server_id, 'gpgkey': gpgkey, 'certif_public': certif_public, 'autoalert': autoalert, 'contactalert': contactalert, 'disabled': disabled, 'change_pw': change_pw, 'termsaccepted': termsaccepted, 'newsread': newsread, 'authkey': authkey } } user = self.misp.add_user( email=email, role_id=role_id, org_id=org_id, password=password, external_auth_required=external_auth_required, external_auth_key=external_auth_key, enable_password=enable_password, nids_sid=nids_sid, server_id=server_id, gpgkey=gpgkey, certif_public=certif_public, autoalert=autoalert, contactalert=contactalert, disabled=disabled, change_pw=change_pw, termsaccepted=termsaccepted, newsread=newsread, authkey=authkey) # delete user to allow reuse of test uid = user.get('User').get('id') self.misp.delete_user(uid) # ---------------------------------- # test interesting keys only (some keys are modified(password) and some keys are added (lastlogin) tested_keys = [ 'email', 'org_id', 'role_id', 'server_id', 'autoalert', 'authkey', 'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted', 'newsread', 'contactalert', 'disabled' ] for k in tested_keys: self.assertEqual( user.get('User').get(k), to_check.get('User').get(k), "Failed to match input with output on key: {}".format(k)) def add_organisation(self): name = 'Organisation tests' description = 'This is a test organisation' orgtype = 'Type is a string' nationality = 'French' sector = 'Bank sector' uuid = '16fd2706-8baf-433b-82eb-8c7fada847da' contacts = 'Text field with no limitations' local = False to_check = { 'Organisation': { 'name': name, 'description': description, 'type': orgtype, 'nationality': nationality, 'sector': sector, 'uuid': uuid, 'contacts': contacts, 'local': local } } org = self.misp.add_organisation( name=name, description=description, type=orgtype, nationality=nationality, sector=sector, uuid=uuid, contacts=contacts, local=local, ) # delete organisation to allow reuse of test oid = org.get('Organisation').get('id') self.misp.delete_organisation(oid) # ---------------------------------- tested_keys = [ 'anonymise', 'contacts', 'description', 'local', 'name', 'nationality', 'sector', 'type', 'uuid' ] for k in tested_keys: self.assertEqual( org.get('Organisation').get(k), to_check.get('Organisation').get(k), "Failed to match input with output on key: {}".format(k)) def test_create_event(self): eventid = self.new_event() time.sleep(1) self.delete(eventid) def test_get_event(self): eventid = self.new_event() time.sleep(1) self.get(eventid) time.sleep(1) self.delete(eventid) def test_add_event(self): self.add() time.sleep(1) self.delete(1) def test_del_attr(self): eventid = self.new_event() time.sleep(1) self.delete_attr(1) time.sleep(1) self.delete(eventid) def test_one_or_more(self): self.assertEqual(self.misp._one_or_more(1), (1, )) self.assertEqual(self.misp._one_or_more([1]), [1]) def test_create_user(self): self.add_user() def test_create_organisation(self): self.add_organisation()
attribute_values['value'] = value else: # No value, ignoring continue comment = extract_field(item, 'Comment') if comment: attribute_values["comment"] = '{} {}'.format( attribute_values["comment"], comment) # change value to composite # 127.0.0.1:80 ip-* to 127.0.0.1|80 ip-*|port if mapping['type'] in ['ip-src', 'ip-dst'] and value.count(':') == 1: attribute_values['type'] = mapping['type'] + '|port' attribute_values['value'] = attribute_values['value'].replace( ':', '|') misp_event.add_attribute(**attribute_values) return misp_event if __name__ == '__main__': # test file for composite # https://github.com/fireeye/iocs/blob/master/BlogPosts/9cee306d-5441-4cd3-932d-f3119752634c.ioc x = open('test.ioc', 'r') mispEvent = load_openioc(x.read()) #~ print(mispEvent._json_full()) from pymisp import PyMISP misp = PyMISP('http://misp.local', 'xxxxx') r = misp.add_event(mispEvent) print(r)
class MISP(): def __init__(self, config: Dict[str, Any]): self.logger = logging.getLogger(f'{self.__class__.__name__}') self.logger.setLevel(get_config('generic', 'loglevel')) if not config.get('apikey'): self.available = False self.logger.info('Module not enabled.') return self.available = True self.enable_lookup = False self.enable_push = False self.allow_auto_trigger = False try: self.client = PyMISP(url=config['url'], key=config['apikey'], ssl=config['verify_tls_cert'], timeout=config['timeout']) except Exception as e: self.available = False self.logger.warning(f'Unable to connect to MISP: {e}') return if config.get('enable_lookup'): self.enable_lookup = True if config.get('enable_push'): self.enable_push = True if config.get('allow_auto_trigger'): self.allow_auto_trigger = True self.default_tags: List[str] = config.get('default_tags') # type: ignore self.auto_publish = config.get('auto_publish') self.storage_dir_misp = get_homedir() / 'misp' self.storage_dir_misp.mkdir(parents=True, exist_ok=True) self.psl = get_public_suffix_list() def get_fav_tags(self): return self.client.tags(pythonify=True, favouritesOnly=1) def _prepare_push(self, to_push: Union[List[MISPEvent], MISPEvent], allow_duplicates: bool=False, auto_publish: Optional[bool]=False) -> Union[List[MISPEvent], Dict]: '''Adds the pre-configured information as required by the instance. If duplicates aren't allowed, they will be automatically skiped and the extends_uuid key in the next element in the list updated''' if isinstance(to_push, MISPEvent): events = [to_push] else: events = to_push events_to_push = [] existing_uuid_to_extend = None for event in events: if not allow_duplicates: existing_event = self.get_existing_event(event.attributes[0].value) if existing_event: existing_uuid_to_extend = existing_event.uuid continue if existing_uuid_to_extend: event.extends_uuid = existing_uuid_to_extend existing_uuid_to_extend = None for tag in self.default_tags: event.add_tag(tag) if auto_publish: event.publish() events_to_push.append(event) return events_to_push def push(self, to_push: Union[List[MISPEvent], MISPEvent], allow_duplicates: bool=False, auto_publish: Optional[bool]=None) -> Union[List[MISPEvent], Dict]: if auto_publish is None: auto_publish = self.auto_publish if self.available and self.enable_push: events = self._prepare_push(to_push, allow_duplicates, auto_publish) if not events: return {'error': 'All the events are already on the MISP instance.'} if isinstance(events, Dict): return {'error': events} to_return = [] for event in events: try: # NOTE: POST the event as published publishes inline, which can tak a long time. # Here, we POST as not published, and trigger the publishing in a second call. if hasattr(event, 'published'): background_publish = event.published else: background_publish = False if background_publish: event.published = False new_event = self.client.add_event(event, pythonify=True) if background_publish and isinstance(new_event, MISPEvent): self.client.publish(new_event) except requests.exceptions.ReadTimeout: return {'error': 'The connection to MISP timed out, try increasing the timeout in the config.'} if isinstance(new_event, MISPEvent): to_return.append(new_event) else: return {'error': new_event} return to_return else: return {'error': 'Module not available or push not enabled.'} def get_existing_event_url(self, permaurl: str) -> Optional[str]: attributes = self.client.search('attributes', value=permaurl, limit=1, page=1, pythonify=True) if not attributes or not isinstance(attributes[0], MISPAttribute): return None url = f'{self.client.root_url}/events/{attributes[0].event_id}' return url def get_existing_event(self, permaurl: str) -> Optional[MISPEvent]: attributes = self.client.search('attributes', value=permaurl, limit=1, page=1, pythonify=True) if not attributes or not isinstance(attributes[0], MISPAttribute): return None event = self.client.get_event(attributes[0].event_id, pythonify=True) if isinstance(event, MISPEvent): return event return None def lookup(self, node: URLNode, hostnode: HostNode) -> Union[Dict[str, Set[str]], Dict[str, Any]]: if self.available and self.enable_lookup: tld = self.psl.get_tld(hostnode.name) domain = re.sub(f'.{tld}$', '', hostnode.name).split('.')[-1] to_lookup = [node.name, hostnode.name, f'{domain}.{tld}'] + hostnode.resolved_ips if hasattr(hostnode, 'cnames'): to_lookup += hostnode.cnames if not node.empty_response: to_lookup.append(node.body_hash) if attributes := self.client.search(controller='attributes', value=to_lookup, enforce_warninglist=True, pythonify=True): if isinstance(attributes, list): to_return: Dict[str, Set[str]] = defaultdict(set) # NOTE: We have MISPAttribute in that list for a in attributes: to_return[a.event_id].add(a.value) # type: ignore return to_return else: # The request returned an error return attributes # type: ignore return {'info': 'No hits.'} else:
#!/usr/bin/env python # -*- coding: utf-8 -*- import argparse from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert from pymisp.tools import load_openioc_file if __name__ == '__main__': parser = argparse.ArgumentParser(description='Convert an OpenIOC file to a MISPEvent. Optionnaly send it to MISP.') parser.add_argument("-i", "--input", required=True, help="Input file") group = parser.add_mutually_exclusive_group(required=True) group.add_argument("-o", "--output", help="Output file") group.add_argument("-m", "--misp", action='store_true', help="Create new event on MISP") args = parser.parse_args() misp_event = load_openioc_file(args.input) if args.misp: pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) pymisp.add_event(misp_event) else: with open(args.output, 'w') as f: f.write(misp_event.to_json())