def from_remote(self, event_id): from pymisp import PyMISP from keys import misp_url, misp_key, misp_verifycert misp = PyMISP(misp_url, misp_key, misp_verifycert) result = misp.get(event_id) self.misp_event = MISPEvent() self.misp_event.load(result)
def test_deleteEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) d = pymisp.delete_event(2) self.assertEqual(d, {'message': 'Event deleted.'}) d = pymisp.delete_event(3) self.assertEqual(d, {'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3'})
def test_getVersions(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) api_version = pymisp.get_api_version() self.assertEqual(api_version, {"version": pm.__version__}) server_version = pymisp.get_version() self.assertEqual(server_version, {"version": "2.4.56"})
def test_auth_error(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) error = pymisp.get(1) response = self.auth_error_msg response["errors"] = [response["message"]] self.assertEqual(error, response)
def test_getEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) e1 = pymisp.get_event(2) e2 = pymisp.get(2) self.assertEqual(e1, e2) self.assertEqual(self.event, e2)
def test_tag_event(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) uuid = self.event["Event"]["uuid"] pymisp.tag(uuid, "foo") self.assertRaises(pm.PyMISPError, pymisp.tag, "test_uuid", "foo") self.assertRaises(pm.PyMISPError, pymisp.tag, uuid.replace("a", "z"), "foo")
def test_freetext_offdomain(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) try: pymisp.freetext(1, None, adhereToWarninglists='hard') self.assertFalse('Exception required for off domain value') except Exception: pass
def test_change_toids_invalid(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) try: pymisp.change_toids(self.key, 42) self.assertFalse('Exception required for off domain value') except Exception: pass
def test_flatten_error_messages_singular(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) pymisp.get(1) response = self.auth_error_msg response['error'] = ['foo', 'bar', 'baz'] messages = pymisp.flatten_error_messages(response) self.assertEqual(["foo", "bar", "baz"], messages)
def test_publish(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) e = pymisp.publish(self.event) pub = self.event pub['Event']['published'] = True self.assertEqual(e, pub) e = pymisp.publish(self.event) self.assertEqual(e, {'error': 'Already published'})
def test_deleteEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) d = pymisp.delete_event(2) self.assertEqual(d, {"message": "Event deleted."}) d = pymisp.delete_event(3) self.assertEqual( d, {"errors": ["Invalid event"], "message": "Invalid event", "name": "Invalid event", "url": "/events/3"} )
def test_flatten_error_messages_plural(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) error = pymisp.get(1) self.assertIn("Authentication failed", error["message"]) response = self.auth_error_msg response['errors'] = {'foo': 42, 'bar': False, 'baz': ['oo', 'ka']} messages = pymisp.flatten_error_messages(response) self.assertEqual(set(['42 (foo)', 'False (bar)', 'oo', 'ka']), set(messages))
def test_updateEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) e0 = pymisp.update_event("5758ebf5-c898-48e6-9fe9-5665c0a83866", json.dumps(self.event)) e1 = pymisp.update_event("5758ebf5-c898-48e6-9fe9-5665c0a83866", self.event) self.assertEqual(e0, e1) e2 = pymisp.update(e0) self.assertEqual(e1, e2) self.assertEqual(self.event, e2)
def test_updateEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) e0 = pymisp.update_event(2, json.dumps(self.event)) e1 = pymisp.update_event(2, self.event) self.assertEqual(e0, e1) e2 = pymisp.update(e0) self.assertEqual(e1, e2) self.assertEqual(self.event, e2)
def test_publish(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) e = pymisp.publish(self.event) # requests-mock always return the non-published event pub = self.event pub["Event"]["published"] = True # self.assertEqual(e, pub) FIXME: broken test, not-published event returned e = pymisp.publish(self.event) self.assertEqual(e, {"error": "Already published"})
def test_flatten_error_messages_nested(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) error = pymisp.get(1) self.assertIn("Authentication failed", error["message"]) response = self.auth_error_msg response['errors'] = { 'fo': {'o': 42}, 'ba': {'r': True}, 'b': {'a': ['z']}, 'd': {'e': {'e': ['p']}}} messages = pymisp.flatten_error_messages(response) self.assertEqual(set(['Error in o: 42', 'Error in r: True', 'Error in a: z', "Error in e: {'e': ['p']}"]), set(messages))
def test_newEvent(self, m): error_empty_info = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': ['Error in info: Info cannot be empty.'], 'url': '/events/add'} error_empty_info_flatten = {u'message': u'The event could not be saved.', u'name': u'Add event failed.', u'errors': [u"Error in info: Info cannot be empty."], u'url': u'/events/add'} self.initURI(m) pymisp = PyMISP(self.domain, self.key) m.register_uri('POST', self.domain + 'events', json=error_empty_info) # TODO Add test exception if info field isn't set response = pymisp.new_event(0, 1, 0, 'Foo') self.assertEqual(response, error_empty_info_flatten) m.register_uri('POST', self.domain + 'events', json=self.new_misp_event) response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False) self.assertEqual(response, self.new_misp_event)
def send(api: PyMISP, request_type: str, url: str, data=None, check_errors: bool = True) -> dict: if data is None: data = {} response = api._prepare_request(request_type, url, data=data) response = api._check_json_response(response) if check_errors: check_response(response) return response
def create_or_update_site_admin(connector: PyMISP, user: MISPUser) -> MISPUser: to_return_user = connector.add_user(user) if isinstance(to_return_user, MISPUser): return to_return_user # The user already exists for u in connector.users(): if u.email == user.email: to_return_user = connector.update_user(user, u.id) # type: ignore if isinstance(to_return_user, MISPUser): return to_return_user raise Exception(f'Unable to update {user.email}: {to_return_user}') else: raise Exception(f'Unable to create {user.email}: {to_return_user}')
def init(self): if PyMISP is None: raise ValueError('Could not import pymisp. Please install it.') # Initialize MISP connection self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key, self.parameters.misp_verify) # URLs used for deleting and adding MISP event tags self.misp_add_tag_url = urljoin(self.parameters.misp_url, 'events/addTag') self.misp_del_tag_url = urljoin(self.parameters.misp_url, 'events/removeTag')
def __init__(self, misp_key, misp_url, misp_verify_cert, siem_mode=False, debugon=False): self.misp = PyMISP(misp_url, misp_key, misp_verify_cert, 'json') self.debugon = debugon if siem_mode: self.siem_mode = True self.separator = "," self.use_headers = True self.use_filename_regex = False
def test_deleteEvent(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) d = pymisp.delete_event(2) self.assertEqual(d, {'message': 'Event deleted.'}) d = pymisp.delete_event(3) self.assertEqual( d, { 'errors': ['Invalid event'], 'message': 'Invalid event', 'name': 'Invalid event', 'url': '/events/3' })
def pcap_analysis(analysis_id): """ Analyze the pcap file, first extract the IOCs then check in the MISP server """ analysis = Analysis.objects.get(pk=analysis_id) print('analyzing : %s' % analysis_id) try: # Fix ME: log issues PyMISPError here server = PyMISP(settings.MISP_SERVER, settings.MISP_KEY, True, 'json') pc = Pcap(settings.TEMP_FOLDER + '/' + analysis_id + '.pcap') malicious = False results = [] pc.extract_indicators() analysis.status = AnalysisStatus.SEARCH analysis.save() print('Searching for %i indicators' % len(pc.indicators)) for ind in pc.indicators: events = server.search(values = [ind['value']]) if len(events['response']) > 0: malicious = True for event in events['response']: results.append({ 'indicator': ind['value'], 'event': event['Event']['info'], 'event_id': event['Event']['id'] }) except PyMISPError: # TODO: create log here analysis.status = AnalysisStatus.ERROR analysis.result = AnalysisResult.ERROR analysis.save() else: for alert in results: a = Alert() a.analysis = analysis a.indicator = alert['indicator'] a.event_id = int(alert['event_id']) a.event_name = alert['event'] a.save() if malicious: analysis.result = AnalysisResult.MALICIOUS else: analysis.result = AnalysisResult.NOTHING analysis.status = AnalysisStatus.DONE analysis.save() # Securely delete the pcap file subprocess.check_call([ 'shred', '-zu', settings.TEMP_FOLDER + '/' + analysis_id + '.pcap'] )
def test_user_must_change_password_by_myself(self): # Admin set that user must change password updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr) check_response(updated_user) self.assertEqual(updated_user.change_pw, "1") # User try to change back trough API logged_in = PyMISP(url, self.test_usr.authkey) logged_in.update_user({'change_pw': 0}, self.test_usr) updated_user = self.admin_misp_connector.get_user(self.test_usr) # Should not be possible self.assertEqual(updated_user.change_pw, "1")
def misp_connection(url, misp_key, proxies_usage): try: if proxies_usage: proxies = {} proxies ["http"] = config_parser("misp","http") proxies ["https"] = config_parser("misp","https") misp = PyMISP(url, misp_key, False, 'json', proxies=proxies) else: misp = PyMISP(url, misp_key, False, 'json',None) except PyMISPError: print("\t [!] Error connecting to MISP instance. Check if your MISP instance it's up!") return None return misp
def test_sample_upload(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) pymisp.upload_sample("tmux", "tests/viper-test-files/test_files/tmux", 1) pymisp.upload_sample("tmux", "non_existing_file", 1) pymisp.upload_sample("tmux", b"binblob", 1)
def _build_iterator(self, now): """ Queries specified MISP instance to get events, filter them according to given filters and returns events. :param now: unused :return: itarable. Holds the events returned by the API that matched the given filters. """ # Sanity checks if self.automation_key is None: raise RuntimeError('{} - MISP Automation Key not set'.format( self.name)) if self.url is None: raise RuntimeError('{} - MISP URL not set'.format(self.name)) kwargs = {'ssl': self.verify_cert} if self.verify_cert and 'REQUESTS_CA_BUNDLE' in os.environ: kwargs['ssl'] = os.environ['REQUESTS_CA_BUNDLE'] if self.client_cert_required: kwargs['cert'] = (self.cert_file, self.key_file) # Establish API connection misp = PyMISP(self.url, self.automation_key, **kwargs) filters = None if self.filters is not None: # Map self.filters in the way they are given by the configuration to the way they are required in order # to actually filter the events returned by the API. filters = self.filters.copy() if 'datefrom' in filters: filters['timestamp'] = filters.pop('datefrom') du = filters.pop('dateuntil', None) if du is not None: filters['dateuntil'] = du if 'event_tags' in filters and filters['event_tags']: filters['tags'] = [ x.strip() for x in filters['event_tags'].split(",") ] LOG.info('{} - query filters: {!r}'.format(self.name, filters)) # Get index from all events via API matching the given filter r = misp.get_index(filters) events = r['response'] return imap(partial(self._load_event, misp), events)
def __init__(self, config): # Initialize config self.config = config # Initialize MISP self.misp = PyMISP(self.config['misp']['url'], self.config['misp']['key'], False, 'json') # Initialize OpenCTI client self.opencti = OpenCTI( self.config['opencti']['api_url'], self.config['opencti']['api_key'], os.path.dirname(os.path.abspath(__file__)) + '/misp.log', True )
def _misp_create_event_function(self, event, *args, **kwargs): """Function: create a MISP event from an incident """ try: def get_config_option(option_name, optional=False): """Given option_name, checks if it is in app.config. Raises ValueError if a mandatory option is missing""" option = self.options.get(option_name) if option is None and optional is False: err = "'{0}' is mandatory and is not set in ~/.resilient/app.config file. You must set this value to run this function".format(option_name) raise ValueError(err) else: return option API_KEY = get_config_option("misp_key") URL = get_config_option("misp_url") VERIFY_CERT = True if get_config_option("verify_cert").lower() == "true" else False # Get the function parameters: misp_event_name = kwargs.get("misp_event_name") # text misp_distribution = kwargs.get("misp_distribution") # number misp_analysis_level = kwargs.get("misp_analysis_level") # number misp_threat_level = kwargs.get("misp_threat_level") # number log = logging.getLogger(__name__) log.info("misp_event_name: %s", misp_event_name) log.info("misp_distribution: %s", misp_distribution) log.info("misp_analysis_level: %s", misp_analysis_level) log.info("misp_threat_level: %s", misp_threat_level) yield StatusMessage("Setting up connection to MISP") misp_client = PyMISP(URL, API_KEY, VERIFY_CERT, 'json') event = misp_client.new_event(misp_distribution, misp_threat_level, misp_analysis_level, misp_event_name) log.info(event) yield StatusMessage("Event has been created") results = { "success": True, "content": event } # Produce a FunctionResult with the results yield FunctionResult(results) except Exception: yield FunctionError()
def getMISPInfo(hash): """ Retrieves information from a MISP instance :param hash: hash value :return info: info object """ info = {'misp_available': False, 'misp_events': ''} requests.packages.urllib3.disable_warnings() # I don't care if MISP_API_KEY == "-": return info # Prepare API request if args.debug: print("[D] Querying MISP: %s" % MISP_URL) misp = PyMISP(MISP_URL, MISP_API_KEY, args.verifycert, 'json') try: if args.debug: print("[D] Query: values=%s" % hash) result = misp.search('attributes', values=[hash]) if result['response']: misp_info = [] misp_events = [] if args.debug: print(json.dumps(result['response'])) for r in result['response']["Attribute"]: # Try to get info on the events event_info = "" misp_events.append(r['event_id']) e_result = misp.search('events', eventid=r['event_id']) if e_result['response']: event_info = e_result['response'][0]['Event']['info'] # too much #if args.debug: # print(json.dumps(e_result['response'])) # Create MISP info object misp_info.append({ 'event_info': event_info, 'event_id': r['event_id'], 'comment': r['comment'], }) info['misp_info'] = misp_info info['misp_events'] = ",".join(misp_events) info['misp_available'] = True else: info['misp_available'] = False except Exception as e: if args.debug: traceback.print_exc() return info
class MISPCollectorBot(CollectorBot): def init(self): if PyMISP is None: self.logger.error('Could not import pymisp. Please install it.') self.stop() # Initialise MISP connection self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key, self.parameters.misp_verify) # URLs used for deleting and adding MISP event tags self.misp_add_tag_url = urljoin(self.parameters.misp_url, 'events/addTag') self.misp_del_tag_url = urljoin(self.parameters.misp_url, 'events/removeTag') def process(self): # Grab the events from MISP misp_result = self.misp.search( tags=self.parameters.misp_tag_to_process ) # Process the response and events if 'response' in misp_result: # Extract the MISP event details for e in misp_result['response']: misp_event = e['Event'] # Send the results to the parser report = self.new_report() report.add('raw', json.dumps(misp_event, sort_keys=True)) report.add('feed.url', self.parameters.misp_url) self.send_message(report) # Finally, update the tags on the MISP events. # Note PyMISP does not currently support this so we use # the API URLs directly with the requests module. for misp_event in misp_result['response']: # Remove the 'to be processed' tag self.misp.remove_tag(misp_event, self.parameters.misp_tag_to_process) # Add a 'processed' tag to the event self.misp.add_tag(misp_event, self.parameters.misp_tag_processed)
def each_with_type(self, target, file_type): search_result = False self.misp = PyMISP(self.misp_api_endpoint, self.misp_api_key, False) if file_type == 'url': misp_result = self.misp.search_all(keyword) if 'response' in misp_result: # Extract the MISP event details for e in misp_result['response']: search_result = True misp_event = e['Event'] for tag in misp_event['Tag']: self.add_tag(tag['name']) Attribute = misp_event['Attribute'] self.results[misp_event['info']] = self.results.get(misp_event['info'], []) for a in Attribute: self.results[misp_event['info']].append({ 'date':misp_event['date'], 'comment':a.get('comment'), 'value':a.get('value'), 'type':a.get('type'), 'category':a.get('category') }) else: with open(target) as f: data = f.read() md5 = hashlib.md5(data).hexdigest() sha256 = hashlib.sha256(data).hexdigest() search_content = [md5, sha256] for keyword in search_content: misp_result = self.misp.search_all(keyword) # Process the response and events if 'response' in misp_result: # Extract the MISP event details for e in misp_result['response']: search_result = True misp_event = e['Event'] for tag in misp_event['Tag']: self.add_tag(tag['name']) Attribute = misp_event['Attribute'] self.results[misp_event['info']] = self.results.get(misp_event['info'], []) for a in Attribute: self.results[misp_event['info']].append({ 'date':misp_event['date'], 'comment':a.get('comment'), 'value':a.get('value'), 'type':a.get('type'), 'category':a.get('category') }) return search_result
def getMISPData(self, since=None): # Connect to your MISP API misp = PyMISP(self.url, self.key, True, 'json') since = since if since else "5d" if since.lower() == "all": since = "" misp_last = misp.download_last(since) # Verify output if 'message' in misp_last.keys(): if misp_last['message'].lower().startswith('no matches'): return [] # No output elif misp_last['message'].startswith('Authentication failed.'): raise Exception("[-] MISP Authentication failed") if not 'response' in misp_last: raise Exception("[-] Error occured while fetching MISP data") return misp_last['response']
def test_change_login_disabled(self): with MISPSetting(self.admin_misp_connector, "MISP.disable_user_login_change", True): new_email = 'testusr@user' + random() + '.local' logged_in = PyMISP(url, self.test_usr.authkey) logged_in.global_pythonify = True # Try to change email updated_user = logged_in.update_user({'email': new_email}, self.test_usr) check_response(updated_user) # Change should be not successful self.assertEqual(self.test_usr.email, updated_user.email)
def get_misp_instance(): """ :return: MISP Instance :rtype: PyMISP """ # Proxy settings are taken from the config file and converted to a dict. if PySight_settings.MISP_PROXY: misp_proxies = { 'http': str(PySight_settings.PROXY_URL), 'https': str(PySight_settings.PROXY_URL) } else: misp_proxies = {} try: # URL of the MISP instance, API key and SSL certificate validation are taken from the config file. return PyMISP(PySight_settings.MISP_URL, PySight_settings.MISP_KEY, PySight_settings.MISP_VERIFYCERT, proxies=misp_proxies) #return PyMISP(PySight_settings.MISP_URL, PySight_settings.MISP_KEY, PySight_settings.MISP_VERIFYCERT, # proxies=misp_proxies) except Exception: PySight_settings.logger.error('Unexpected error in MISP init: %s', sys.exc_info()) return False
def get_misp_instance(): """ :return: MISP Instance :rtype: PyMISP """ print('****hello****in get_misp_instance') # Proxy settings are taken from the config file and converted to a dict. if PySight_settings.USE_MISP_PROXY: misp_proxies = { 'http': str(PySight_settings.proxy_address), 'https': str(PySight_settings.proxy_address) } else: misp_proxies = {} try: # URL of the MISP instance, API key and SSL certificate validation are taken from the config file. return ExpandedPyMISP(PySight_settings.misp_url, PySight_settings.misp_key, PySight_settings.misp_verifycert) return PyMISP(PySight_settings.misp_url, PySight_settings.misp_key, PySight_settings.misp_verifycert) except Exception: PySight_settings.logger.error('Unexpected error in MISP init: %s', sys.exc_info()) return False
def init(self): if PyMISP is None: raise ValueError('Could not import pymisp. Please install it.') # Initialize MISP connection self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key, self.parameters.misp_verify)
def test_eventObject(self, m): self.initURI(m) pymisp = PyMISP(self.domain, self.key) misp_event = MISPEvent(pymisp.describe_types) with open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r') as f: misp_event.load(f.read()) json.dumps(misp_event, cls=MISPEncode)
class MISP(): def __init__(self, config: Dict[str, Any]): self.logger = logging.getLogger(f'{self.__class__.__name__}') self.logger.setLevel(get_config('generic', 'loglevel')) if not config.get('apikey'): self.available = False self.logger.info('Module not enabled.') return self.available = True self.enable_lookup = False self.enable_push = False try: self.client = PyMISP(url=config['url'], key=config['apikey'], ssl=config['verify_tls_cert']) except Exception as e: self.available = False self.logger.warning(f'Unable to connect to MISP: {e}') return if config.get('enable_lookup'): self.enable_lookup = True if config.get('enable_push'): self.enable_push = True self.storage_dir_misp = get_homedir() / 'misp' self.storage_dir_misp.mkdir(parents=True, exist_ok=True) def push(self, event: MISPEvent) -> Union[MISPEvent, Dict]: if self.available and self.enable_push: return self.client.add_event(event, pythonify=True) else: return {'error': 'Module not available or push not enabled.'}
def init(): """ Initialize PyMISP Get configuration settings from config file """ global source source = PyMISP(misp_url, misp_key, misp_verifycert, 'json')
def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None): if event_info: event.info = event_info event.distribution = sanitize_event_distribution(distribution) event.threat_level_id = sanitize_event_threat_level_id(threat_level_id) event.analysis = sanitize_event_analysis(analysis) if publish: event.publish() # # TODO: handle multiple MISP instance misp = PyMISP(misp_url, misp_key, misp_verifycert) #print(event.to_json()) misp_event = misp.add_event(event, pythonify=True) # # TODO: handle error event_metadata = extract_event_metadata(misp_event) return event_metadata
def get_misp_connection(config=None): global misp_connection if misp_connection: return misp_connection if not config: raise MaltegoException( "ERROR: MISP connection not yet established, and config not provided as parameter." ) if config['MISP_maltego.local.misp_verify'] in [ 'True', 'true', 1, 'yes', 'Yes' ]: misp_verify = True else: misp_verify = False if config['MISP_maltego.local.misp_debug'] in [ 'True', 'true', 1, 'yes', 'Yes' ]: misp_debug = True else: misp_debug = False try: misp_connection = PyMISP(config['MISP_maltego.local.misp_url'], config['MISP_maltego.local.misp_key'], misp_verify, 'json', misp_debug) except Exception: raise MaltegoException( "ERROR: Cannot connect to MISP server. Please verify your MISP_Maltego.conf settings" ) return misp_connection
def test_user_must_change_password(self): updated_user = self.admin_misp_connector.update_user({'change_pw': 1}, self.test_usr) check_response(updated_user) self.assertEqual(updated_user.change_pw, "1") # Try to login, should still work because key is still valid PyMISP(url, self.test_usr.authkey) updated_user = self.admin_misp_connector.update_user({'change_pw': 0}, self.test_usr) check_response(updated_user) self.assertEqual(updated_user.change_pw, "0") # Try to login, should also still works PyMISP(url, self.test_usr.authkey)
def ping_misp(): try: PyMISP(misp_url, misp_key, misp_verifycert) return True except Exception as e: print(e) return False
def get_misp_connection(config=None, parameters=None): global misp_connection if misp_connection: return misp_connection if not config: raise MaltegoException("ERROR: MISP connection not yet established, and config not provided as parameter.") misp_verify = True misp_debug = False misp_url = None misp_key = None try: if is_local_exec_mode(): misp_url = config['MISP_maltego.local.misp_url'] misp_key = config['MISP_maltego.local.misp_key'] if config['MISP_maltego.local.misp_verify'] in ['False', 'false', 0, 'no', 'No']: misp_verify = False if config['MISP_maltego.local.misp_debug'] in ['True', 'true', 1, 'yes', 'Yes']: misp_debug = True if is_remote_exec_mode(): try: misp_url = parameters['mispurl'].value misp_key = parameters['mispkey'].value except AttributeError: raise MaltegoException("ERROR: mispurl and mispkey need to be set to something valid") misp_connection = PyMISP(misp_url, misp_key, misp_verify, 'json', misp_debug, tool='misp_maltego') except Exception: if is_local_exec_mode(): raise MaltegoException("ERROR: Cannot connect to MISP server. Please verify your MISP_Maltego.conf settings.") if is_remote_exec_mode(): raise MaltegoException("ERROR: Cannot connect to MISP server. Please verify your settings (MISP URL and API key), and ensure the MISP server is reachable from the internet.") return misp_connection
def init(misp_url, misp_key): return PyMISP(misp_url, misp_key, misp_verifycert, 'json', debug=False, proxies=proxies)
class MISPCollectorBot(CollectorBot): def init(self): if PyMISP is None: self.logger.error('Could not import pymisp. Please install it.') self.stop() # Initialise MISP connection self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key, self.parameters.misp_verify) # URLs used for deleting and adding MISP event tags self.misp_add_tag_url = urljoin(self.parameters.misp_url, 'events/addTag') self.misp_del_tag_url = urljoin(self.parameters.misp_url, 'events/removeTag') def process(self): # Grab the events from MISP misp_result = self.misp.search( tags=self.parameters.misp_tag_to_process) # Process the response and events if 'response' in misp_result: # Extract the MISP event details for e in misp_result['response']: misp_event = e['Event'] # Send the results to the parser report = self.new_report() report.add('raw', json.dumps(misp_event, sort_keys=True)) report.add('feed.url', self.parameters.misp_url) self.send_message(report) # Finally, update the tags on the MISP events. # Note PyMISP does not currently support this so we use # the API URLs directly with the requests module. for misp_event in misp_result['response']: # Remove the 'to be processed' tag self.misp.remove_tag(misp_event, self.parameters.misp_tag_to_process) # Add a 'processed' tag to the event self.misp.add_tag(misp_event, self.parameters.misp_tag_processed)
def onDatabaseUpdate(self): lastUpdate = db.p_readSetting(self.collectionName, "last_update") now = datetime.utcnow().replace(tzinfo = pytz.utc) if lastUpdate: last = dateutil.parser.parse(lastUpdate) delta = now - last since = "%sm"%math.ceil(delta.total_seconds()/60) else: since = "" if self.url and self.key: try: # Misp interface misp = PyMISP(self.url, self.key, True, 'json') except: return "[-] Failed to connect to MISP. Wrong URL?" try: # Fetch data misp_last = misp.download_last(since) # Check data if 'message' in misp_last.keys(): if misp_last['message'].lower().startswith('no matches'): return "[+] MISP collection updated (0 updates)" elif misp_last['message'].startswith('Authentication failed.'): return "[-] MISP Authentication failed" if not 'response' in misp_last: print(misp_last); return "[-] Error occured while fetching MISP data" # Nothing wrong so far, so let's continue bulk =[] for entry in progressbar(misp_last['response']): # Get info attrs=entry['Event']['Attribute'] CVEs= [x['value'] for x in attrs if x['type'] == 'vulnerability'] if len(CVEs) == 0: continue threats= [x['value'] for x in attrs if x['category'] == 'Attribution' and x['type'] == 'threat-actor'] tags = [x['value'] for x in attrs if x['category'] == 'Other' and x['type'] == 'text'] tags.extend([x['value'] for x in attrs if x['category'] == 'External analysis' and x['type'] == 'text']) # Add info to each CVE for cve in CVEs: item={'id':cve} if len(threats) !=0: item['threats'] = threats if len(tags) !=0: item['tags'] = tags if len(item.keys())>1: bulk.append(item) # Avoid empty collections db.p_bulkUpdate(self.collectionName, "id", bulk) #update database info after successful program-run db.p_writeSetting(self.collectionName, "last_update", now.strftime("%a, %d %h %Y %H:%M:%S %Z")) return "[+] MISP collection updated (%s updates)"%len(bulk) except Exception as e: print(e);print(e);return "[-] Something went wrong..." else: return "[-] MISP credentials not specified"
def init(self): # Initialise MISP connection self.misp = PyMISP(self.parameters.misp_url, self.parameters.misp_key, 'json') # URLs used for deleting and adding MISP event tags self.misp_add_tag_url = urljoin(self.parameters.misp_url, 'events/addTag') self.misp_del_tag_url = urljoin(self.parameters.misp_url, 'events/removeTag')
def test_newEvent(self, m): error_empty_info = {'message': 'The event could not be saved.', 'name': 'Add event failed.', 'errors': {'Event': {'info': ['Info cannot be empty.']}}, 'url': '/events/add'} error_empty_info_flatten = {u'message': u'The event could not be saved.', u'name': u'Add event failed.', u'errors': [u"Error in info: Info cannot be empty."], u'url': u'/events/add'} self.initURI(m) pymisp = PyMISP(self.domain, self.key) with self.assertRaises(pm.api.NewEventError): pymisp.new_event() with self.assertRaises(pm.api.NewEventError): pymisp.new_event(0) with self.assertRaises(pm.api.NewEventError): pymisp.new_event(0, 1) m.register_uri('POST', self.domain + 'events', json=error_empty_info) response = pymisp.new_event(0, 1, 0) self.assertEqual(response, error_empty_info_flatten) m.register_uri('POST', self.domain + 'events', json=self.new_misp_event) response = pymisp.new_event(0, 1, 0, "This is a test.", '2016-08-26', False) self.assertEqual(response, self.new_misp_event)
def test_newEvent(self, m): error_empty_info = { "message": "The event could not be saved.", "name": "Add event failed.", "errors": {"Event": {"info": ["Info cannot be empty."]}}, "url": "/events/add", } error_empty_info_flatten = { u"message": u"The event could not be saved.", u"name": u"Add event failed.", u"errors": [u"Error in info: Info cannot be empty."], u"url": u"/events/add", } self.initURI(m) pymisp = PyMISP(self.domain, self.key) m.register_uri("POST", self.domain + "events", json=error_empty_info) # TODO Add test exception if info field isn't set response = pymisp.new_event(0, 1, 0, "Foo") self.assertEqual(response, error_empty_info_flatten) m.register_uri("POST", self.domain + "events", json=self.new_misp_event) response = pymisp.new_event(0, 1, 0, "This is a test.", "2016-08-26", False) self.assertEqual(response, self.new_misp_event)
def mispextract(fpath): # md5="626576e5f0f85d77c460a322a92bb267" url="https://misppriv.circl.lu" apikey="Enter your API key HERE" conn=internet_on() if conn: api=PyMISP(url,apikey,ssl=True, out_type='json' ,debug=False, proxies=None) md5=prelim(fpath) response = api.search_all(md5) length=len(response["response"]) retu={"connection":True, "positives":length } return retu #print(len(list(root))) else: print("No connectivity with MISP") retu={"connection":False, "positives":0 } return retu
class MISPReceiver(): hash_iocs = {} filename_iocs = {} c2_iocs = {} yara_rules = {} debugon = False # Output siem_mode = False separator = ";" use_headers = False use_filename_regex = True def __init__(self, misp_key, misp_url, misp_verify_cert, siem_mode=False, debugon=False): self.misp = PyMISP(misp_url, misp_key, misp_verify_cert, 'json') self.debugon = debugon if siem_mode: self.siem_mode = True self.separator = "," self.use_headers = True self.use_filename_regex = False def get_iocs_last(self, last): # Retrieve events from MISP result = self.misp.download_last(last) self.events = result['response'] # Process each element (without related eevents) for event_element in self.events: event = event_element["Event"] # Info for Comment info = event['info'] uuid = event['uuid'] comment = "{0} - UUID: {1}".format(info.encode('unicode_escape'), uuid) # Event data for attribute in event['Attribute']: # Skip iocs that are not meant for ioc detection if attribute['to_ids'] == False: continue # Value value = attribute['value'] # Non split type if '|' not in attribute['type']: self.add_ioc(attribute['type'], value, comment, uuid, info) # Split type else: # Prepare values type1, type2 = attribute['type'].split('|') value1, value2 = value.split('|') # self.add_ioc(type1, value1, comment) self.add_ioc(type2, value2, comment, uuid, info) def add_ioc(self, ioc_type, value, comment, uuid, info): # Cleanup value value = value.encode('unicode_escape') # Debug if self.debugon: print "{0} = {1}".format(ioc_type, value) # C2s if ioc_type in ('hostname', 'ip-dst', 'domain'): if value == '127.0.0.1': return self.c2_iocs[value] = comment # Hash if ioc_type in ('md5', 'sha1', 'sha256'): # No empty files if value == 'd41d8cd98f00b204e9800998ecf8427e' or \ value == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' or \ value == 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855': return self.hash_iocs[value] = comment # Filenames if ioc_type in ('filename', 'filepath'): # Add prefix to filenames if not re.search(r'^([a-zA-Z]:|%)', value): if not self.siem_mode: value = "\\\\{0}".format(value) if self.use_filename_regex: self.filename_iocs[my_escape(value)] = comment else: self.filename_iocs[value.decode('string_escape')] = comment # Yara if ioc_type in ('yara'): self.add_yara_rule(value, uuid, info) def add_yara_rule(self, yara_rule, uuid, info): identifier = generate_identifier(info) self.yara_rules[identifier] = ur'%s' % repair_yara_rule(yara_rule.decode('string_escape'), uuid) def write_iocs(self, output_path, output_path_yara): # Write C2 IOCs self.write_file(os.path.join(output_path, "misp-c2-iocs.txt"), self.c2_iocs, "c2") # Write Filename IOCs self.write_file(os.path.join(output_path, "misp-filename-iocs.txt"), self.filename_iocs, "filename") # Write Hash IOCs self.write_file(os.path.join(output_path, "misp-hash-iocs.txt"), self.hash_iocs, "hash") # Yara if len(self.yara_rules) > 0: # Create dir if not exists if not os.path.exists(output_path_yara): os.makedirs(output_path_yara) # Loop through rules (keys are identifiers used for file names) for yara_rule in self.yara_rules: output_rule_filename = os.path.join(output_path_yara, "%s.yar" % yara_rule) self.write_yara_rule(output_rule_filename, self.yara_rules[yara_rule]) print "{0} YARA rules written to directory {1}".format(len(self.yara_rules), output_path_yara) def write_file(self, ioc_file, iocs, ioc_type): with open(ioc_file, 'w') as file: if self.use_headers: file.write("{0}{1}description\n".format(ioc_type, self.separator)) for ioc in iocs: file.write("{0}{2}{1}\n".format(ioc,iocs[ioc],self.separator)) print "{0} IOCs written to file {1}".format(len(iocs), ioc_file) def write_yara_rule(self, yara_file, yara_rule): # Write the YARA rule with io.open(yara_file, 'wb') as fh: fh.write(ur'%s' % yara_rule)
def setUp(self): self.maxDiff = None self.misp = PyMISP(url, key, True, 'json')
class TestBasic(unittest.TestCase): def setUp(self): self.maxDiff = None self.misp = PyMISP(url, key, True, 'json') def _clean_event(self, event): event['Event'].pop('uuid', None) event['Event'].pop('timestamp', None) event['Event'].pop('date', None) event['Event'].pop('org', None) event['Event'].pop('orgc', None) event['Event'].pop('RelatedEvent', None) event['Event'].pop('publish_timestamp', None) if event['Event'].get('Attribute'): for a in event['Event'].get('Attribute'): a.pop('uuid', None) a.pop('event_id', None) a.pop('id', None) a.pop('timestamp', None) return event['Event'].pop('id', None) def new_event(self): event = self.misp.new_event(0, 1, 0, "This is a test") event_id = self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'0', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [], u'proposal_email_lock': False, u'threat_level_id': u'1'}}, self.assertEqual(event, to_check, 'Failed at creating a new Event') return int(event_id) def add_hashes(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at adding hashes') def publish(self, eventid): r = self.misp.get_event(eventid) event = r.json() event = self.misp.publish(event) self._clean_event(event) to_check = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} self.assertEqual(event, to_check, 'Failed at publishing event') def delete(self, eventid): event = self.misp.delete_event(eventid) print event.json() def delete_attr(self, attrid): event = self.misp.delete_attribute(attrid) print event.json() def get(self, eventid): event = self.misp.get_event(eventid) print event.json() def add(self): event = {u'Event': {u'info': u'This is a test', u'locked': False, u'attribute_count': u'3', u'analysis': u'0', u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'Attribute': [ {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'}, {u'category': u'Payload installation', u'comment': u'Fanny modules', u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}], u'proposal_email_lock': False, u'threat_level_id': u'1'}} event = self.misp.add_event(event) print event.json() def test_create_event(self): eventid = self.new_event() time.sleep(1) self.delete(eventid) def test_get_event(self): eventid = self.new_event() time.sleep(1) self.get(eventid) time.sleep(1) self.delete(eventid) def test_add_event(self): self.add() time.sleep(1) self.delete(1) def test_del_attr(self): eventid = self.new_event() time.sleep(1) self.delete_attr(1) time.sleep(1) self.delete(eventid)
import numpy import tools import date_tools import bokeh_tools import time if __name__ == '__main__': parser = argparse.ArgumentParser(description='Show the evolution of trend of tags.') parser.add_argument("-d", "--days", type=int, required=True, help='') parser.add_argument("-s", "--begindate", required=True, help='format yyyy-mm-dd') parser.add_argument("-e", "--enddate", required=True, help='format yyyy-mm-dd') args = parser.parse_args() misp = PyMISP(misp_url, misp_key, misp_verifycert) result = misp.search(date_from=args.begindate, date_to=args.enddate, metadata=False) # Getting data if 'response' in result: events = tools.eventsListBuildFromArray(result) NbTags = [] dates = [] enddate = date_tools.toDatetime(args.enddate) begindate = date_tools.toDatetime(args.begindate) for i in range(round(date_tools.days_between(enddate, begindate)/args.days)): begindate = date_tools.getNDaysBefore(enddate, args.days) eventstemp = tools.selectInRange(events, begindate, enddate)