def run(self, conf, args, plugins): server = ExpandedPyMISP(conf['Misp']['url'], conf['Misp']['key'], not args.no_tls) if args.list: # List events events = server.events(pythonify=True) for event in sorted(events, key=lambda x: x.id): print("%i : %s" % (event.id, event.info)) elif args.event is not None: event = server.get_event(args.event, pythonify=True) if args.attr is None and args.type is None: if args.raw: for a in event.attributes: print(a.value) else: print("Event {} : {}".format(event.id, event.info)) print("Tags : {}".format(", ".join( map(lambda x: str(x.name), event.tags)))) print("{} Attributes including:".format( len(event.attributes))) attrs = Counter(map(lambda x: x.type, event.attributes)) attrs_ids = Counter( map(lambda x: x.type, filter(lambda x: x.to_ids, event.attributes))) for type in attrs: print("- %i %s (%i for detection)" % (attrs[type], type, attrs_ids[type])) else: if args.type is not None: # Display all attributes from this type for attr in event.attributes: if attr.type == args.type: if args.raw: print("%s" % attr.value) else: print("{:20}{:10}{:40}{}{}".format( attr.category, attr.type, attr.value, attr.comment, attr.to_ids)) elif args.attr is not None: # search by attribute value for attr in event.attributes: if args.attr in str(attr.value): print("%s\t%s\t%s\t%s\t%s" % (attr.category, attr.type, attr.value, attr.comment, attr.to_ids)) elif args.attr is not None: res = server.search('attributes', value=args.attr) if len(res['Attribute']) == 0: print("Search %s: no results" % args.attr) else: print("Search %s, result founds" % args.attr) for attr in res['Attribute']: print('{} - {}'.format(attr['Event']['id'], attr['Event']['info'])) else: self.parser.print_help()
class VMRayAutomation: def __init__( self, misp_url: str, misp_key: str, verify_cert: bool = False, debug: bool = False, ) -> None: # setup logging log_level = logging.DEBUG if debug else logging.INFO log_format = "%(asctime)s - %(name)s - %(levelname)8s - %(message)s" logging.basicConfig(level=log_level, format=log_format) logging.getLogger("pymisp").setLevel(log_level) self.logger = logging.getLogger(self.__class__.__name__) self.misp_url = misp_url.rstrip("/") self.misp_key = misp_key self.verifycert = verify_cert self.misp = ExpandedPyMISP(misp_url, misp_key, ssl=verify_cert, debug=debug) self.config = {} self.tag_incomplete = 'workflow:state="incomplete"' @staticmethod def _setting_enabled(value: bool) -> bool: if not value: raise VMRayAutomationException( "VMRay import is disabled. " "Please enable `vmray_import` in the MISP settings.") return True @staticmethod def _setting_apikey(value: str) -> str: if not value: raise VMRayAutomationException( "VMRay API key not set. Please set the API key in the MISP settings." ) return value @staticmethod def _setting_url(value: str) -> str: if not value: raise VMRayAutomationException( "VMRay URL not set. Please set the URL in the MISP settings.") if not is_url(value): raise VMRayAutomationException("Not a valid URL") return value @staticmethod def _setting_disabled(value: str) -> bool: return value.lower() in ["no", "false"] @staticmethod def _services_port(value: int) -> bool: if value == 0: return 6666 return value @staticmethod def services_url(value: str) -> bool: if not is_url(value): raise VMRayAutomationException("Services URL is not valid.") return value @property def vmray_settings(self) -> Dict[str, Any]: return { "vmray_import_enabled": self._setting_enabled, "vmray_import_apikey": self._setting_apikey, "vmray_import_url": self._setting_url, "vmray_import_disable_tags": self._setting_disabled, "vmray_import_disable_misp_objects": self._setting_disabled, "vmray_import_ignore_analysis_finished": self._setting_disabled, "services_port": self._services_port, "services_url": self.services_url, } def _get_misp_settings(self) -> List[Dict[str, Any]]: misp_headers = { "Content-Type": "application/json", "Accept": "application/json", "Authorization": self.misp_key, } response = requests.get( f"{self.misp_url}/servers/serverSettings.json", verify=self.verifycert, headers=misp_headers, ) if response.status_code == 200: settings = response.json() if "finalSettings" in settings: return settings["finalSettings"] raise VMRayAutomationException( "Could not get settings from MISP server.") def get_config(self) -> None: self.logger.debug("Loading confing...") # get settings from MISP server settings = self._get_misp_settings() for setting in settings: config_name = setting["setting"].replace("Plugin.Import_", "") if config_name in self.vmray_settings: func = self.vmray_settings[config_name] value = func(setting["value"]) self.config[config_name] = value # set default `vmray_import` settings self.config.setdefault("VTI", "1") self.config.setdefault("IOCs", "1") self.config.setdefault("Artifacts", "0") self.config.setdefault("Analysis Details", "1") self.logger.info("Loading config: Done.") def overwrite_config(self, config: Dict[str, Any]) -> None: self.config.update(config) def _get_sample_id(self, value: str) -> Optional[int]: vmray_sample_id_text = "VMRay Sample ID: " if not value.startswith(vmray_sample_id_text): self.logger.warning("Invalid Sample ID: %s.", value) return None return int(value.replace(vmray_sample_id_text, "")) def _call_vmray_import(self, sample_id: int, event_id: str) -> Dict[str, Any]: url = f"{self.config['services_url']}:{self.config['services_port']}/query" config = {"Sample ID": sample_id} for key, value in self.config.items(): vmray_config_key = key.replace("vmray_import_", "") config[vmray_config_key] = str(value) data = { "module": "vmray_import", "event_id": event_id, "config": config, "data": "", } self.logger.debug("calling `vmray_import`: url=%s, config=%s", url, config) response = requests.post(url, json=data) if response.status_code != 200: raise VMRayAutomationException( f"MISP modules returned status code `{response.status_code}`") json_response = response.json() if "error" in json_response: error = json_response["error"] raise VMRayAutomationException( f"MISP modules returned error: {error}") return json_response def _add_event_attributes(self, event_id: int, attributes: Dict[str, Any]) -> None: event = self.misp.get_event(event_id, pythonify=True) for attr in attributes["Attribute"]: event.add_attribute(**attr) self.misp.update_event(event) def _add_event_objects(self, event_id: int, objects: Dict[str, Any]) -> None: event = self.misp.get_event(event_id, pythonify=True) for obj in objects["Object"]: event.add_object(**obj) if "Tag" in objects: for tag in objects["Tag"]: event.add_tag(tag["name"]) self.misp.update_event(event) def _add_misp_event(self, event_id: int, response: Dict[str, Any]) -> None: if self.config["vmray_import_disable_misp_objects"]: self._add_event_attributes(event_id, response["results"]) else: self._add_event_objects(event_id, response["results"]) def import_incomplete_analyses(self) -> None: self.logger.info("Searching for attributes with tag='%s'", self.tag_incomplete) result = self.misp.search("attributes", tags=self.tag_incomplete) attributes = result["Attribute"] for attr in attributes: event_id = int(attr["event_id"]) self.logger.info("Processing event ID `%d`.", event_id) sample_id = self._get_sample_id(attr["value"]) if not sample_id: continue response = self._call_vmray_import(sample_id, event_id) self._add_misp_event(event_id, response) self.misp.untag(attr["uuid"], self.tag_incomplete)
'http': 'http://127.0.0.1:8123', 'https': 'http://127.0.0.1:8123', } proxies = None if __name__ == '__main__': parser = argparse.ArgumentParser( description='Get an event from a MISP instance.') parser.add_argument("-e", "--event", required=True, help="Event ID to get.") parser.add_argument("-o", "--output", help="Output file") args = parser.parse_args() if args.output is not None and os.path.exists(args.output): print('Output file already exists, abort.') exit(0) misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert, proxies=proxies) event = misp.get_event(args.event, pythonify=True) if args.output: with open(args.output, 'w') as f: f.write(event.to_json()) else: print(event.to_json())
class MISPController(object): ''' MISP Controller ''' def __init__(self, misp_param, debug=False): self.misp_param = misp_param self.debug = debug if misp_param.get('connect_immediately', False): self._connect() else: self.misp = None def import_event(self, event_data): ''' Import event ''' # Check registered same event info print('importing: {}'.format(event_data['title'])) events = self._search_event(eventinfo=event_data['title']) if events != None: for event in events: if event_data['title'] == event['Event']['info']: self._remove_event(event['Event']['id']) event = self._add_event(event_data) if event: print('created event: {}'.format(event.id)) else: print("Import failed.Please retry: {}".format(event_data['title'])) def _connect(self): self.debug_print('URL: {}'.format(self.misp_param['url'])) self.debug_print('authkey: {}'.format(self.misp_param['authkey'])) self.misp = ExpandedPyMISP(self.misp_param['url'], self.misp_param['authkey'], ssl=False, debug=False) self._registered_tags = self.misp.tags() def _check_tag(self, target_tag): if self.misp == None: self._connect() if self._registered_tags == None: self._get_tags() for tag_info in self._registered_tags: if tag_info.get('name', '') == target_tag: return True self.debug_print('new tag: {}'.format(target_tag)) cnt = 0 while True: try: if self.misp == None: self._connect() tmp = MISPTag() tmp.from_dict(name=target_tag) self.misp.add_tag(tmp) self._get_tags() return True except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('add new tag retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return False def _add_event(self, value): for tag in value['event_tags']: self._check_tag(tag) for attribute in value['attributes']: for tag in attribute['tags']: self._check_tag(tag) cnt = 0 while True: try: if self.misp == None: self._connect() tmp = MISPEvent() tmp.from_dict( distribution=self.misp_param['distribution'], threat_level_id=self.misp_param['threat_level_id'], analysis=self.misp_param['analysis'], info=value['title'], date=value['date'], published=False) response = self.misp.add_event(tmp) if response.get('errors'): raise Exception(str(response['errors'])) event = MISPEvent() event.load(response) break except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('add new event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return None self.debug_print(event.id) for tag in value['event_tags']: event.add_tag(tag) for attribute in value['attributes']: attribute_tags = [] event.add_attribute(type=attribute['type'], value=attribute['value'], category=attribute['category'], comment=attribute.get('comment', ''), distribution=self.misp_param['distribution'], Tag=self._create_tags(attribute['tags'])) event.published = True if self._update_event(event): self.debug_print('completed') return event else: self.debug_print('add failed') return None def _get_event(self, id): cnt = 0 while True: try: if self.misp == None: self._connect() self.debug_print('get event start: {}'.format(id)) event = self.misp.get_event(id) if event.get('errors'): raise Exception(str(event['errors'])) self.debug_print('get event end: {}'.format(id)) return event except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('get event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return None def _remove_event(self, id): if id: print('delete event: {}'.format(id)) cnt = 0 while True: try: if self.misp == None: self._connect() response = self.misp.delete_event(id) if response.get('errors'): raise Exception(str(response['errors'])) return True except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('remove event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return False def _search_event(self, **cons): cnt = 0 while True: try: if self.misp == None: self._connect() self.debug_print('search event start') response = self.misp.search_index(**cons) self.debug_print('search event end') results = [] for json in response: if json.get('id', ''): results.append(self._get_event(json['id'])) else: print('no event ID') print(json) return results except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('search event retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: return None def _update_event(self, event): cnt = 0 while True: try: if self.misp == None: self._connect() self.debug_print('event update start: {}'.format(event.id)) response = self.misp.update_event(event) if response.get('errors'): raise Exception(str(response['errors'])) self.debug_print('{} updated'.format(event.id)) return True except: print(traceback.format_exc()) if cnt < int(self.misp_param.get('max_retry_count', '0')): print('retry: {}'.format(cnt)) cnt = cnt + 1 time.sleep(10) else: print('event update failed: {}'.format(event.info)) try: self._remove_event(event.id) except: pass return False def _create_tags(self, values): tags = [] for value in values: if value: tags.append({'name': value}) return tags def debug_print(self, message): if self.debug == False: return # nowstr = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') nowstr = datetime.datetime.now().strftime('%H:%M:%S') print('{}\t{}'.format(nowstr, message))