def __init__(self, url: str, key: str, ssl: Union[bool, str] = True, debug: bool = False, ignore: Union[str, None] = None): if ssl and not os.path.isfile(ssl): ssl = True self.debug = debug self._debug('Connecting to {}.'.format(url)) self.misp = pymisp.PyMISP(url=url, key=key, ssl=ssl) if ignore: self.ignore = ignore.split(',') else: self.ignore = None
def __init__(self, url, key, verify=False, misp_modules=False): """ Initialise the MISP instance :param url: The URL of the MISP instance :param key: The API key to authenticate with :param verify: Shall we verify SSL? Can be bool or location of a signature """ log.debug("Starting MISP") log.debug("%s -- %s", url, key) self.url = url self.key = key self.verify = verify self.mispAPI = pymisp.PyMISP(url, key, ssl=self.verify, debug=False) self.misp_modules = misp_modules
def __init__(self, url: str, key: str, ssl=True, debug: bool = False, ignore=None): # Python <= 3.5 doesn support typing.Union. This method will be removed # when we stop supporting python 3.4 if ssl and not os.path.isfile(ssl): ssl = True self.debug = debug self._debug('Connecting to {}.'.format(url)) self.misp = pymisp.PyMISP(url=url, key=key, ssl=ssl) if ignore: self.ignore = ignore.split(',') else: self.ignore = None
def run(self): """Run analysis. @return: MISP results dict. """ self.url = self.options.get("url", "") self.apikey = self.options.get("apikey", "") maxioc = int(self.options.get("maxioc", 100)) if not self.url or not self.apikey: raise CuckooProcessingError( "Please configure the URL and API key for your MISP instance." ) self.key = "misp" self.iocs = {} with warnings.catch_warnings(): warnings.simplefilter("ignore") import pymisp self.misp = pymisp.PyMISP(self.url, self.apikey, False, "json") iocs = set() iocs.add(self.results.get("target", {}).get("file", {}).get("md5")) for dropped in self.results.get("dropped", []): iocs.add(dropped.get("md5")) iocs.update(self.results.get("network", {}).get("hosts", [])) for block in self.results.get("network", {}).get("domains", []): iocs.add(block.get("ip")) iocs.add(block.get("domain")) # Remove empty entries and turn the collection into a list. iocs = list(iocs.difference((None, ""))) # Acquire all information related to IOCs. for ioc in iocs[:maxioc]: self.search_ioc(ioc) # Sort IOC information by date and return all information. return sorted( self.iocs.values(), key=self._parse_date, reverse=True )
def run(self, results): """Submits results to MISP. @param results: Cuckoo results dict. """ url = self.options.get("url") apikey = self.options.get("apikey") mode = shlex.split(self.options.get("mode") or "") if not url or not apikey: raise CuckooProcessingError( "Please configure the URL and API key for your MISP instance.") with warnings.catch_warnings(): warnings.simplefilter("ignore") import pymisp self.misp = pymisp.PyMISP(url, apikey, False, "json") event = self.misp.new_event( distribution=pymisp.Distribution.all_communities.value, threat_level_id=pymisp.ThreatLevel.undefined.value, analysis=pymisp.Analysis.completed.value, info="Cuckoo Sandbox analysis #%d" % self.task["id"], ) if results.get("target", {}).get("category") == "file": self.misp.upload_sample( filename=os.path.basename(self.task["target"]), filepath_or_bytes=self.task["target"], event_id=event["Event"]["id"], category="External analysis", ) self.signature(results, event) if "hashes" in mode: self.sample_hashes(results, event) if "url" in mode: self.all_urls(results, event) if "ipaddr" in mode: self.domain_ipaddr(results, event) self.family(results, event)
def run(self, results): """Submits results to MISP. @param results: Cuckoo results dict. """ url = self.options.get("url") apikey = self.options.get("apikey") mode = shlex.split(self.options.get("mode") or "") if not url or not apikey: raise CuckooProcessingError( "Please configure the URL and API key for your MISP instance." ) self.misp = pymisp.PyMISP(url, apikey, False, "json") event = self.misp.new_event( distribution=self.misp.distributions.all_communities, threat_level_id=self.misp.threat_level.undefined, analysis=self.misp.analysis.completed, info="Cuckoo Sandbox analysis #%d" % self.task["id"], ) if results.get("target", {}).get("category") == "file": self.misp.upload_sample( filename=os.path.basename(self.task["target"]), filepath=self.task["target"], event_id=event["Event"]["id"], category="External analysis", ) if "hashes" in mode: self.sample_hashes(results, event) if "maldoc" in mode: self.maldoc_network(results, event) if "url" in mode: self.all_urls(results, event) if "ipaddr" in mode: self.domain_ipaddr(results, event)
def run(self): api_key = secrets.get_secret(self.api_key_name) if not api_key: raise AnalyzerRunException( f"no MISP API key retrieved with name: {self.api_key_name}" ) if not self.url_name: raise AnalyzerRunException( f"no MISP URL retrieved, key value: {self.url_key_name}" ) misp_instance = pymisp.PyMISP( url=self.url_name, key=api_key, ssl=self.ssl_check, debug=self.debug, timeout=5, ) # we check only for events not older than 90 days and max 50 results now = datetime.datetime.now() date_from = now - datetime.timedelta(days=90) params = { # even if docs say to use "values",... # .. at the moment it works correctly only with "value" "value": self.observable_name, "type_attribute": [self.observable_classification], "date_from": date_from.strftime("%Y-%m-%d %H:%M:%S"), "limit": 50, "enforce_warninglist": True, } if self.observable_classification == "hash": params["type_attribute"] = ["md5", "sha1", "sha256"] result_search = misp_instance.search(**params) if isinstance(result_search, dict): errors = result_search.get("errors", []) if errors: raise AnalyzerRunException(errors) return {"result_search": result_search, "instance_url": self.url_name}
def __init__(self, conf, logger, targs=None, responders=[]): Thread.__init__(self) self.args = targs self.conf = conf self.lh = logger self.responders = responders self.feyerex = re.compile(r'^\s*([\w\-\"]*):(.*)') self.esenrichment = conf["esenrichment"] if self.esenrichment: self.ese = ESEnrichment(self.lh, self.conf["esenrichment_server"]) if self.conf["email_alerts"]: from smtplib import SMTP from email.mime.text import MIMEText self.smtp = SMTP self.mimetext = MIMEText if self.conf['misp_enabled']: self.misp = pymisp.PyMISP(self.conf['mispurl'], self.conf['mispkey']) else: self.misp = None if conf["elasticsearch"]: if "phishing" in conf["elasticsearch_config"]: self.esphish = ES(conf["elasticsearch_config"]["phishing"], logger) if "fireeye" in conf["elasticsearch_config"]: self.esfire = ES(conf["elasticsearch_config"]["fireeye"], logger) if conf["activedirectory-enrichment"]: self.ad = ADEnrichment( conf["activedirectory-enrichment-configuration"]) class RootCAAdapter(requests.adapters.HTTPAdapter): def cert_verify(self, conn, url, verify, cert): cert_file = conf['certs'][urlparse(url).hostname] super(RootCAAdapter, self).cert_verify(conn=conn, url=url, verify=cert_file, cert=cert)
for j in normalize(i): response = misp.search_all(j) if response['response']: for resp in response['response']: hashesDict[j] = { resp['Event']['id']: [ resp['Event']['Org']['name'], resp['Event']['Orgc']['name'], resp['Event']['date'], resp['Event']['timestamp'], resp['Event']['publish_timestamp'], resp['Event']['info'], j ] } return hashesDict if __name__ == '__main__': misp = pymisp.PyMISP(misp_url, misp_key, misp_verifycert, 'json') #d=queryUrl(misp, sourceFile='submittedUrl_list.txt') #with open('result.json', 'w') as fp: # json.dump(d, fp) #d=queryhashes(misp, "md5", sourceFile='dataFull.json') #with open('result_md5.json', 'w') as fp: # json.dump(d, fp) d = queryhashes(misp, "sha1", sourceFile='dataFull.json') with open('result_sha1.json', 'w') as fp: json.dump(d, fp) d = queryhashes(misp, "sha256", sourceFile='dataFull.json') with open('result_sha256.json', 'w') as fp: json.dump(d, fp)
def initialize(self): self._misp = pymisp.PyMISP(self._url, self._key, False) ### false self.load_extra_info()
def create_events(pulse_or_list, author=False, server=False, key=False, misp=False, distribution=0, threat_level=4, analysis=2, publish=False, tlp=True, discover_tags=False, to_ids=False, author_tag=False, bulk_tag=None, dedup_titles=False, stop_on_error=False): """ Parse a Pulse or a list of Pulses and add it/them to MISP if server and key are present :param pulse_or_list: a Pulse or list of Pulses as returned by `get_pulses` :param author: Prepend the author to the Pulse name :type author: Boolean :param server: MISP server URL :param key: MISP API key :param misp: MISP connection object :type misp: :class:`pymisp.PyMISP` :param distribution: distribution of the MISP event (0-4) :param threat_level: threat level of the MISP object (1-4) :param analysis: analysis stae of the MISP object (0-2) :param publish: Is the MISP event should be published? :type publish: Boolean :param tlp: Add TLP level tag to event :type tlp: Boolean :param discover_tags: discover MISP tags from Pulse tags :type discover_tags: Boolean :param to_ids: Flag pulse attributes as being sent to an IDS :type to_ids: Boolean :param author_tag: Add the pulse author as an event tag :type author_tag: Boolean :param bulk_tag: A tag that will be added to all events for categorization (e.g. OTX) :type bulk_tag: String :param dedup_titles: Search MISP for an existing event title and update it, rather than create a new one :type dedup_titles: Boolean :return: a dict or a list of dict with the selected attributes """ if not misp and (server and key): log.debug("Connection to MISP instance: {}".format(server)) try: misp = pymisp.PyMISP(server, key, ssl=False, out_type='json') except pymisp.PyMISPError as ex: raise ImportException("Cannot connect to MISP instance: {}".format(ex.message)) except Exception as ex: raise ImportException("Cannot connect to MISP instance, unknown exception: {}".format(ex.message)) if discover_tags: def get_tag_name(complete): parts = complete.split('=') if not len(parts): return complete last = parts[-1] if not len(last): return complete if last[0] == '"': last = last[1:] if last[-1] == '"': last = last[:-1] return last.lower() raw_tags = misp.get_all_tags() tags = dict() for tag in raw_tags['Tag']: tags[get_tag_name(tag['name'])] = tag['name'] misp.discovered_tags = tags if isinstance(pulse_or_list, (list, tuple)) or inspect.isgenerator(pulse_or_list): misp_events = [] for pulse in pulse_or_list: try: misp_event = create_events(pulse, author=author, server=server, key=key, misp=misp, distribution=distribution, threat_level=threat_level, analysis=analysis, publish=publish, tlp=tlp, to_ids=to_ids, author_tag=author_tag, bulk_tag=bulk_tag, dedup_titles=dedup_titles, stop_on_error=stop_on_error) misp_events.append(misp_event) except Exception as ex: if stop_on_error: raise name = '' if pulse and 'name' in pulse: name = pulse['name'] log.error("Cannot import pulse {}: {}".format(name, ex)) return misp_events pulse = pulse_or_list if author: event_name = pulse['author_name'] + ' | ' + pulse['name'] else: event_name = pulse['name'] try: dt = date_parser.parse(pulse['created']) except (ValueError, OverflowError): log.error("Cannot parse Pulse 'created' date.") dt = datetime.utcnow() event_date = dt.strftime('%Y-%m-%d') log.info("## {name} - {date}".format(name=event_name, date=event_date)) result_event = { 'name': event_name, 'date': event_date, 'tags': list(), 'attributes': { 'hashes': { 'md5': list(), 'sha1': list(), 'sha256': list(), 'imphash': list(), 'pehash': list() }, 'hostnames': list(), 'domains': list(), 'urls': list(), 'ips': list(), 'emails': list(), 'mutexes': list(), 'references': list(), 'cves': list(), 'filenames': list(), 'yara': list() }, } if misp: if not dedup_titles: event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish) else: event = '' # Check if username is added to title # Build the title if author: event_name = pulse['author_name'] + ' | ' + pulse['name'] else: event_name = pulse['name'] # Search MISP for the title result = misp.search_index(eventinfo=event_name) if 'message' in result: if result['message'] == "No matches.": event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish) else: for evt in result['response']: # If it exists, set 'event' to the event if evt['info'] == event_name: if 'SharingGroup' in evt: del evt['SharingGroup'] # This deletes the SharingGroup from the list, thx SparkyNZL event = {'Event': evt} break if event == '': # Event not found, even though search results were returned # Build new event event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish) time.sleep(0.2) if tlp: tag = None if 'TLP' in pulse: tag = "tlp:{}".format(pulse['TLP']) elif 'tlp' in pulse: tag = "tlp:{}".format(pulse['tlp']) if tag is not None: log.info("\t - Adding tag: {}".format(tag)) tag_event(misp, event, tag) result_event['tags'].append(tag) if author_tag: tag_event(misp, event, pulse['author_name']) if bulk_tag is not None: tag_event(misp, event, bulk_tag) if misp and hasattr(misp, 'discovered_tags') and 'tags' in pulse: for pulse_tag in pulse['tags']: if pulse_tag.lower() in misp.discovered_tags: tag = misp.discovered_tags[pulse_tag.lower()] log.info("\t - Adding tag: {}".format(tag)) tag_event(misp, event, tag) result_event['tags'].append(tag) if 'references' in pulse: for reference in pulse['references']: if reference: log.info("\t - Adding external analysis link: {}".format(reference)) if misp: misp.add_named_attribute(event, 'link', reference, category='External analysis') result_event['attributes']['references'].append(reference) if misp and 'description' in pulse and isinstance(pulse['description'], six.text_type) and pulse['description']: log.info("\t - Adding external analysis comment") misp.add_named_attribute(event, 'comment', pulse['description'], category='External analysis') for ind in pulse['indicators']: ind_type = ind['type'] ind_val = ind['indicator'] ind_kwargs = {'to_ids': to_ids} if 'description' in ind and isinstance(ind['description'], six.text_type) and ind['description']: ind_kwargs['comment'] = ind['description'] if ind_type == 'FileHash-SHA256': log.info("\t - Adding SHA256 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, sha256=ind_val, **ind_kwargs) result_event['attributes']['hashes']['sha256'].append(ind_val) elif ind_type == 'FileHash-SHA1': log.info("\t - Adding SHA1 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, sha1=ind_val, **ind_kwargs) result_event['attributes']['hashes']['sha1'].append(ind_val) elif ind_type == 'FileHash-MD5': log.info("\t - Adding MD5 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, md5=ind_val, **ind_kwargs) result_event['attributes']['hashes']['md5'].append(ind_val) elif ind_type == 'URI' or ind_type == 'URL': log.info("\t - Adding URL: {}".format(ind_val)) if misp: misp.add_url(event, ind_val, **ind_kwargs) result_event['attributes']['urls'].append(ind_val) elif ind_type == 'domain': log.info("\t - Adding domain: {}".format(ind_val)) if misp: misp.add_domain(event, ind_val, **ind_kwargs) result_event['attributes']['domains'].append(ind_val) elif ind_type == 'hostname': log.info("\t - Adding hostname: {}".format(ind_val)) if misp: misp.add_hostname(event, ind_val, **ind_kwargs) result_event['attributes']['hostnames'].append(ind_val) elif ind_type == 'IPv4' or ind_type == 'IPv6': log.info("\t - Adding ip: {}".format(ind_val)) if misp: misp.add_ipdst(event, ind_val, **ind_kwargs) result_event['attributes']['ips'].append(ind_val) elif ind_type == 'email': log.info("\t - Adding email: {}".format(ind_val)) if misp: misp.add_email_dst(event, ind_val, **ind_kwargs) result_event['attributes']['emails'].append(ind_val) elif ind_type == 'Mutex': log.info("\t - Adding mutex: {}".format(ind_val)) if misp: misp.add_mutex(event, ind_val, **ind_kwargs) result_event['attributes']['mutexes'].append(ind_val) elif ind_type == 'CVE': log.info("\t - Adding CVE: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'vulnerability', ind_val, category='External analysis', **ind_kwargs) result_event['attributes']['cves'].append(ind_val) elif ind_type == 'FileHash-IMPHASH': log.info("\t - Adding IMPHASH hash: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'imphash', ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['hashes']['imphash'].append(ind_val) elif ind_type == 'FileHash-PEHASH': log.info("\t - Adding PEHASH hash: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'pehash', ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['hashes']['pehash'].append(ind_val) elif ind_type == 'FilePath': log.info("\t - Adding filename: {}".format(ind_val)) if misp: misp.add_filename(event, ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['filenames'].append(ind_val) elif ind_type == 'YARA': ind_title = ind.get('title', ind_val) ind_desc = ind.get('description', '') if ind_title == '': ind_title = ind_val if not ind_desc == '': ind_kwargs['comment'] = ind_desc else: ind_kwargs['comment'] = "{} {}".format(ind_title, ind_desc) ind_val = ind.get('content', None) if ind_val is None or ind_val == "": log.warning("YARA indicator is empty: %s" % ind_title) continue log.info("\t - Adding YARA rule: {}".format(ind_title)) if misp: misp.add_yara(event, ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['yara'].append({'title': ind_title, 'content': ind_val}) else: log.warning("Unsupported indicator type: %s" % ind_type) if misp : event['Event']['published'] = False misp.publish(event) return result_event
value = json.loads(value) for i in range(1, len(subkeys)): if subkeys[i] in value: value = value[subkeys[i]] else: break except ValueError as e: pass if value is None: value = _default if not _default is None else '' return value try: misp = pymisp.PyMISP(url=config['MISP']['proto'] + '://' + config['MISP']['host'] + ':' + config['MISP']['port'], key=config['MISP']['token'], ssl=False) resp = misp.search(controller=args.controller, type_attribute=args.type, org=args.org, last=args.last, date_from=args.date_from, date_to=args.date_to, tags=args.tags, not_tags=args.not_tags, eventid=args.eventid) idxcol = args.idx_col events = {} if args.controller == 'attributes':
def run(self, results): """Run analysis. @return: MISP results dict. """ url = self.options.get("url", "") apikey = self.options.get("apikey", "") if not url or not apikey: log.error("MISP URL or API key not configured.") return with warnings.catch_warnings(): warnings.simplefilter("ignore") import pymisp self.misp = pymisp.PyMISP(url, apikey, False, "json") self.threads = self.options.get("threads", "") if not self.threads: self.threads = 5 whitelist = list() self.iocs = deque() self.misper = dict() self.misp_full_report = dict() self.lock = threading.Lock() try: # load whitelist if exists if os.path.exists(os.path.join(CUCKOO_ROOT, "conf", "misp.conf")): whitelist = Config("misp").whitelist.whitelist if whitelist: whitelist = [ioc.strip() for ioc in whitelist.split(",")] if self.options.get("upload_iocs", False) and results.get("malscore", 0) >= self.options.get("min_malscore", 0): distribution = int(self.options.get("distribution", 0)) threat_level_id = int(self.options.get("threat_level_id", 4)) analysis = int(self.options.get("analysis", 0)) tag = self.options.get("tag") or "CAPEv2" info = self.options.get("title", "") upload_sample = self.options.get("upload_sample") malfamily = "" filtered_iocs = deque() if results.get("malfamily", ""): malfamily = results["malfamily"] event = self.misp.new_event( distribution=distribution, threat_level_id=threat_level_id, analysis=analysis, info="{} {} - {}".format(info, malfamily, results.get('info', {}).get('id')) ) # Add a specific tag to flag Cuckoo's event if tag: mispresult = self.misp.tag(event["Event"]["uuid"], tag) if mispresult.has_key("message"): log.debug("tag event: %s" % mispresult["message"]) #ToDo? #self.signature(results, event) self.sample_hashes(results, event) self.all_network(results, event, whitelist) self.dropped_files(results, event, whitelist) #ToDo add? upload sample """ if upload_sample: target = results.get("target", {}) f = target.get("file", {}) if target.get("category") == "file" and f: self.misp.upload_sample( filename=os.path.basename(f["name"]), filepath_or_bytes=f["path"], event_id=event["Event"]["id"], category="Payload delivery", comment="Sample run", ) """ self.misper.setdefault("iocs", list()) #if results.get("target", {}).get("url", "") and results["target"]["url"] not in whitelist: # filtered_iocs.append(results["target"]["url"]) # #parsed = urlsplit(results["target"]["url"]) # ToDo migth be outdated! #if self.options.get("ids_files", False) and "suricata" in results.keys(): # for surifile in results["suricata"]["files"]: # if "file_info" in surifile.keys(): # self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]}) # self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]}) # self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]}) if self.options.get("mutexes", False) and "behavior" in results and "summary" in results["behavior"]: if "mutexes" in results.get("behavior", {}).get("summary", {}): for mutex in results["behavior"]["summary"]["mutexes"]: if mutex not in whitelist: self.misp.add_mutex(event, mutex) if self.options.get("registry", False) and "behavior" in results and "summary" in results["behavior"]: if "read_keys" in results["behavior"].get("summary", {}): for regkey in results["behavior"]["summary"]["read_keys"]: self.misp.add_regkey(event, regkey) except Exception as e: log.error("Failed to generate JSON report: %s" % e, exc_info=True)
config["zmq"]["host"], config["zmq"]["port"] )) # Set the option to subscribe socket.setsockopt_string(zmq.SUBSCRIBE, '') # Get MISP-Modules mod list log.info("Connecting to Modules server at http://%s:%s", config["misp"]["modules"]["host"], config["misp"]["modules"]["port"]) modulesURL = "http://{}:{}".format(config["misp"]["modules"]["host"], config["misp"]["modules"]["port"]) modules = requests.get(urljoin(modulesURL, "modules")).json() configModules = config["misp"]["modules"]["run-modules"] api = pymisp.PyMISP(config["misp"]["url"], config["misp"]["apikey"]) while True: # Wait for something to come in on the ZMQ socket message = socket.recv().decode("utf-8")[10:] log.info("Recieved a message!") log.debug("Processing...") # Load the message JSON msg = json.loads(message) log.debug(msg) # Load it as a misp object ev = pymisp.mispevent.MISPEvent()
def getMISPInfo(hash): """ Retrieves information from a MISP instance :param hash: hash value :return info: info object """ info = {'misp_available': False, 'misp_events': ''} requests.packages.urllib3.disable_warnings() # I don't care # Check if any auth key is set key_set = False for m in MISP_AUTH_KEYS: if m != '' and m != '-': key_set = True if not key_set or 'pymisp' in deactivated_features: return info # Loop through MISP instances misp_info = [] misp_events = [] for c, m_url in enumerate(MISP_URLS, start=0): # Get the corresponding auth key m_auth_key = MISP_AUTH_KEYS[c] if args.debug: print("[D] Querying MISP: %s" % m_url) try: # Preparing API request misp = pymisp.PyMISP(m_url, m_auth_key, args.verifycert, debug=args.debug, proxies={},cert=None,auth=None,tool='Munin : Online hash checker') if args.debug: print("[D] Query: values=%s" % hash) result = misp.search('attributes', type_attribute=fetchHash(hash)[1] ,value=hash) # Processing the result if result['Attribute']: events_added = list() if args.debug: print("[D] Dump Attribute : "+json.dumps(result['Attribute'], indent=2)) for r in result['Attribute']: # Check for duplicates if r['event_id'] in events_added: continue # Try to get info on the events event_info = "" misp_events.append('MISP%d:%s' % (c+1, r['event_id'])) e_result = misp.search('events', eventid=r['event_id']) if e_result: event_info = e_result[0]['Event']['info'] # too much # if args.debug: # print(json.dumps(e_result['response'], indent=2)) # Create MISP info object misp_info.append({ 'misp_nr': c+1, 'event_info': event_info, 'event_id': r['event_id'], 'comment': r['comment'], 'url': '%s/events/view/%s' % (m_url, r['event_id']) }) events_added.append(r['event_id']) else: info['misp_available'] = False except Exception as e: if args.debug: traceback.print_exc() info['misp_info'] = misp_info info['misp_events'] = ", ".join(misp_events) if len(misp_events) > 0: info['misp_available'] = True return info
def create_events(pulse_or_list, author=False, server=False, key=False, misp=False, distribution=0, threat_level=4, analysis=2, publish=True, tlp=True, discover_tags=False, to_ids=False): """ Parse a Pulse or a list of Pulses and add it/them to MISP if server and key are present :param pulse_or_list: a Pulse or list of Pulses as returned by `get_pulses` :param author: Prepend the author to the Pulse name :type author: Boolean :param server: MISP server URL :param key: MISP API key :param misp: MISP connection object :type misp: :class:`pymisp.PyMISP` :param distribution: distribution of the MISP event (0-4) :param threat_level: threat level of the MISP object (1-4) :param analysis: analysis stae of the MISP object (0-2) :param publish: Is the MISP event should be published? :type publish: Boolean :param tlp: Add TLP level tag to event :type tlp: Boolean :param discover_tags: discover MISP tags from Pulse tags :type discover_tags: Boolean :return: a dict or a list of dict with the selected attributes """ if not misp and (server and key): log.debug("Connection to MISP instance: {}".format(server)) try: misp = pymisp.PyMISP(server, key, ssl=False, out_type='json', debug=False) except pymisp.PyMISPError as ex: raise ImportException("Cannot connect to MISP instance: {}".format( ex.message)) except Exception as ex: raise ImportException( "Cannot connect to MISP instance, unknown exception: {}". format(ex.message)) if discover_tags: def get_tag_name(complete): parts = complete.split('=') if not len(parts): return complete last = parts[-1] if last[0] == '"': last = last[1:] if last[-1] == '"': last = last[:-1] return last.lower() raw_tags = misp.get_all_tags() tags = dict() for tag in raw_tags['Tag']: tags[get_tag_name(tag['name'])] = tag['name'] misp.discovered_tags = tags if isinstance(pulse_or_list, (list, tuple)) or inspect.isgenerator(pulse_or_list): return [ create_events(pulse, author=author, server=server, key=key, misp=misp, distribution=distribution, threat_level=threat_level, analysis=analysis, publish=publish, tlp=tlp, to_ids=to_ids) for pulse in pulse_or_list ] pulse = pulse_or_list if author: event_name = pulse['author_name'] + ' | ' + pulse['name'] else: event_name = pulse['name'] try: dt = date_parser.parse(pulse['modified']) except (ValueError, OverflowError): log.error("Cannot parse Pulse 'modified' date.") dt = datetime.utcnow() event_date = dt.strftime('%Y-%m-%d') log.info("## {name} - {date}".format(name=event_name, date=event_date)) result_event = { 'name': event_name, 'date': event_date, 'tags': list(), 'attributes': { 'hashes': { 'md5': list(), 'sha1': list(), 'sha256': list(), 'imphash': list(), 'pehash': list() }, 'hostnames': list(), 'domains': list(), 'urls': list(), 'ips': list(), 'emails': list(), 'mutexes': list(), 'references': list(), 'cves': list() }, } if misp: event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish) time.sleep(0.2) if tlp and 'TLP' in pulse: tag = "tlp:{}".format(pulse['TLP']) log.info("\t - Adding tag: {}".format(tag)) misp.add_tag(event, tag) result_event['tags'].append(tag) if misp and hasattr(misp, 'discovered_tags') and 'tags' in pulse: for pulse_tag in pulse['tags']: if pulse_tag.lower() in misp.discovered_tags: tag = misp.discovered_tags[pulse_tag.lower()] log.info("\t - Adding tag: {}".format(tag)) misp.add_tag(event, tag) result_event['tags'].append(tag) if 'references' in pulse: for reference in pulse['references']: log.info( "\t - Adding external analysis link: {}".format(reference)) if misp: misp.add_named_attribute(event, 'link', reference, category='External analysis') result_event['attributes']['references'].append(reference) if misp and 'description' in pulse and isinstance( pulse['description'], six.text_type) and pulse['description']: log.info("\t - Adding external analysis comment") misp.add_named_attribute(event, 'comment', pulse['description'], category='External analysis') for ind in pulse['indicators']: ind_type = ind['type'] ind_val = ind['indicator'] ind_kwargs = {'to_ids': to_ids} if 'description' in ind and isinstance( ind['description'], six.text_type) and ind['description']: ind_kwargs['comment'] = ind['description'] if ind_type == 'FileHash-SHA256': log.info("\t - Adding SH256 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, sha256=ind_val, **ind_kwargs) result_event['attributes']['hashes']['sha256'].append(ind_val) elif ind_type == 'FileHash-SHA1': log.info("\t - Adding SHA1 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, sha1=ind_val, **ind_kwargs) result_event['attributes']['hashes']['sha1'].append(ind_val) elif ind_type == 'FileHash-MD5': log.info("\t - Adding MD5 hash: {}".format(ind_val)) if misp: misp.add_hashes(event, md5=ind_val, **ind_kwargs) result_event['attributes']['hashes']['md5'].append(ind_val) elif ind_type == 'URI' or ind_type == 'URL': log.info("\t - Adding URL: {}".format(ind_val)) if misp: misp.add_url(event, ind_val, **ind_kwargs) result_event['attributes']['urls'].append(ind_val) elif ind_type == 'domain': log.info("\t - Adding domain: {}".format(ind_val)) if misp: misp.add_domain(event, ind_val, **ind_kwargs) result_event['attributes']['domains'].append(ind_val) elif ind_type == 'hostname': log.info("\t - Adding hostname: {}".format(ind_val)) if misp: misp.add_hostname(event, ind_val, **ind_kwargs) result_event['attributes']['hostnames'].append(ind_val) elif ind_type == 'IPv4' or ind_type == 'IPv6': log.info("\t - Adding ip: {}".format(ind_val)) if misp: misp.add_ipdst(event, ind_val, **ind_kwargs) result_event['attributes']['ips'].append(ind_val) elif ind_type == 'email': log.info("\t - Adding email: {}".format(ind_val)) if misp: misp.add_email_dst(event, ind_val, **ind_kwargs) result_event['attributes']['emails'].append(ind_val) elif ind_type == 'Mutex': log.info("\t - Adding mutex: {}".format(ind_val)) if misp: misp.add_mutex(event, ind_val, **ind_kwargs) result_event['attributes']['mutexes'].append(ind_val) elif ind_type == 'CVE': log.info("\t - Adding CVE: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'vulnerability', ind_val, category='External analysis', **ind_kwargs) result_event['attributes']['cves'].append(ind_val) elif ind_type == 'FileHash-IMPHASH': log.info("\t - Adding IMPHASH hash: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'imphash', ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['hashes']['imphash'].append(ind_val) elif ind_type == 'FileHash-PEHASH': log.info("\t - Adding PEHASH hash: {}".format(ind_val)) if misp: misp.add_named_attribute(event, 'pehash', ind_val, category='Artifacts dropped', **ind_kwargs) result_event['attributes']['hashes']['pehash'].append(ind_val) else: log.warning("Unsupported indicator type: %s" % ind_type) if misp and publish: event['Event']['published'] = False misp.publish(event) return result_event
Submit a list of MISP objects to a MISP event :misp: PyMISP API object for interfacing with MISP :misp_event: MISPEvent object :misp_objects: List of MISPObject objects. Must be a list ''' # go through round one and only add MISP objects for misp_object in misp_objects: template_id = misp.get_object_template_id(misp_object.template_uuid) misp.add_object(misp_event.id, template_id, misp_object) # go through round two and add all the object references for each object for misp_object in misp_objects: for reference in misp_object.ObjectReference: misp.add_object_reference(reference) if __name__ == "__main__": try: args = build_cli() config = build_config(args.config) # change the 'ssl' value if you want to verify your MISP's SSL instance misp = pymisp.PyMISP(url=config["misp"]["url"], key=config["misp"]["key"], ssl=False) # finally, let's start checking VT and converting the reports main(misp, config, args) except KeyboardInterrupt: print("Bye Felicia") except pymisp.exceptions.InvalidMISPObject as err: logging.error(err)
def run(self, results): """Submit results to MISP. @param results: Cuckoo results dict. """ url = self.options.get("url") apikey = self.options.get("apikey") mode = shlex.split(self.options.get("mode") or "") score = results.get("info", {}).get("score", 0) upload_sample = self.options.get("upload_sample") if results.get("target", {}).get("category") == "file": f = results.get("target", {}).get("file", {}) hash_safelisted = is_safelisted_misphash(f["md5"]) or \ is_safelisted_misphash(f["sha1"]) or \ is_safelisted_misphash(f["sha256"]) if hash_safelisted: return if score < self.options.get("min_malscore", 0): return if not url or not apikey: raise CuckooProcessingError( "Please configure the URL and API key for your MISP " "instance." ) with warnings.catch_warnings(): warnings.simplefilter("ignore") import pymisp self.misp = pymisp.PyMISP(url, apikey, False, "json") # Get default settings for a new event distribution = self.options.get("distribution") or 0 threat_level = self.options.get("threat_level") or 4 analysis = self.options.get("analysis") or 0 tag = self.options.get("tag") or "Cuckoo" event = self.misp.new_event( distribution=distribution, threat_level_id=threat_level, analysis=analysis, info="Cuckoo Sandbox analysis #%d" % self.task["id"] ) # Add a specific tag to flag Cuckoo's event if tag: mispresult = self.misp.tag(event["Event"]["uuid"], tag) if mispresult.has_key("message"): log.debug("tag event: %s" % mispresult["message"]) if upload_sample: target = results.get("target", {}) if target.get("category") == "file" and target.get("file"): self.misp.upload_sample( filename=os.path.basename(self.task["target"]), filepath_or_bytes=self.task["target"], event_id=event["Event"]["id"], category="External analysis", ) self.signature(results, event) if "hashes" in mode: self.sample_hashes(results, event) if "url" in mode: self.all_urls(results, event) if "ipaddr" in mode: self.domain_ipaddr(results, event) self.family(results, event)
logger.setLevel(logging.INFO) logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', format=pymisp.FORMAT) events_hashes = [] if len(keys.misp_url) == 0 or len(keys.misp_key) == 0: print("Error: missing info about MISP database, please check keys.py file !") exit(-1) if "-h" in sys.argv or sys.argv[-1][0] == '-' or len(sys.argv) < 2 or ".yara" in sys.argv: print("Command : yara_export.py [-h : show help] [-json : add a json dump along with yara dump] <yara output filename>") exit(0) if ".yara" in sys.argv[-1]: sys.argv[-1] = sys.argv[-1].replace(".yara","") misp = pymisp.PyMISP(keys.misp_url,keys.misp_key,False,False) events = misp.search(controller='events') for event in events: # For each event for event_content in event.values(): # Get the content of the key "Event" hashes_types = {} event_dict = {"event_id": event_content["id"]} event_dict.update({"info": event_content["info"]}) for attribute in event_content["Attribute"]: # For each attribute from a event if attribute["type"] == "md5" or attribute["type"] == "sha1" or attribute["type"] == "sha256" : if attribute["type"] not in hashes_types : hashes_types.update({ attribute["type"]: [ attribute["value"] ] } ) else : if attribute["value"] not in hashes_types[attribute["type"]] : # If hash already in list, ignore hashes_types[attribute["type"]].append(attribute["value"])
print("Trying to use env variables...") if "MISP_URL" in os.environ: misp_url = os.environ["MISP_URL"] else: print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.") misp_url = "UNKNOWN" if "MISP_API" in os.environ: misp_api = os.environ["MISP_API"] else: print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.") misp_api = "UNKNOWN" CONFIG = {"misp": {"url": misp_url, "api": misp_api}} MISP = pymisp.PyMISP(CONFIG["misp"]["url"], CONFIG["misp"]["api"], ssl=CONFIG["misp"].get("verifySSL", True)) def post_stix(manager, content_block, collection_ids, service_id): ''' Callback function for when our taxii server gets new data Will convert it to a MISPEvent and push to the server ''' # Load the package log.info("Posting STIX...") package = pymisp.tools.stix.load_stix(content_block.content) log.info("STIX loaded succesfully.") values = [x.value for x in package.attributes] log.info("Extracted %s", values)
def handler(q: Union[bool, str] = False) -> Union[bool, Dict[str, Any]]: """ Implement interface. :param bool|str q: the input received :rtype: bool|dict[str, any] """ if q is False: return False request = json.loads(q) config = request.get("config", {}) # Load the client to connect to VMware NSX ATA (hard-fail) try: analysis_url = config.get("analysis_url") login_params = { "key": config["analysis_key"], "api_token": config["analysis_api_token"], } # If 'analysis_url' is specified we are connecting on-premise if analysis_url: analysis_clients = { DEFAULT_ENDPOINT: nsx_defender.AnalysisClient( api_url=analysis_url, login_params=login_params, verify_ssl=bool(config.get("analysis_verify_ssl", True)), ) } logger.info( "Connected NSX AnalysisClient to on-premise infrastructure") else: analysis_clients = { data_center: nsx_defender.AnalysisClient( api_url=tau_clients. NSX_DEFENDER_ANALYSIS_URLS[data_center], login_params=login_params, verify_ssl=bool(config.get("analysis_verify_ssl", True)), ) for data_center in [ tau_clients.NSX_DEFENDER_DC_WESTUS, tau_clients.NSX_DEFENDER_DC_NLEMEA, ] } logger.info( "Connected NSX AnalysisClient to hosted infrastructure") except KeyError as ke: logger.error("Integration with VMware NSX ATA failed to connect: %s", str(ke)) return {"error": "Error connecting to VMware NSX ATA: {}".format(ke)} # Load the client to connect to MISP (soft-fail) try: misp_client = pymisp.PyMISP( url=config["misp_url"], key=config["misp_key"], ssl=bool(config.get("misp_verify_ssl", True)), ) except (KeyError, pymisp.PyMISPError): logger.error( "Integration with pyMISP disabled: no MITRE techniques tags") misp_client = None # Load the client to connect to VT (soft-fail) try: vt_client = vt.Client(apikey=config["vt_key"]) except (KeyError, ValueError): logger.error( "Integration with VT disabled: no automatic download of samples") vt_client = None # Decode and issue the request try: if request["attribute"]["type"] == "url": sample_url = request["attribute"]["value"] response = analysis_clients[DEFAULT_ENDPOINT].submit_url( sample_url) task_uuid, tags = _parse_submission_response(response) else: if request["attribute"]["type"] == "malware-sample": # Raise TypeError file_data = _unzip( base64.b64decode(request["attribute"]["data"])) file_name = request["attribute"]["value"].split("|", 1)[0] hash_value = hashlib.sha1(file_data).hexdigest() elif request["attribute"]["type"] == "attachment": # Raise TypeError file_data = base64.b64decode(request["attribute"]["data"]) file_name = request["attribute"].get("value") hash_value = hashlib.sha1(file_data).hexdigest() else: hash_value = request["attribute"]["value"] file_data = None file_name = "{}.bin".format(hash_value) # Check whether we have a task for that file tags = [] task_uuid = _get_latest_analysis(analysis_clients, hash_value) if not task_uuid: # If we have no analysis, download the sample from VT if not file_data: if not vt_client: raise ValueError( "No file available locally and VT is disabled") file_data = _download_from_vt(vt_client, hash_value) tags.append(VT_DOWNLOAD_TAG) # ... and submit it (_download_from_vt fails if no sample availabe) response = analysis_clients[DEFAULT_ENDPOINT].submit_file( file_data, file_name) task_uuid, _tags = _parse_submission_response(response) tags.extend(_tags) except KeyError as e: logger.error("Error parsing input: %s", request["attribute"]) return {"error": "Error parsing input: {}".format(e)} except TypeError as e: logger.error("Error decoding input: %s", request["attribute"]) return {"error": "Error decoding input: {}".format(e)} except ValueError as e: logger.error("Error processing input: %s", request["attribute"]) return {"error": "Error processing input: {}".format(e)} except (exceptions.CommunicationError, exceptions.ApiError) as e: logger.error("Error issuing API call: %s", str(e)) return {"error": "Error issuing API call: {}".format(e)} else: analysis_link = tau_clients.get_task_link( uuid=task_uuid, analysis_url=analysis_clients[DEFAULT_ENDPOINT].base, prefer_load_balancer=True, ) # Return partial results if the analysis has yet to terminate try: tags.extend(_get_analysis_tags(analysis_clients, task_uuid)) report = analysis_clients[DEFAULT_ENDPOINT].get_result(task_uuid) except (exceptions.CommunicationError, exceptions.ApiError) as e: logger.error("Error retrieving the report: %s", str(e)) return { "results": { "types": "link", "categories": ["External analysis"], "values": analysis_link, "tags": tags, } } # Return the enrichment try: techniques_galaxy = None if misp_client: techniques_galaxy = _get_mitre_techniques_galaxy(misp_client) result_parser = ResultParser(techniques_galaxy=techniques_galaxy) misp_event = result_parser.parse(analysis_link, report) for tag in tags: if tag not in frozenset([WORKFLOW_COMPLETE_TAG]): misp_event.add_tag(tag) return { "results": { key: json.loads(misp_event.to_json())[key] for key in ("Attribute", "Object", "Tag") if (key in misp_event and misp_event[key]) } } except pymisp.PyMISPError as e: logger.error("Error parsing the report: %s", str(e)) return {"error": "Error parsing the report: {}".format(e)}