def __init__(self,
url: str,
key: str,
ssl: Union[bool, str] = True,
debug: bool = False,
ignore: Union[str, None] = None):
if ssl and not os.path.isfile(ssl):
ssl = True
self.debug = debug
self._debug('Connecting to {}.'.format(url))
self.misp = pymisp.PyMISP(url=url, key=key, ssl=ssl)
if ignore:
self.ignore = ignore.split(',')
else:
self.ignore = None
def __init__(self, url, key, verify=False, misp_modules=False):
"""
Initialise the MISP instance
:param url: The URL of the MISP instance
:param key: The API key to authenticate with
:param verify: Shall we verify SSL? Can be bool or location of a signature
"""
log.debug("Starting MISP")
log.debug("%s -- %s", url, key)
self.url = url
self.key = key
self.verify = verify
self.mispAPI = pymisp.PyMISP(url, key, ssl=self.verify, debug=False)
self.misp_modules = misp_modules
def __init__(self,
url: str,
key: str,
ssl=True,
debug: bool = False,
ignore=None):
# Python <= 3.5 doesn support typing.Union. This method will be removed
# when we stop supporting python 3.4
if ssl and not os.path.isfile(ssl):
ssl = True
self.debug = debug
self._debug('Connecting to {}.'.format(url))
self.misp = pymisp.PyMISP(url=url, key=key, ssl=ssl)
if ignore:
self.ignore = ignore.split(',')
else:
self.ignore = None
def run(self):
"""Run analysis.
@return: MISP results dict.
"""
self.url = self.options.get("url", "")
self.apikey = self.options.get("apikey", "")
maxioc = int(self.options.get("maxioc", 100))
if not self.url or not self.apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance."
)
self.key = "misp"
self.iocs = {}
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import pymisp
self.misp = pymisp.PyMISP(self.url, self.apikey, False, "json")
iocs = set()
iocs.add(self.results.get("target", {}).get("file", {}).get("md5"))
for dropped in self.results.get("dropped", []):
iocs.add(dropped.get("md5"))
iocs.update(self.results.get("network", {}).get("hosts", []))
for block in self.results.get("network", {}).get("domains", []):
iocs.add(block.get("ip"))
iocs.add(block.get("domain"))
# Remove empty entries and turn the collection into a list.
iocs = list(iocs.difference((None, "")))
# Acquire all information related to IOCs.
for ioc in iocs[:maxioc]:
self.search_ioc(ioc)
# Sort IOC information by date and return all information.
return sorted(
self.iocs.values(), key=self._parse_date, reverse=True
)
def run(self, results):
"""Submits results to MISP.
@param results: Cuckoo results dict.
"""
url = self.options.get("url")
apikey = self.options.get("apikey")
mode = shlex.split(self.options.get("mode") or "")
if not url or not apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance.")
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import pymisp
self.misp = pymisp.PyMISP(url, apikey, False, "json")
event = self.misp.new_event(
distribution=pymisp.Distribution.all_communities.value,
threat_level_id=pymisp.ThreatLevel.undefined.value,
analysis=pymisp.Analysis.completed.value,
info="Cuckoo Sandbox analysis #%d" % self.task["id"],
)
if results.get("target", {}).get("category") == "file":
self.misp.upload_sample(
filename=os.path.basename(self.task["target"]),
filepath_or_bytes=self.task["target"],
event_id=event["Event"]["id"],
category="External analysis",
)
self.signature(results, event)
if "hashes" in mode:
self.sample_hashes(results, event)
if "url" in mode:
self.all_urls(results, event)
if "ipaddr" in mode:
self.domain_ipaddr(results, event)
self.family(results, event)
def run(self, results):
"""Submits results to MISP.
@param results: Cuckoo results dict.
"""
url = self.options.get("url")
apikey = self.options.get("apikey")
mode = shlex.split(self.options.get("mode") or "")
if not url or not apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP instance."
)
self.misp = pymisp.PyMISP(url, apikey, False, "json")
event = self.misp.new_event(
distribution=self.misp.distributions.all_communities,
threat_level_id=self.misp.threat_level.undefined,
analysis=self.misp.analysis.completed,
info="Cuckoo Sandbox analysis #%d" % self.task["id"],
)
if results.get("target", {}).get("category") == "file":
self.misp.upload_sample(
filename=os.path.basename(self.task["target"]),
filepath=self.task["target"],
event_id=event["Event"]["id"],
category="External analysis",
)
if "hashes" in mode:
self.sample_hashes(results, event)
if "maldoc" in mode:
self.maldoc_network(results, event)
if "url" in mode:
self.all_urls(results, event)
if "ipaddr" in mode:
self.domain_ipaddr(results, event)
def run(self):
api_key = secrets.get_secret(self.api_key_name)
if not api_key:
raise AnalyzerRunException(
f"no MISP API key retrieved with name: {self.api_key_name}"
)
if not self.url_name:
raise AnalyzerRunException(
f"no MISP URL retrieved, key value: {self.url_key_name}"
)
misp_instance = pymisp.PyMISP(
url=self.url_name,
key=api_key,
ssl=self.ssl_check,
debug=self.debug,
timeout=5,
)
# we check only for events not older than 90 days and max 50 results
now = datetime.datetime.now()
date_from = now - datetime.timedelta(days=90)
params = {
# even if docs say to use "values",...
# .. at the moment it works correctly only with "value"
"value": self.observable_name,
"type_attribute": [self.observable_classification],
"date_from": date_from.strftime("%Y-%m-%d %H:%M:%S"),
"limit": 50,
"enforce_warninglist": True,
}
if self.observable_classification == "hash":
params["type_attribute"] = ["md5", "sha1", "sha256"]
result_search = misp_instance.search(**params)
if isinstance(result_search, dict):
errors = result_search.get("errors", [])
if errors:
raise AnalyzerRunException(errors)
return {"result_search": result_search, "instance_url": self.url_name}
def __init__(self, conf, logger, targs=None, responders=[]):
Thread.__init__(self)
self.args = targs
self.conf = conf
self.lh = logger
self.responders = responders
self.feyerex = re.compile(r'^\s*([\w\-\"]*):(.*)')
self.esenrichment = conf["esenrichment"]
if self.esenrichment:
self.ese = ESEnrichment(self.lh, self.conf["esenrichment_server"])
if self.conf["email_alerts"]:
from smtplib import SMTP
from email.mime.text import MIMEText
self.smtp = SMTP
self.mimetext = MIMEText
if self.conf['misp_enabled']:
self.misp = pymisp.PyMISP(self.conf['mispurl'],
self.conf['mispkey'])
else:
self.misp = None
if conf["elasticsearch"]:
if "phishing" in conf["elasticsearch_config"]:
self.esphish = ES(conf["elasticsearch_config"]["phishing"],
logger)
if "fireeye" in conf["elasticsearch_config"]:
self.esfire = ES(conf["elasticsearch_config"]["fireeye"],
logger)
if conf["activedirectory-enrichment"]:
self.ad = ADEnrichment(
conf["activedirectory-enrichment-configuration"])
class RootCAAdapter(requests.adapters.HTTPAdapter):
def cert_verify(self, conn, url, verify, cert):
cert_file = conf['certs'][urlparse(url).hostname]
super(RootCAAdapter, self).cert_verify(conn=conn,
url=url,
verify=cert_file,
cert=cert)
for j in normalize(i):
response = misp.search_all(j)
if response['response']:
for resp in response['response']:
hashesDict[j] = {
resp['Event']['id']: [
resp['Event']['Org']['name'],
resp['Event']['Orgc']['name'],
resp['Event']['date'], resp['Event']['timestamp'],
resp['Event']['publish_timestamp'],
resp['Event']['info'], j
]
}
return hashesDict
if __name__ == '__main__':
misp = pymisp.PyMISP(misp_url, misp_key, misp_verifycert, 'json')
#d=queryUrl(misp, sourceFile='submittedUrl_list.txt')
#with open('result.json', 'w') as fp:
# json.dump(d, fp)
#d=queryhashes(misp, "md5", sourceFile='dataFull.json')
#with open('result_md5.json', 'w') as fp:
# json.dump(d, fp)
d = queryhashes(misp, "sha1", sourceFile='dataFull.json')
with open('result_sha1.json', 'w') as fp:
json.dump(d, fp)
d = queryhashes(misp, "sha256", sourceFile='dataFull.json')
with open('result_sha256.json', 'w') as fp:
json.dump(d, fp)
def initialize(self):
self._misp = pymisp.PyMISP(self._url, self._key, False) ### false
self.load_extra_info()
def create_events(pulse_or_list, author=False, server=False, key=False, misp=False, distribution=0, threat_level=4,
analysis=2, publish=False, tlp=True, discover_tags=False, to_ids=False, author_tag=False,
bulk_tag=None, dedup_titles=False, stop_on_error=False):
"""
Parse a Pulse or a list of Pulses and add it/them to MISP if server and key are present
:param pulse_or_list: a Pulse or list of Pulses as returned by `get_pulses`
:param author: Prepend the author to the Pulse name
:type author: Boolean
:param server: MISP server URL
:param key: MISP API key
:param misp: MISP connection object
:type misp: :class:`pymisp.PyMISP`
:param distribution: distribution of the MISP event (0-4)
:param threat_level: threat level of the MISP object (1-4)
:param analysis: analysis stae of the MISP object (0-2)
:param publish: Is the MISP event should be published?
:type publish: Boolean
:param tlp: Add TLP level tag to event
:type tlp: Boolean
:param discover_tags: discover MISP tags from Pulse tags
:type discover_tags: Boolean
:param to_ids: Flag pulse attributes as being sent to an IDS
:type to_ids: Boolean
:param author_tag: Add the pulse author as an event tag
:type author_tag: Boolean
:param bulk_tag: A tag that will be added to all events for categorization (e.g. OTX)
:type bulk_tag: String
:param dedup_titles: Search MISP for an existing event title and update it, rather than create a new one
:type dedup_titles: Boolean
:return: a dict or a list of dict with the selected attributes
"""
if not misp and (server and key):
log.debug("Connection to MISP instance: {}".format(server))
try:
misp = pymisp.PyMISP(server, key, ssl=False, out_type='json')
except pymisp.PyMISPError as ex:
raise ImportException("Cannot connect to MISP instance: {}".format(ex.message))
except Exception as ex:
raise ImportException("Cannot connect to MISP instance, unknown exception: {}".format(ex.message))
if discover_tags:
def get_tag_name(complete):
parts = complete.split('=')
if not len(parts):
return complete
last = parts[-1]
if not len(last):
return complete
if last[0] == '"':
last = last[1:]
if last[-1] == '"':
last = last[:-1]
return last.lower()
raw_tags = misp.get_all_tags()
tags = dict()
for tag in raw_tags['Tag']:
tags[get_tag_name(tag['name'])] = tag['name']
misp.discovered_tags = tags
if isinstance(pulse_or_list, (list, tuple)) or inspect.isgenerator(pulse_or_list):
misp_events = []
for pulse in pulse_or_list:
try:
misp_event = create_events(pulse, author=author, server=server, key=key, misp=misp,
distribution=distribution, threat_level=threat_level, analysis=analysis,
publish=publish, tlp=tlp, to_ids=to_ids, author_tag=author_tag,
bulk_tag=bulk_tag, dedup_titles=dedup_titles, stop_on_error=stop_on_error)
misp_events.append(misp_event)
except Exception as ex:
if stop_on_error:
raise
name = ''
if pulse and 'name' in pulse:
name = pulse['name']
log.error("Cannot import pulse {}: {}".format(name, ex))
return misp_events
pulse = pulse_or_list
if author:
event_name = pulse['author_name'] + ' | ' + pulse['name']
else:
event_name = pulse['name']
try:
dt = date_parser.parse(pulse['created'])
except (ValueError, OverflowError):
log.error("Cannot parse Pulse 'created' date.")
dt = datetime.utcnow()
event_date = dt.strftime('%Y-%m-%d')
log.info("## {name} - {date}".format(name=event_name, date=event_date))
result_event = {
'name': event_name,
'date': event_date,
'tags': list(),
'attributes': {
'hashes': {
'md5': list(),
'sha1': list(),
'sha256': list(),
'imphash': list(),
'pehash': list()
},
'hostnames': list(),
'domains': list(),
'urls': list(),
'ips': list(),
'emails': list(),
'mutexes': list(),
'references': list(),
'cves': list(),
'filenames': list(),
'yara': list()
},
}
if misp:
if not dedup_titles:
event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date, published=publish)
else:
event = ''
# Check if username is added to title
# Build the title
if author:
event_name = pulse['author_name'] + ' | ' + pulse['name']
else:
event_name = pulse['name']
# Search MISP for the title
result = misp.search_index(eventinfo=event_name)
if 'message' in result:
if result['message'] == "No matches.":
event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date,
published=publish)
else:
for evt in result['response']:
# If it exists, set 'event' to the event
if evt['info'] == event_name:
if 'SharingGroup' in evt:
del evt['SharingGroup'] # This deletes the SharingGroup from the list, thx SparkyNZL
event = {'Event': evt}
break
if event == '':
# Event not found, even though search results were returned
# Build new event
event = misp.new_event(distribution, threat_level, analysis, event_name, date=event_date,
published=publish)
time.sleep(0.2)
if tlp:
tag = None
if 'TLP' in pulse:
tag = "tlp:{}".format(pulse['TLP'])
elif 'tlp' in pulse:
tag = "tlp:{}".format(pulse['tlp'])
if tag is not None:
log.info("\t - Adding tag: {}".format(tag))
tag_event(misp, event, tag)
result_event['tags'].append(tag)
if author_tag:
tag_event(misp, event, pulse['author_name'])
if bulk_tag is not None:
tag_event(misp, event, bulk_tag)
if misp and hasattr(misp, 'discovered_tags') and 'tags' in pulse:
for pulse_tag in pulse['tags']:
if pulse_tag.lower() in misp.discovered_tags:
tag = misp.discovered_tags[pulse_tag.lower()]
log.info("\t - Adding tag: {}".format(tag))
tag_event(misp, event, tag)
result_event['tags'].append(tag)
if 'references' in pulse:
for reference in pulse['references']:
if reference:
log.info("\t - Adding external analysis link: {}".format(reference))
if misp:
misp.add_named_attribute(event, 'link', reference, category='External analysis')
result_event['attributes']['references'].append(reference)
if misp and 'description' in pulse and isinstance(pulse['description'], six.text_type) and pulse['description']:
log.info("\t - Adding external analysis comment")
misp.add_named_attribute(event, 'comment', pulse['description'], category='External analysis')
for ind in pulse['indicators']:
ind_type = ind['type']
ind_val = ind['indicator']
ind_kwargs = {'to_ids': to_ids}
if 'description' in ind and isinstance(ind['description'], six.text_type) and ind['description']:
ind_kwargs['comment'] = ind['description']
if ind_type == 'FileHash-SHA256':
log.info("\t - Adding SHA256 hash: {}".format(ind_val))
if misp:
misp.add_hashes(event, sha256=ind_val, **ind_kwargs)
result_event['attributes']['hashes']['sha256'].append(ind_val)
elif ind_type == 'FileHash-SHA1':
log.info("\t - Adding SHA1 hash: {}".format(ind_val))
if misp:
misp.add_hashes(event, sha1=ind_val, **ind_kwargs)
result_event['attributes']['hashes']['sha1'].append(ind_val)
elif ind_type == 'FileHash-MD5':
log.info("\t - Adding MD5 hash: {}".format(ind_val))
if misp:
misp.add_hashes(event, md5=ind_val, **ind_kwargs)
result_event['attributes']['hashes']['md5'].append(ind_val)
elif ind_type == 'URI' or ind_type == 'URL':
log.info("\t - Adding URL: {}".format(ind_val))
if misp:
misp.add_url(event, ind_val, **ind_kwargs)
result_event['attributes']['urls'].append(ind_val)
elif ind_type == 'domain':
log.info("\t - Adding domain: {}".format(ind_val))
if misp:
misp.add_domain(event, ind_val, **ind_kwargs)
result_event['attributes']['domains'].append(ind_val)
elif ind_type == 'hostname':
log.info("\t - Adding hostname: {}".format(ind_val))
if misp:
misp.add_hostname(event, ind_val, **ind_kwargs)
result_event['attributes']['hostnames'].append(ind_val)
elif ind_type == 'IPv4' or ind_type == 'IPv6':
log.info("\t - Adding ip: {}".format(ind_val))
if misp:
misp.add_ipdst(event, ind_val, **ind_kwargs)
result_event['attributes']['ips'].append(ind_val)
elif ind_type == 'email':
log.info("\t - Adding email: {}".format(ind_val))
if misp:
misp.add_email_dst(event, ind_val, **ind_kwargs)
result_event['attributes']['emails'].append(ind_val)
elif ind_type == 'Mutex':
log.info("\t - Adding mutex: {}".format(ind_val))
if misp:
misp.add_mutex(event, ind_val, **ind_kwargs)
result_event['attributes']['mutexes'].append(ind_val)
elif ind_type == 'CVE':
log.info("\t - Adding CVE: {}".format(ind_val))
if misp:
misp.add_named_attribute(event, 'vulnerability', ind_val, category='External analysis', **ind_kwargs)
result_event['attributes']['cves'].append(ind_val)
elif ind_type == 'FileHash-IMPHASH':
log.info("\t - Adding IMPHASH hash: {}".format(ind_val))
if misp:
misp.add_named_attribute(event, 'imphash', ind_val, category='Artifacts dropped', **ind_kwargs)
result_event['attributes']['hashes']['imphash'].append(ind_val)
elif ind_type == 'FileHash-PEHASH':
log.info("\t - Adding PEHASH hash: {}".format(ind_val))
if misp:
misp.add_named_attribute(event, 'pehash', ind_val, category='Artifacts dropped', **ind_kwargs)
result_event['attributes']['hashes']['pehash'].append(ind_val)
elif ind_type == 'FilePath':
log.info("\t - Adding filename: {}".format(ind_val))
if misp:
misp.add_filename(event, ind_val, category='Artifacts dropped', **ind_kwargs)
result_event['attributes']['filenames'].append(ind_val)
elif ind_type == 'YARA':
ind_title = ind.get('title', ind_val)
ind_desc = ind.get('description', '')
if ind_title == '':
ind_title = ind_val
if not ind_desc == '':
ind_kwargs['comment'] = ind_desc
else:
ind_kwargs['comment'] = "{} {}".format(ind_title, ind_desc)
ind_val = ind.get('content', None)
if ind_val is None or ind_val == "":
log.warning("YARA indicator is empty: %s" % ind_title)
continue
log.info("\t - Adding YARA rule: {}".format(ind_title))
if misp:
misp.add_yara(event, ind_val, category='Artifacts dropped', **ind_kwargs)
result_event['attributes']['yara'].append({'title': ind_title, 'content': ind_val})
else:
log.warning("Unsupported indicator type: %s" % ind_type)
if misp :
event['Event']['published'] = False
misp.publish(event)
return result_event
value = json.loads(value)
for i in range(1, len(subkeys)):
if subkeys[i] in value:
value = value[subkeys[i]]
else:
break
except ValueError as e:
pass
if value is None:
value = _default if not _default is None else ''
return value
try:
misp = pymisp.PyMISP(url=config['MISP']['proto'] + '://' +
config['MISP']['host'] + ':' + config['MISP']['port'],
key=config['MISP']['token'],
ssl=False)
resp = misp.search(controller=args.controller,
type_attribute=args.type,
org=args.org,
last=args.last,
date_from=args.date_from,
date_to=args.date_to,
tags=args.tags,
not_tags=args.not_tags,
eventid=args.eventid)
idxcol = args.idx_col
events = {}
if args.controller == 'attributes':
def run(self, results):
"""Run analysis.
@return: MISP results dict.
"""
url = self.options.get("url", "")
apikey = self.options.get("apikey", "")
if not url or not apikey:
log.error("MISP URL or API key not configured.")
return
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import pymisp
self.misp = pymisp.PyMISP(url, apikey, False, "json")
self.threads = self.options.get("threads", "")
if not self.threads:
self.threads = 5
whitelist = list()
self.iocs = deque()
self.misper = dict()
self.misp_full_report = dict()
self.lock = threading.Lock()
try:
# load whitelist if exists
if os.path.exists(os.path.join(CUCKOO_ROOT, "conf", "misp.conf")):
whitelist = Config("misp").whitelist.whitelist
if whitelist:
whitelist = [ioc.strip() for ioc in whitelist.split(",")]
if self.options.get("upload_iocs", False) and results.get("malscore", 0) >= self.options.get("min_malscore", 0):
distribution = int(self.options.get("distribution", 0))
threat_level_id = int(self.options.get("threat_level_id", 4))
analysis = int(self.options.get("analysis", 0))
tag = self.options.get("tag") or "CAPEv2"
info = self.options.get("title", "")
upload_sample = self.options.get("upload_sample")
malfamily = ""
filtered_iocs = deque()
if results.get("malfamily", ""):
malfamily = results["malfamily"]
event = self.misp.new_event(
distribution=distribution,
threat_level_id=threat_level_id,
analysis=analysis,
info="{} {} - {}".format(info, malfamily, results.get('info', {}).get('id'))
)
# Add a specific tag to flag Cuckoo's event
if tag:
mispresult = self.misp.tag(event["Event"]["uuid"], tag)
if mispresult.has_key("message"):
log.debug("tag event: %s" % mispresult["message"])
#ToDo?
#self.signature(results, event)
self.sample_hashes(results, event)
self.all_network(results, event, whitelist)
self.dropped_files(results, event, whitelist)
#ToDo add? upload sample
"""
if upload_sample:
target = results.get("target", {})
f = target.get("file", {})
if target.get("category") == "file" and f:
self.misp.upload_sample(
filename=os.path.basename(f["name"]),
filepath_or_bytes=f["path"],
event_id=event["Event"]["id"],
category="Payload delivery",
comment="Sample run",
)
"""
self.misper.setdefault("iocs", list())
#if results.get("target", {}).get("url", "") and results["target"]["url"] not in whitelist:
# filtered_iocs.append(results["target"]["url"])
# #parsed = urlsplit(results["target"]["url"])
# ToDo migth be outdated!
#if self.options.get("ids_files", False) and "suricata" in results.keys():
# for surifile in results["suricata"]["files"]:
# if "file_info" in surifile.keys():
# self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]})
# self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]})
# self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]})
if self.options.get("mutexes", False) and "behavior" in results and "summary" in results["behavior"]:
if "mutexes" in results.get("behavior", {}).get("summary", {}):
for mutex in results["behavior"]["summary"]["mutexes"]:
if mutex not in whitelist:
self.misp.add_mutex(event, mutex)
if self.options.get("registry", False) and "behavior" in results and "summary" in results["behavior"]:
if "read_keys" in results["behavior"].get("summary", {}):
for regkey in results["behavior"]["summary"]["read_keys"]:
self.misp.add_regkey(event, regkey)
except Exception as e:
log.error("Failed to generate JSON report: %s" % e, exc_info=True)
config["zmq"]["host"],
config["zmq"]["port"]
))
# Set the option to subscribe
socket.setsockopt_string(zmq.SUBSCRIBE, '')
# Get MISP-Modules mod list
log.info("Connecting to Modules server at http://%s:%s", config["misp"]["modules"]["host"],
config["misp"]["modules"]["port"])
modulesURL = "http://{}:{}".format(config["misp"]["modules"]["host"],
config["misp"]["modules"]["port"])
modules = requests.get(urljoin(modulesURL, "modules")).json()
configModules = config["misp"]["modules"]["run-modules"]
api = pymisp.PyMISP(config["misp"]["url"], config["misp"]["apikey"])
while True:
# Wait for something to come in on the ZMQ socket
message = socket.recv().decode("utf-8")[10:]
log.info("Recieved a message!")
log.debug("Processing...")
# Load the message JSON
msg = json.loads(message)
log.debug(msg)
# Load it as a misp object
ev = pymisp.mispevent.MISPEvent()
def getMISPInfo(hash):
"""
Retrieves information from a MISP instance
:param hash: hash value
:return info: info object
"""
info = {'misp_available': False, 'misp_events': ''}
requests.packages.urllib3.disable_warnings() # I don't care
# Check if any auth key is set
key_set = False
for m in MISP_AUTH_KEYS:
if m != '' and m != '-':
key_set = True
if not key_set or 'pymisp' in deactivated_features:
return info
# Loop through MISP instances
misp_info = []
misp_events = []
for c, m_url in enumerate(MISP_URLS, start=0):
# Get the corresponding auth key
m_auth_key = MISP_AUTH_KEYS[c]
if args.debug:
print("[D] Querying MISP: %s" % m_url)
try:
# Preparing API request
misp = pymisp.PyMISP(m_url, m_auth_key, args.verifycert, debug=args.debug, proxies={},cert=None,auth=None,tool='Munin : Online hash checker')
if args.debug:
print("[D] Query: values=%s" % hash)
result = misp.search('attributes', type_attribute=fetchHash(hash)[1] ,value=hash)
# Processing the result
if result['Attribute']:
events_added = list()
if args.debug:
print("[D] Dump Attribute : "+json.dumps(result['Attribute'], indent=2))
for r in result['Attribute']:
# Check for duplicates
if r['event_id'] in events_added:
continue
# Try to get info on the events
event_info = ""
misp_events.append('MISP%d:%s' % (c+1, r['event_id']))
e_result = misp.search('events', eventid=r['event_id'])
if e_result:
event_info = e_result[0]['Event']['info']
# too much
# if args.debug:
# print(json.dumps(e_result['response'], indent=2))
# Create MISP info object
misp_info.append({
'misp_nr': c+1,
'event_info': event_info,
'event_id': r['event_id'],
'comment': r['comment'],
'url': '%s/events/view/%s' % (m_url, r['event_id'])
})
events_added.append(r['event_id'])
else:
info['misp_available'] = False
except Exception as e:
if args.debug:
traceback.print_exc()
info['misp_info'] = misp_info
info['misp_events'] = ", ".join(misp_events)
if len(misp_events) > 0:
info['misp_available'] = True
return info
def create_events(pulse_or_list,
author=False,
server=False,
key=False,
misp=False,
distribution=0,
threat_level=4,
analysis=2,
publish=True,
tlp=True,
discover_tags=False,
to_ids=False):
"""
Parse a Pulse or a list of Pulses and add it/them to MISP if server and key are present
:param pulse_or_list: a Pulse or list of Pulses as returned by `get_pulses`
:param author: Prepend the author to the Pulse name
:type author: Boolean
:param server: MISP server URL
:param key: MISP API key
:param misp: MISP connection object
:type misp: :class:`pymisp.PyMISP`
:param distribution: distribution of the MISP event (0-4)
:param threat_level: threat level of the MISP object (1-4)
:param analysis: analysis stae of the MISP object (0-2)
:param publish: Is the MISP event should be published?
:type publish: Boolean
:param tlp: Add TLP level tag to event
:type tlp: Boolean
:param discover_tags: discover MISP tags from Pulse tags
:type discover_tags: Boolean
:return: a dict or a list of dict with the selected attributes
"""
if not misp and (server and key):
log.debug("Connection to MISP instance: {}".format(server))
try:
misp = pymisp.PyMISP(server,
key,
ssl=False,
out_type='json',
debug=False)
except pymisp.PyMISPError as ex:
raise ImportException("Cannot connect to MISP instance: {}".format(
ex.message))
except Exception as ex:
raise ImportException(
"Cannot connect to MISP instance, unknown exception: {}".
format(ex.message))
if discover_tags:
def get_tag_name(complete):
parts = complete.split('=')
if not len(parts):
return complete
last = parts[-1]
if last[0] == '"':
last = last[1:]
if last[-1] == '"':
last = last[:-1]
return last.lower()
raw_tags = misp.get_all_tags()
tags = dict()
for tag in raw_tags['Tag']:
tags[get_tag_name(tag['name'])] = tag['name']
misp.discovered_tags = tags
if isinstance(pulse_or_list,
(list, tuple)) or inspect.isgenerator(pulse_or_list):
return [
create_events(pulse,
author=author,
server=server,
key=key,
misp=misp,
distribution=distribution,
threat_level=threat_level,
analysis=analysis,
publish=publish,
tlp=tlp,
to_ids=to_ids) for pulse in pulse_or_list
]
pulse = pulse_or_list
if author:
event_name = pulse['author_name'] + ' | ' + pulse['name']
else:
event_name = pulse['name']
try:
dt = date_parser.parse(pulse['modified'])
except (ValueError, OverflowError):
log.error("Cannot parse Pulse 'modified' date.")
dt = datetime.utcnow()
event_date = dt.strftime('%Y-%m-%d')
log.info("## {name} - {date}".format(name=event_name, date=event_date))
result_event = {
'name': event_name,
'date': event_date,
'tags': list(),
'attributes': {
'hashes': {
'md5': list(),
'sha1': list(),
'sha256': list(),
'imphash': list(),
'pehash': list()
},
'hostnames': list(),
'domains': list(),
'urls': list(),
'ips': list(),
'emails': list(),
'mutexes': list(),
'references': list(),
'cves': list()
},
}
if misp:
event = misp.new_event(distribution,
threat_level,
analysis,
event_name,
date=event_date,
published=publish)
time.sleep(0.2)
if tlp and 'TLP' in pulse:
tag = "tlp:{}".format(pulse['TLP'])
log.info("\t - Adding tag: {}".format(tag))
misp.add_tag(event, tag)
result_event['tags'].append(tag)
if misp and hasattr(misp, 'discovered_tags') and 'tags' in pulse:
for pulse_tag in pulse['tags']:
if pulse_tag.lower() in misp.discovered_tags:
tag = misp.discovered_tags[pulse_tag.lower()]
log.info("\t - Adding tag: {}".format(tag))
misp.add_tag(event, tag)
result_event['tags'].append(tag)
if 'references' in pulse:
for reference in pulse['references']:
log.info(
"\t - Adding external analysis link: {}".format(reference))
if misp:
misp.add_named_attribute(event,
'link',
reference,
category='External analysis')
result_event['attributes']['references'].append(reference)
if misp and 'description' in pulse and isinstance(
pulse['description'], six.text_type) and pulse['description']:
log.info("\t - Adding external analysis comment")
misp.add_named_attribute(event,
'comment',
pulse['description'],
category='External analysis')
for ind in pulse['indicators']:
ind_type = ind['type']
ind_val = ind['indicator']
ind_kwargs = {'to_ids': to_ids}
if 'description' in ind and isinstance(
ind['description'], six.text_type) and ind['description']:
ind_kwargs['comment'] = ind['description']
if ind_type == 'FileHash-SHA256':
log.info("\t - Adding SH256 hash: {}".format(ind_val))
if misp:
misp.add_hashes(event, sha256=ind_val, **ind_kwargs)
result_event['attributes']['hashes']['sha256'].append(ind_val)
elif ind_type == 'FileHash-SHA1':
log.info("\t - Adding SHA1 hash: {}".format(ind_val))
if misp:
misp.add_hashes(event, sha1=ind_val, **ind_kwargs)
result_event['attributes']['hashes']['sha1'].append(ind_val)
elif ind_type == 'FileHash-MD5':
log.info("\t - Adding MD5 hash: {}".format(ind_val))
if misp:
misp.add_hashes(event, md5=ind_val, **ind_kwargs)
result_event['attributes']['hashes']['md5'].append(ind_val)
elif ind_type == 'URI' or ind_type == 'URL':
log.info("\t - Adding URL: {}".format(ind_val))
if misp:
misp.add_url(event, ind_val, **ind_kwargs)
result_event['attributes']['urls'].append(ind_val)
elif ind_type == 'domain':
log.info("\t - Adding domain: {}".format(ind_val))
if misp:
misp.add_domain(event, ind_val, **ind_kwargs)
result_event['attributes']['domains'].append(ind_val)
elif ind_type == 'hostname':
log.info("\t - Adding hostname: {}".format(ind_val))
if misp:
misp.add_hostname(event, ind_val, **ind_kwargs)
result_event['attributes']['hostnames'].append(ind_val)
elif ind_type == 'IPv4' or ind_type == 'IPv6':
log.info("\t - Adding ip: {}".format(ind_val))
if misp:
misp.add_ipdst(event, ind_val, **ind_kwargs)
result_event['attributes']['ips'].append(ind_val)
elif ind_type == 'email':
log.info("\t - Adding email: {}".format(ind_val))
if misp:
misp.add_email_dst(event, ind_val, **ind_kwargs)
result_event['attributes']['emails'].append(ind_val)
elif ind_type == 'Mutex':
log.info("\t - Adding mutex: {}".format(ind_val))
if misp:
misp.add_mutex(event, ind_val, **ind_kwargs)
result_event['attributes']['mutexes'].append(ind_val)
elif ind_type == 'CVE':
log.info("\t - Adding CVE: {}".format(ind_val))
if misp:
misp.add_named_attribute(event,
'vulnerability',
ind_val,
category='External analysis',
**ind_kwargs)
result_event['attributes']['cves'].append(ind_val)
elif ind_type == 'FileHash-IMPHASH':
log.info("\t - Adding IMPHASH hash: {}".format(ind_val))
if misp:
misp.add_named_attribute(event,
'imphash',
ind_val,
category='Artifacts dropped',
**ind_kwargs)
result_event['attributes']['hashes']['imphash'].append(ind_val)
elif ind_type == 'FileHash-PEHASH':
log.info("\t - Adding PEHASH hash: {}".format(ind_val))
if misp:
misp.add_named_attribute(event,
'pehash',
ind_val,
category='Artifacts dropped',
**ind_kwargs)
result_event['attributes']['hashes']['pehash'].append(ind_val)
else:
log.warning("Unsupported indicator type: %s" % ind_type)
if misp and publish:
event['Event']['published'] = False
misp.publish(event)
return result_event
Submit a list of MISP objects to a MISP event
:misp: PyMISP API object for interfacing with MISP
:misp_event: MISPEvent object
:misp_objects: List of MISPObject objects. Must be a list
'''
# go through round one and only add MISP objects
for misp_object in misp_objects:
template_id = misp.get_object_template_id(misp_object.template_uuid)
misp.add_object(misp_event.id, template_id, misp_object)
# go through round two and add all the object references for each object
for misp_object in misp_objects:
for reference in misp_object.ObjectReference:
misp.add_object_reference(reference)
if __name__ == "__main__":
try:
args = build_cli()
config = build_config(args.config)
# change the 'ssl' value if you want to verify your MISP's SSL instance
misp = pymisp.PyMISP(url=config["misp"]["url"], key=config["misp"]["key"], ssl=False)
# finally, let's start checking VT and converting the reports
main(misp, config, args)
except KeyboardInterrupt:
print("Bye Felicia")
except pymisp.exceptions.InvalidMISPObject as err:
logging.error(err)
def run(self, results):
"""Submit results to MISP.
@param results: Cuckoo results dict.
"""
url = self.options.get("url")
apikey = self.options.get("apikey")
mode = shlex.split(self.options.get("mode") or "")
score = results.get("info", {}).get("score", 0)
upload_sample = self.options.get("upload_sample")
if results.get("target", {}).get("category") == "file":
f = results.get("target", {}).get("file", {})
hash_safelisted = is_safelisted_misphash(f["md5"]) or \
is_safelisted_misphash(f["sha1"]) or \
is_safelisted_misphash(f["sha256"])
if hash_safelisted:
return
if score < self.options.get("min_malscore", 0):
return
if not url or not apikey:
raise CuckooProcessingError(
"Please configure the URL and API key for your MISP "
"instance."
)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import pymisp
self.misp = pymisp.PyMISP(url, apikey, False, "json")
# Get default settings for a new event
distribution = self.options.get("distribution") or 0
threat_level = self.options.get("threat_level") or 4
analysis = self.options.get("analysis") or 0
tag = self.options.get("tag") or "Cuckoo"
event = self.misp.new_event(
distribution=distribution,
threat_level_id=threat_level,
analysis=analysis,
info="Cuckoo Sandbox analysis #%d" % self.task["id"]
)
# Add a specific tag to flag Cuckoo's event
if tag:
mispresult = self.misp.tag(event["Event"]["uuid"], tag)
if mispresult.has_key("message"):
log.debug("tag event: %s" % mispresult["message"])
if upload_sample:
target = results.get("target", {})
if target.get("category") == "file" and target.get("file"):
self.misp.upload_sample(
filename=os.path.basename(self.task["target"]),
filepath_or_bytes=self.task["target"],
event_id=event["Event"]["id"],
category="External analysis",
)
self.signature(results, event)
if "hashes" in mode:
self.sample_hashes(results, event)
if "url" in mode:
self.all_urls(results, event)
if "ipaddr" in mode:
self.domain_ipaddr(results, event)
self.family(results, event)
logger.setLevel(logging.INFO)
logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', format=pymisp.FORMAT)
events_hashes = []
if len(keys.misp_url) == 0 or len(keys.misp_key) == 0:
print("Error: missing info about MISP database, please check keys.py file !")
exit(-1)
if "-h" in sys.argv or sys.argv[-1][0] == '-' or len(sys.argv) < 2 or ".yara" in sys.argv:
print("Command : yara_export.py [-h : show help] [-json : add a json dump along with yara dump] <yara output filename>")
exit(0)
if ".yara" in sys.argv[-1]:
sys.argv[-1] = sys.argv[-1].replace(".yara","")
misp = pymisp.PyMISP(keys.misp_url,keys.misp_key,False,False)
events = misp.search(controller='events')
for event in events: # For each event
for event_content in event.values(): # Get the content of the key "Event"
hashes_types = {}
event_dict = {"event_id": event_content["id"]}
event_dict.update({"info": event_content["info"]})
for attribute in event_content["Attribute"]: # For each attribute from a event
if attribute["type"] == "md5" or attribute["type"] == "sha1" or attribute["type"] == "sha256" :
if attribute["type"] not in hashes_types :
hashes_types.update({ attribute["type"]: [ attribute["value"] ] } )
else :
if attribute["value"] not in hashes_types[attribute["type"]] : # If hash already in list, ignore
hashes_types[attribute["type"]].append(attribute["value"])
print("Trying to use env variables...")
if "MISP_URL" in os.environ:
misp_url = os.environ["MISP_URL"]
else:
print("Unkown misp URL. Set OPENTAXII_CONFIG or MISP_URL.")
misp_url = "UNKNOWN"
if "MISP_API" in os.environ:
misp_api = os.environ["MISP_API"]
else:
print("Unknown misp API key. Set OPENTAXII_CONFIG or MISP_API.")
misp_api = "UNKNOWN"
CONFIG = {"misp": {"url": misp_url, "api": misp_api}}
MISP = pymisp.PyMISP(CONFIG["misp"]["url"],
CONFIG["misp"]["api"],
ssl=CONFIG["misp"].get("verifySSL", True))
def post_stix(manager, content_block, collection_ids, service_id):
'''
Callback function for when our taxii server gets new data
Will convert it to a MISPEvent and push to the server
'''
# Load the package
log.info("Posting STIX...")
package = pymisp.tools.stix.load_stix(content_block.content)
log.info("STIX loaded succesfully.")
values = [x.value for x in package.attributes]
log.info("Extracted %s", values)
def handler(q: Union[bool, str] = False) -> Union[bool, Dict[str, Any]]:
"""
Implement interface.
:param bool|str q: the input received
:rtype: bool|dict[str, any]
"""
if q is False:
return False
request = json.loads(q)
config = request.get("config", {})
# Load the client to connect to VMware NSX ATA (hard-fail)
try:
analysis_url = config.get("analysis_url")
login_params = {
"key": config["analysis_key"],
"api_token": config["analysis_api_token"],
}
# If 'analysis_url' is specified we are connecting on-premise
if analysis_url:
analysis_clients = {
DEFAULT_ENDPOINT:
nsx_defender.AnalysisClient(
api_url=analysis_url,
login_params=login_params,
verify_ssl=bool(config.get("analysis_verify_ssl", True)),
)
}
logger.info(
"Connected NSX AnalysisClient to on-premise infrastructure")
else:
analysis_clients = {
data_center: nsx_defender.AnalysisClient(
api_url=tau_clients.
NSX_DEFENDER_ANALYSIS_URLS[data_center],
login_params=login_params,
verify_ssl=bool(config.get("analysis_verify_ssl", True)),
)
for data_center in [
tau_clients.NSX_DEFENDER_DC_WESTUS,
tau_clients.NSX_DEFENDER_DC_NLEMEA,
]
}
logger.info(
"Connected NSX AnalysisClient to hosted infrastructure")
except KeyError as ke:
logger.error("Integration with VMware NSX ATA failed to connect: %s",
str(ke))
return {"error": "Error connecting to VMware NSX ATA: {}".format(ke)}
# Load the client to connect to MISP (soft-fail)
try:
misp_client = pymisp.PyMISP(
url=config["misp_url"],
key=config["misp_key"],
ssl=bool(config.get("misp_verify_ssl", True)),
)
except (KeyError, pymisp.PyMISPError):
logger.error(
"Integration with pyMISP disabled: no MITRE techniques tags")
misp_client = None
# Load the client to connect to VT (soft-fail)
try:
vt_client = vt.Client(apikey=config["vt_key"])
except (KeyError, ValueError):
logger.error(
"Integration with VT disabled: no automatic download of samples")
vt_client = None
# Decode and issue the request
try:
if request["attribute"]["type"] == "url":
sample_url = request["attribute"]["value"]
response = analysis_clients[DEFAULT_ENDPOINT].submit_url(
sample_url)
task_uuid, tags = _parse_submission_response(response)
else:
if request["attribute"]["type"] == "malware-sample":
# Raise TypeError
file_data = _unzip(
base64.b64decode(request["attribute"]["data"]))
file_name = request["attribute"]["value"].split("|", 1)[0]
hash_value = hashlib.sha1(file_data).hexdigest()
elif request["attribute"]["type"] == "attachment":
# Raise TypeError
file_data = base64.b64decode(request["attribute"]["data"])
file_name = request["attribute"].get("value")
hash_value = hashlib.sha1(file_data).hexdigest()
else:
hash_value = request["attribute"]["value"]
file_data = None
file_name = "{}.bin".format(hash_value)
# Check whether we have a task for that file
tags = []
task_uuid = _get_latest_analysis(analysis_clients, hash_value)
if not task_uuid:
# If we have no analysis, download the sample from VT
if not file_data:
if not vt_client:
raise ValueError(
"No file available locally and VT is disabled")
file_data = _download_from_vt(vt_client, hash_value)
tags.append(VT_DOWNLOAD_TAG)
# ... and submit it (_download_from_vt fails if no sample availabe)
response = analysis_clients[DEFAULT_ENDPOINT].submit_file(
file_data, file_name)
task_uuid, _tags = _parse_submission_response(response)
tags.extend(_tags)
except KeyError as e:
logger.error("Error parsing input: %s", request["attribute"])
return {"error": "Error parsing input: {}".format(e)}
except TypeError as e:
logger.error("Error decoding input: %s", request["attribute"])
return {"error": "Error decoding input: {}".format(e)}
except ValueError as e:
logger.error("Error processing input: %s", request["attribute"])
return {"error": "Error processing input: {}".format(e)}
except (exceptions.CommunicationError, exceptions.ApiError) as e:
logger.error("Error issuing API call: %s", str(e))
return {"error": "Error issuing API call: {}".format(e)}
else:
analysis_link = tau_clients.get_task_link(
uuid=task_uuid,
analysis_url=analysis_clients[DEFAULT_ENDPOINT].base,
prefer_load_balancer=True,
)
# Return partial results if the analysis has yet to terminate
try:
tags.extend(_get_analysis_tags(analysis_clients, task_uuid))
report = analysis_clients[DEFAULT_ENDPOINT].get_result(task_uuid)
except (exceptions.CommunicationError, exceptions.ApiError) as e:
logger.error("Error retrieving the report: %s", str(e))
return {
"results": {
"types": "link",
"categories": ["External analysis"],
"values": analysis_link,
"tags": tags,
}
}
# Return the enrichment
try:
techniques_galaxy = None
if misp_client:
techniques_galaxy = _get_mitre_techniques_galaxy(misp_client)
result_parser = ResultParser(techniques_galaxy=techniques_galaxy)
misp_event = result_parser.parse(analysis_link, report)
for tag in tags:
if tag not in frozenset([WORKFLOW_COMPLETE_TAG]):
misp_event.add_tag(tag)
return {
"results": {
key: json.loads(misp_event.to_json())[key]
for key in ("Attribute", "Object", "Tag")
if (key in misp_event and misp_event[key])
}
}
except pymisp.PyMISPError as e:
logger.error("Error parsing the report: %s", str(e))
return {"error": "Error parsing the report: {}".format(e)}
