def deleteEvent(iUUID="", iEventID=""):
try:
mispDB = pm.ExpandedPyMISP(url=gv._MISP_URL,
key=gv._MISP_KEY,
ssl=gv._MISP_VERIFYCERT,
debug=gv._DEBUG)
event_id = ""
if iUUID != "":
kwargs = {"uuid": iUUID}
result = mispDB.search(controller='events',
return_format='json',
limit=1,
**kwargs)
for val in result:
event_id = val["Event"]["id"]
elif iEventID != "":
event_id = iEventID
if event_id != "":
if gv._DEBUG:
print(
"f(x) deleteEvent: ATTEMPTING TO DELETE EVENT [IF EXISTS]: {}"
.format(event_id))
mispDB.delete_event(event_id)
else:
print(
"f(x) deleteEvent: EMPTY EVENT_ID FOUND. NO DELETION MADE\niUUID: {}\niEventID: {}\nRETURNED EVENT ID [IF APPLICABLE]: {}"
.format(iUUID, iEventID, event_id))
except Exception as e:
print(e)
def run(self):
api_key = secrets.get_secret(self.api_key_name)
if not api_key:
raise AnalyzerRunException(
f"no MISP API key retrieved with name: {self.api_key_name}")
if not self.url_name:
raise AnalyzerRunException(
f"no MISP URL retrieved, key value: {self.url_key_name}")
misp_instance = pymisp.ExpandedPyMISP(self.url_name, api_key)
# debug=True)
# we check only for events not older than 90 days and max 50 results
now = datetime.datetime.now()
date_from = now - datetime.timedelta(days=90)
params = {
# even if docs say to use "values",...
# .. at the moment it works correctly only with "value"
"value": self.observable_name,
"type_attribute": [self.observable_classification],
"date_from": date_from.strftime("%Y-%m-%d %H:%M:%S"),
"limit": 50,
"enforce_warninglist": True,
}
if self.observable_classification == "hash":
params["type_attribute"] = ["md5", "sha1", "sha256"]
result_search = misp_instance.search(**params)
if isinstance(result_search, dict):
errors = result_search.get("errors", [])
if errors:
raise AnalyzerRunException(errors)
return {"result_search": result_search, "instance_url": self.url_name}
def __init__(
self,
url: str,
key: str,
ssl: bool = True,
tags: Optional[List[str]] = None,
artifact_types: Optional[List[Type[Artifact]]] = None,
filter_string: Optional[str] = None,
allowed_sources: Optional[List[str]] = None,
include_artifact_source_name: bool = True,
include_event_id_in_info: bool = False,
):
"""MISP operator."""
self.api = pymisp.ExpandedPyMISP(url, key, ssl)
if tags:
self.tags = tags
else:
self.tags = ["type:OSINT"]
self.event_info = "{source_name}"
self.url = url
super().__init__(artifact_types, filter_string, allowed_sources)
self.artifact_types = artifact_types or [Domain, Hash, IPAddress, URL]
self.include_artifact_source_name = include_artifact_source_name
self.include_event_id_in_info = include_event_id_in_info
def __init__(self,
url,
key,
ssl=True,
tags=None,
artifact_types=None,
filter_string=None,
allowed_sources=None):
"""MISP operator."""
self.api = pymisp.ExpandedPyMISP(url, key, ssl)
if tags:
self.tags = tags
else:
self.tags = ['type:OSINT']
self.event_info = 'ThreatIngestor Event: {source_name}'
super(Plugin, self).__init__(artifact_types, filter_string,
allowed_sources)
self.artifact_types = artifact_types or [
threatingestor.artifacts.Domain,
threatingestor.artifacts.Hash,
threatingestor.artifacts.IPAddress,
threatingestor.artifacts.URL,
threatingestor.artifacts.YARASignature,
]
def pushToMISP(event,
iUpdate=False,
mURL="",
mKey="",
mVerifycert="",
mDebug=""):
try:
mispDB = pm.ExpandedPyMISP(url=mURL,
key=mKey,
ssl=mVerifycert,
debug=mDebug)
if gv._DEBUG:
print("f(x) pushToMISP(): PUSHING EVENT TO MISP: {}".format(event))
# NEW EVENT
if iUpdate == False:
event.publish()
event = mispDB.add_event(event, pythonify=True)
else:
event.publish()
event = mispDB.update_event(event, pythonify=True)
except Exception as e:
if gv._DEBUG:
print("f(x) pushToMISP() ERROR: {}".format(e))
pass
finally:
print("f(x) pushToMISP: CREATED MISP EVENT: {}".format(event.info))
return True
def init():
# If we have an old settings.py file then this variable won't exist
global valid_attribute_distributions
try:
valid_attribute_distributions = valid_attribute_distribution_levels
except Exception:
valid_attribute_distributions = ['0', '1', '2', '3', '4', '5']
return pymisp.ExpandedPyMISP(url, key, ssl)
def run(
config: Subview,
logging: Subview,
inq: JoinableQueue,
subscribe_callback: Callable,
unsubscribe_callback: Callable,
):
global logger, filter_config, workers
logger = threatbus.logger.setup(logging, __name__)
config = config[plugin_name]
try:
validate_config(config)
except Exception as e:
logger.fatal("Invalid config for plugin {}: {}".format(
plugin_name, str(e)))
filter_config = config["filter"].get(list)
# start Attribute-update receiver
if config["zmq"].get():
workers.append(ZmqReceiver(config["zmq"], inq))
elif config["kafka"].get():
workers.append(KafkaReceiver(config["kafka"], inq))
# bind to MISP
if config["api"].get(dict):
# TODO: MISP instances shall subscribe themselves to threatbus and each
# subscription shall have an individual outq and receiving thread for intel
# updates.
host, key, ssl = (
config["api"]["host"].get(),
config["api"]["key"].get(),
config["api"]["ssl"].get(),
)
try:
global misp, lock
lock.acquire()
misp = pymisp.ExpandedPyMISP(url=host, key=key, ssl=ssl)
lock.release()
except Exception:
# TODO: log individual error per MISP subscriber
logger.error(
f"Cannot subscribe to MISP at {host}, using SSL: {ssl}")
lock.release()
if not misp:
logger.error("Failed to start MISP plugin")
return
else:
logger.warning(
"Starting MISP plugin without API connection, cannot report back sightings or request snapshots."
)
outq = JoinableQueue()
subscribe_callback("stix2/sighting", outq)
workers.append(SightingsPublisher(outq))
for w in workers:
w.start()
logger.info("MISP plugin started")
def __init__(self):
self.misp_backend = pymisp.ExpandedPyMISP(
OCD_DTL_MISP_HOST,
OCD_DTL_MISP_API_KEY,
ssl=OCD_DTL_MISP_USE_SSL,
timeout=60,
)
# Init a common thread pool to run sync operation in background
self.thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=OCD_DTL_MISP_WORKER)
def getOrgEvents(iOrgID):
try:
misp = pm.ExpandedPyMISP(gv._MISP_URL, gv._MISP_KEY,
gv._MISP_VERIFYCERT)
kwargs = {"org_id": iOrgID}
# result = misp.search('events', published=0, **kwargs)
result = misp.search('events', published=1, **kwargs)
return result
except Exception as e:
print(e)
def connect_misp(host: str, key: str, ssl: bool):
"""
Connects to a MISP instance. Returns the pymisp.MISP object.
@param host The MISP URL to connect to
@param key The MISP API key
@param ssl Boolean flag to use SSL
"""
try:
return pymisp.ExpandedPyMISP(url=host, key=key, ssl=ssl)
except Exception:
logger.critical(
f"Cannot connect to MISP at '{host}', using SSL '{ssl}'")
def run(
config: DynaBox,
logging: DynaBox,
inq: JoinableQueue,
subscribe_callback: Callable,
unsubscribe_callback: Callable,
):
global logger, filter_config, workers
logger = threatbus.logger.setup(logging, __name__)
assert plugin_name in config, f"Cannot find configuration for {plugin_name} plugin"
config = config[plugin_name]
filter_config = config.filter if "filter" in config else {}
# start Attribute-update receiver
if "zmq" in config:
workers.append(ZmqReceiver(config.zmq, inq))
elif "kafka" in config:
workers.append(KafkaReceiver(config.kafka, inq))
# bind to MISP
if "api" in config:
# TODO: MISP instances shall subscribe themselves to threatbus and each
# subscription shall have an individual outq and receiving thread for intel
# updates.
try:
global misp, lock
lock.acquire()
misp = pymisp.ExpandedPyMISP(
url=config.api.host, key=config.api.key, ssl=config.api.ssl
)
lock.release()
except Exception:
# TODO: log individual error per MISP subscriber
logger.error(
f"Cannot subscribe to MISP at {config.api.host}, using SSL: {config.api.ssl}"
)
lock.release()
if not misp:
logger.error("Failed to start MISP plugin")
return
else:
logger.warning(
"Starting MISP plugin without API connection, cannot report back sightings or request snapshots."
)
outq = JoinableQueue()
subscribe_callback("stix2/sighting", outq)
workers.append(SightingsPublisher(outq))
for w in workers:
w.start()
logger.info("MISP plugin started")
def __init__(self, url, key, ssl=True, name='Unnamed', proxies=None):
self.misp_connections = []
if type(url) is list:
for idx, server in enumerate(url):
verify = True
# Given ssl parameter is a list
if isinstance(ssl, list):
if isinstance(ssl[idx], str) and os.path.isfile(ssl[idx]):
verify = ssl[idx]
elif isinstance(ssl[idx], str) and not os.path.isfile(ssl[idx]) and ssl[idx] != "":
raise CertificateNotFoundError('Certificate not found under {}.'.format(ssl[idx]))
elif isinstance(ssl[idx], bool):
verify = ssl[idx]
# Do the same checks again, for the non-list type
elif isinstance(ssl, str) and os.path.isfile(ssl):
verify = ssl
elif isinstance(ssl, str) and not os.path.isfile(ssl) and ssl != "":
raise CertificateNotFoundError('Certificate not found under {}.'.format(ssl))
elif isinstance(ssl, bool):
verify = ssl
self.misp_connections.append(pymisp.ExpandedPyMISP(url=server,
key=key[idx],
ssl=verify,
proxies=proxies))
else:
verify = True
if isinstance(ssl, str) and os.path.isfile(ssl):
verify = ssl
elif isinstance(ssl, str) and not os.path.isfile(ssl) and ssl != "":
raise CertificateNotFoundError('Certificate not found under {}.'.format(ssl))
elif isinstance(ssl, bool):
verify = ssl
self.misp_connections.append(pymisp.ExpandedPyMISP(url=url,
key=key,
ssl=verify,
proxies=proxies))
self.misp_name = name
def __init__(self, configfile):
self.load_config(configfile)
if not self.config['verify_ssl']:
import urllib3
urllib3.disable_warnings()
# logging.warning('InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings')
try:
self.misp = pymisp.ExpandedPyMISP(self.config['url'], self.config['key'], self.config['verify_ssl'])
# except TimeoutError:
# logging.error("Timeout error connecting to MISP instance")
# sys.exit(1)
# except OSError as e:
# print(e)
except pymisp.exceptions.PyMISPError as e:
logging.error(e)
sys.exit(1)
def uuidSearch(iUUID):
try:
retVal = 0
mispDB = pm.ExpandedPyMISP(url=gv._MISP_URL,
key=gv._MISP_KEY,
ssl=gv._MISP_VERIFYCERT,
debug=gv._DEBUG)
kwargs = {"uuid": iUUID}
result = mispDB.search(controller='events',
return_format='json',
limit=1,
**kwargs)
retVal = int(len(result))
return retVal
except Exception as e:
print(e)
def run(config, logging, inq, subscribe_callback, unsubscribe_callback):
global logger
logger = threatbus.logger.setup(logging, __name__)
config = config[plugin_name]
try:
validate_config(config)
except Exception as e:
logger.fatal("Invalid config for plugin {}: {}".format(
plugin_name, str(e)))
if config["api"].get():
host, key, ssl = (
config["api"]["host"].get(),
config["api"]["key"].get(),
config["api"]["ssl"].get(),
)
try:
global misp, lock
lock.acquire()
misp = pymisp.ExpandedPyMISP(url=host, key=key, ssl=ssl)
lock.release()
except Exception:
# TODO: log individual error per MISP subscriber
logger.error(
f"Cannot subscribe to MISP at {host}, using SSL: {ssl}")
lock.release()
# TODO: MISP instances shall subscribe themselves to threatbus and each subscription shall have an individual outq and receiving thread for intel updates.
outq = Queue()
subscribe_callback("threatbus/sighting", outq)
if misp:
threading.Thread(target=publish_sightings, args=(outq, ),
daemon=True).start()
if config["zmq"].get():
threading.Thread(target=receive_zmq,
args=(config["zmq"], inq),
daemon=True).start()
if config["kafka"].get():
threading.Thread(target=receive_kafka,
args=(config["kafka"], inq),
daemon=True).start()
logger.info("MISP plugin started")
def main(args):
logging.info('Passing through misp_exporter')
with open(args.fingerprint) as file:
try:
data = json.load(file)
except Exception:
logging.error('Fingerprint not compatible.')
return
misp = pymisp.ExpandedPyMISP(args.misp_url,
args.misp_key,
ssl=False,
debug=True)
event = pymisp.MISPEvent(strict_validation=True)
event.info = args.event_info
event.distribution = args.distribution
event.threat_level_id = args.threat_level
event.analysis = args.analysis_level
event.add_tag(tag='validated')
ddos = pymisp.MISPObject('ddos', strict=True)
logging.info('Fingerprint processed: {}'.format(args.fingerprint))
if 'attack_vector' in data:
av = data['attack_vector'][0]
snort = {
'protocol': [],
'src_ip': [],
'src_port': [],
'dst_port': [],
'options': [],
}
df = find_ips(av)
logging.info('IPs found: {}'.format(len(df['ip'])))
subnets = utils.smart_aggregate(df)
logging.info('The IPs were summarized in: {} subnets'.format(
len(subnets)))
if args.subnets:
add_attributes(event, subnets, 'ip-src', 'attacker subnet')
ddos.add_attributes('ip-src', *subnets)
snort['src_ip'] += subnets
else:
add_attributes(event, df['ip'], 'ip-src', 'attacker ip')
ddos.add_attributes('ip-src', *df['ip'])
snort['src_ip'] += df['ip'].tolist()
if 'srcport' in av:
sport = find_attr(av['srcport'], 'srcport')
add_attributes(event, sport['srcport'], 'port',
'source port of attack')
ddos.add_attributes('src-port', *sport['srcport'])
snort['src_port'] += sport['srcport'].tolist()
if 'dstport' in av:
dport = find_attr(av['dstport'], 'dstport')
add_attributes(event, dport['dstport'], 'port',
'destination port of attack')
ddos.add_attributes('dst-port', *sport['dstport'])
snort['dst_port'] += sport['dstport']
if 'ip_proto' in av:
proto4 = find_attr(av['ip_proto'], 'ip_proto')
add_attributes(event, proto4['ip_proto'], 'other',
'4 level protocol of attack')
ddos.add_attributes('protocol', *proto4['ip_proto'])
snort['protocol'] += proto4['ip_proto'].tolist()
if 'dns_qry_name' in av:
snort['options'] += snort_content(av['dns_qry_name'])
if 'http_request' in av:
snort['options'] += snort_content(av['http_request'])
if 'http_response' in av:
snort['options'] += snort_content(av['http_response'])
if 'http_user_agent' in av:
snort['options'] += snort_content(av['http_user_agent'])
if 'icmp_type' in av:
snort['options'] += snort_itype(av['icmp_type'])
if 'icmp_code' in av:
snort['options'] += snort_icode(av['icmp_code'])
if 'ip_ttl' in av:
snort['options'] += snort_ttl(av['ip_ttl'])
if 'ntp_priv_reqcode' in av:
snort['options'] += snort_content(av['ntp_priv_reqcode'])
if 'tcp_flags' in av:
snort['options'] += snort_flags(av['tcp_flags'])
if 'tags' in av:
for tag in av['tags']:
event.add_tag(tag=tag)
if 'duration_sec' in av:
add_attribute_comment(event, 'duration_sec', av['duration_sec'])
if 'total_dst_ports' in av:
add_attribute_comment(event, 'total_dst_ports',
av['total_dst_ports'])
if 'total_ips' in av:
add_attribute_comment(event, 'total_ips', av['total_ips'])
if 'total_packets' in av:
add_attribute_comment(event, 'total_packets', av['total_packets'])
add_attribute_snort(event, snort)
event.add_tag(tag='ddos attack')
if 'amplifiers' in data:
amp = data['amplifiers'][0]
dfa = find_ips(amp)
logging.info('Amplifier IPs found: {}'.format(len(dfa['ip'])))
subnetsa = utils.smart_aggregate(dfa)
logging.info('The IPs were summarized in: {} subnets'.format(
len(subnetsa)))
if args.subnets:
add_attributes(event, subnetsa, 'ip-src', 'amplifier subnet')
for elem in subnetsa:
ddos.add_attribute('ip-src', elem, to_ids=0)
else:
add_attributes(event, dfa['ip'], 'ip-src', 'amplifier ip')
for elem in dfa['ip']:
ddos.add_attribute('ip-src', elem, to_ids=0)
event.add_tag(tag='ddos amplification attack')
p = pathlib.Path(args.fingerprint)
comment = 'DDoS fingerprint json file generated by dissector'
event.add_attribute(type='attachment',
value=p.name,
data=p,
comment=comment)
event.add_object(ddos, pythonify=True)
event = misp.add_event(event, pythonify=True)
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info(
"started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name)
)
report = general.get_basic_report_template(analyzer_name)
try:
api_key_name = additional_config_params.get("api_key_name", "MISP_KEY")
api_key = secrets.get_secret(api_key_name)
if not api_key:
raise AnalyzerRunException(
"no MISP API key retrieved, key value: {}".format(api_key_name)
)
url_key_name = additional_config_params.get("url_key_name", "MISP_URL")
url_name = secrets.get_secret(url_key_name)
if not url_name:
raise AnalyzerRunException(
"no MISP URL retrieved, key value: {}".format(url_key_name)
)
misp_instance = pymisp.ExpandedPyMISP(url_name, api_key) # debug=True)
# we check only for events not older than 90 days and max 50 results
now = datetime.datetime.now()
date_from = now - datetime.timedelta(days=90)
params = {
# even if docs say to use "values",...
# .. at the moment it works correctly only with "value"
"value": observable_name,
"type_attribute": [observable_classification],
"date_from": date_from.strftime("%Y-%m-%d %H:%M:%S"),
"limit": 50,
"enforce_warninglist": True,
}
if observable_classification == "hash":
params["type_attribute"] = ["md5", "sha1", "sha256"]
result_search = misp_instance.search(**params)
if isinstance(result_search, dict):
errors = result_search.get("errors", [])
if errors:
raise AnalyzerRunException(errors)
report_to_give = {"result_search": result_search, "instance_url": url_name}
report["report"] = report_to_give
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e)
)
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e)
)
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
# pprint.pprint(report)
logger.info(
"ended analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name)
)
return report
def create_events(pulse_or_list, author=False, server=False, key=False, misp=False, blacklist_file = None, distribution=0, threat_level=4,
analysis=2, publish=True, tlp=True, discover_tags=False, to_ids=False, author_tag=False,
bulk_tag=None, dedup_titles=False, stop_on_error=False):
"""
Parse a Pulse or a list of Pulses and add it/them to MISP if server and key are present
:param pulse_or_list: a Pulse or list of Pulses as returned by `get_pulses`
:param author: Prepend the author to the Pulse name
:type author: Boolean
:param server: MISP server URL
:param key: MISP API key
:param misp: MISP connection object
:type misp: :class:`pymisp.PyMISP`
:param blacklist_file: The name of a file containing a list of pulse title regexes for pulses taht should be ignored
:type blacklist_file: String
:param distribution: distribution of the MISP event (0-4)
:param threat_level: threat level of the MISP object (1-4)
:param analysis: analysis stae of the MISP object (0-2)
:param publish: Is the MISP event should be published?
:type publish: Boolean
:param tlp: Add TLP level tag to event
:type tlp: Boolean
:param discover_tags: discover MISP tags from Pulse tags
:type discover_tags: Boolean
:param to_ids: Flag pulse attributes as being sent to an IDS
:type to_ids: Boolean
:param author_tag: Add the pulse author as an event tag
:type author_tag: Boolean
:param bulk_tag: A tag that will be added to all events for categorization (e.g. OTX)
:type bulk_tag: String
:param dedup_titles: Search MISP for an existing event title and update it, rather than create a new one
:type dedup_titles: Boolean
:return: a dict or a list of dict with the selected attributes
"""
if not misp and (server and key):
log.debug("Connection to MISP instance: {}".format(server))
try:
misp = pymisp.ExpandedPyMISP(server, key, ssl=False)
except pymisp.PyMISPError as ex:
raise ImportException("Cannot connect to MISP instance: {}".format(ex.message))
except Exception as ex:
raise ImportException("Cannot connect to MISP instance, unknown exception: {}".format(ex.message))
# Let's load the list of blacklisted pulse titles, if exists
global _otx_blacklisted_titles
if blacklist_file and not _otx_blacklisted_titles:
if not os.path.isfile(blacklist_file):
log.warn("The blacklisted pulse titles file that has been provided does not exist: {}".format(blacklist_file))
else:
with open(blacklist_file) as f:
lines = f.read().splitlines()
for line in lines:
if line != "":
_otx_blacklisted_titles.append(re.compile(line))
# Let's load in cache all MISP tags available on the instance
global _otx_tags_cache
if _otx_tags_cache is None:
_otx_tags_cache = misp.tags()
if discover_tags:
def get_tag_name(complete):
parts = complete.split('=')
if not len(parts):
return complete
last = parts[-1]
if not len(last):
return complete
if last[0] == '"':
last = last[1:]
if last[-1] == '"':
last = last[:-1]
return last.lower()
tags = dict()
for tag in _otx_tags_cache:
tags[get_tag_name(tag['name'])] = tag['name']
misp.discovered_tags = tags
if isinstance(pulse_or_list, (list, tuple)) or inspect.isgenerator(pulse_or_list):
misp_events = []
for pulse in pulse_or_list:
try:
misp_event = create_events(pulse, author=author, server=server, key=key, misp=misp,
distribution=distribution, threat_level=threat_level, analysis=analysis,
publish=publish, tlp=tlp, to_ids=to_ids, author_tag=author_tag,
bulk_tag=bulk_tag, dedup_titles=dedup_titles, stop_on_error=stop_on_error)
if misp_event:
misp_events.append(misp_event)
except Exception as ex:
if stop_on_error:
raise
name = ''
if pulse and 'name' in pulse:
name = pulse['name']
log.error("Cannot import pulse {}: {}".format(name, ex))
return misp_events
pulse = pulse_or_list
if author:
event_name = pulse['author_name'] + ' | ' + pulse['name']
else:
event_name = pulse['name']
event_name = event_name.strip()
try:
dt = date_parser.parse(pulse['created'])
except (ValueError, OverflowError):
log.error("Cannot parse Pulse 'created' date.")
dt = datetime.utcnow()
event_date = dt.strftime('%Y-%m-%d')
# Let's ignore the pulse if its title is blacklisted
for regex in _otx_blacklisted_titles:
if regex.match(pulse['name']):
log.info("## *** IGNORED *** {name} - {date}".format(name=event_name, date=event_date))
return None
log.info("## {name} - {date}".format(name=event_name, date=event_date))
result_event = {
'name': event_name,
'date': event_date,
'tags': list(),
'attributes': {
'hashes': {
'md5': list(),
'sha1': list(),
'sha256': list(),
'imphash': list(),
'pehash': list()
},
'hostnames': list(),
'domains': list(),
'urls': list(),
'ips': list(),
'emails': list(),
'mutexes': list(),
'references': list(),
'cves': list(),
'filenames': list(),
'yara': list()
},
}
if misp:
if not dedup_titles:
event = pymisp.MISPEvent()
event.distribution = distribution
event.threat_level_id = threat_level
event.analysis = analysis
event.info = event_name
event.date = dt
event = misp.add_event(event)['Event']
else:
event = ''
# Search MISP for the title
result = misp.search(eventinfo=event_name, metadata=True)
if len(result) == 0:
event = pymisp.MISPEvent()
event.distribution = distribution
event.threat_level = threat_level
event.analysis = analysis
event.info = event_name
event.set_date(dt)
event = misp.add_event(event)['Event']
else:
for evt in result:
# If it exists, set 'event' to the event
if evt['Event']['info'] == event_name:
if 'SharingGroup' in evt['Event']:
del evt['Event']['SharingGroup'] # This deletes the SharingGroup from the list, thx SparkyNZL
event = evt['Event']
break
if event == '':
# Event not found, even though search results were returned
# Build new event
event = pymisp.MISPEvent()
event.distribution = distribution
event.threat_level = threat_level
event.analysis = analysis
event.info = event_name
event.set_date(dt)
misp.add_event(event)
time.sleep(0.2)
if tlp:
tag = None
if 'TLP' in pulse:
tag = "tlp:{}".format(pulse['TLP'])
elif 'tlp' in pulse:
tag = "tlp:{}".format(pulse['tlp'])
if tag is not None:
log.info("\t - Adding tag: {}".format(tag))
tag_event(misp, event, tag)
result_event['tags'].append(tag)
if author_tag:
tag_event(misp, event, pulse['author_name'])
if bulk_tag is not None:
tag_event(misp, event, bulk_tag)
if misp and hasattr(misp, 'discovered_tags') and 'tags' in pulse:
for pulse_tag in pulse['tags']:
if pulse_tag.lower() in misp.discovered_tags:
tag = misp.discovered_tags[pulse_tag.lower()]
log.info("\t - Adding tag: {}".format(tag))
tag_event(misp, event, tag)
result_event['tags'].append(tag)
if 'references' in pulse:
for reference in pulse['references']:
if reference:
log.info("\t - Adding external analysis link: {}".format(reference))
if misp:
a = pymisp.MISPAttribute()
a.category = "External analysis"
a.type = 'link'
a.value = reference
add_attribute(misp, event, a)
result_event['attributes']['references'].append(reference)
if misp and 'description' in pulse and isinstance(pulse['description'], six.text_type) and pulse['description']:
log.info("\t - Adding external analysis comment")
a = pymisp.MISPAttribute()
a.category = 'External analysis'
a.type = 'comment'
a.value = pulse['description']
add_attribute(misp, event, a)
for ind in pulse['indicators']:
ind_type = ind['type']
ind_val = ind['indicator']
a = pymisp.MISPAttribute()
a.value = ind_val
a.to_ids = to_ids
if 'description' in ind and isinstance(ind['description'], six.text_type) and ind['description']:
a.comment = ind['description']
if ind_type == 'FileHash-SHA256':
log.info("\t - Adding SHA256 hash: {}".format(ind_val))
a.category = 'Artifacts dropped'
a.type = 'sha256'
add_attribute(misp, event, a)
result_event['attributes']['hashes']['sha256'].append(ind_val)
elif ind_type == 'FileHash-SHA1':
log.info("\t - Adding SHA1 hash: {}".format(ind_val))
a.category = 'Artifacts dropped'
a.type = 'sha1'
add_attribute(misp, event, a)
result_event['attributes']['hashes']['sha1'].append(ind_val)
elif ind_type == 'FileHash-MD5':
log.info("\t - Adding MD5 hash: {}".format(ind_val))
a.category = 'Artifacts dropped'
a.type = 'md5'
add_attribute(misp, event, a)
result_event['attributes']['hashes']['md5'].append(ind_val)
elif ind_type == 'FileHash-IMPHASH':
log.info("\t - Adding IMPHASH hash: {}".format(ind_val))
a.category = 'Artifacts dropped'
a.type = 'imphash'
add_attribute(misp, event, a)
result_event['attributes']['hashes']['imphash'].append(ind_val)
elif ind_type == 'FileHash-PEHASH':
log.info("\t - Adding PEHASH hash: {}".format(ind_val))
a.category = 'Artifacts dropped'
a.type = 'pehash'
add_attribute(misp, event, a)
result_event['attributes']['hashes']['pehash'].append(ind_val)
elif ind_type == 'YARA':
ind_title = ind.get('title', ind_val)
ind_desc = ind.get('description', '')
if ind_title == '':
ind_title = ind_val
if not ind_desc == '':
a.comment = ind_desc
else:
a.comment = "{} {}".format(ind_title, ind_desc)
ind_val = ind.get('content', None)
if ind_val is None or ind_val == "":
log.warning("YARA indicator is empty: %s" % ind_title)
continue
log.info("\t - Adding YARA rule: {}".format(ind_title))
a.category = 'Artifacts dropped'
a.type = 'yara'
a.value = ind_val
add_attribute(misp, event, a)
result_event['attributes']['yara'].append({'title': ind_title, 'content': ind_val})
elif ind_type == 'Mutex':
log.info("\t - Adding mutex: {}".format(ind_val))
a.category = 'Artifacts dropped'
a.type = 'mutex'
add_attribute(misp, event, a)
result_event['attributes']['mutexes'].append(ind_val)
elif ind_type == 'FilePath':
log.info("\t - Adding filename: {}".format(ind_val))
a.category = 'Artifacts dropped'
a.type = 'filename'
add_attribute(misp, event, a)
result_event['attributes']['filenames'].append(ind_val)
elif ind_type == 'URI' or ind_type == 'URL':
log.info("\t - Adding URL: {}".format(ind_val))
a.category = 'Network activity'
a.type = 'url'
add_attribute(misp, event, a)
result_event['attributes']['urls'].append(ind_val)
elif ind_type == 'domain':
log.info("\t - Adding domain: {}".format(ind_val))
a.category = 'Network activity'
a.type = 'domain'
add_attribute(misp, event, a)
result_event['attributes']['domains'].append(ind_val)
elif ind_type == 'hostname':
log.info("\t - Adding hostname: {}".format(ind_val))
a.category = 'Network activity'
a.type = 'hostname'
add_attribute(misp, event, a)
result_event['attributes']['hostnames'].append(ind_val)
elif ind_type == 'IPv4' or ind_type == 'IPv6':
log.info("\t - Adding ip: {}".format(ind_val))
a.category = 'Network activity'
a.type = 'ip-dst'
add_attribute(misp, event, a)
result_event['attributes']['ips'].append(ind_val)
elif ind_type == 'email':
log.info("\t - Adding email: {}".format(ind_val))
a.category = 'Network activity'
a.type = 'email-dst'
result_event['attributes']['emails'].append(ind_val)
add_attribute(misp, event, a)
elif ind_type == 'CVE':
log.info("\t - Adding CVE: {}".format(ind_val))
a.type = 'External analysis'
a.type = 'vulnerability'
add_attribute(misp, event, a)
result_event['attributes']['cves'].append(ind_val)
else:
log.warning("Unsupported indicator type: %s" % ind_type)
if misp and publish:
misp.publish(event)
return result_event
def pushToMISPWithAttachment(event,
iPath,
iUpdate=False,
mURL="",
mKey="",
mVerifycert="",
mDebug="",
fo=None,
peo=None,
seos=None):
try:
mispDB = pm.ExpandedPyMISP(url=mURL,
key=mKey,
ssl=mVerifycert,
debug=mDebug)
if gv._DEBUG:
print("f(x) pushToMISPWithAttachment() EVENT: {}".format(event))
# CREATE EVENT
if iUpdate == False:
event.publish()
mispDB.add_event(event, pythonify=True)
else:
event.publish()
mispDB.update_event(event, pythonify=True)
# # ADD ATTACHMENT
if iUpdate == False:
# myPath = iPath
# fo = None
# peo = None
# seos = None
# for f in glob.glob(myPath):
# try:
# fo , peo, seos = make_binary_objects(f)
# except Exception as e:
# continue
if seos:
try:
for s in seos:
try:
mispDB.add_object(event.uuid, s)
except Exception as e:
continue
except Exception as e:
pass
if peo:
try:
mispDB.add_object(event.uuid, peo, pythonify=True)
for ref in peo.ObjectReference:
try:
mispDB.add_object_reference(ref)
except Exception as e:
continue
except Exception as e:
pass
if fo:
try:
mispDB.add_object(event.uuid, fo)
for ref in fo.ObjectReference:
try:
mispDB.add_object_reference(ref, pythonify=True)
except Exception as e:
continue
except Exception as e:
pass
# UPDATE EVENT AFTER ADDING ATTACHMENT
try:
event.publish()
mispDB.publish(event)
print("f(x) pushToMISPWithAttachment: CREATED MISP EVENT: {}".
format(event.info))
except Exception as e:
pass
except Exception as e:
if gv._DEBUG:
print("f(x) pushToMISPWithAttachment() ERROR: {}".format(e))
pass
# gv._THREAD_LIST.append(uexecutor.submit(pushToMISPWithAttachment,event, iPath, iUpdate, mURL, mKey, mVerifycert, mDebug, fo, peo, seos))
finally:
return True
def run(self, results):
"""Run analysis.
@return: MISP results dict.
"""
url = self.options.get("url", "")
apikey = self.options.get("apikey", "")
if not url or not apikey:
log.error("MISP URL or API key not configured.")
return
self.misp = pymisp.ExpandedPyMISP(url, apikey, False, "json")
self.threads = self.options.get("threads", "")
if not self.threads:
self.threads = 5
self.iocs = deque()
self.misper = dict()
try:
if self.options.get("upload_iocs", False) and results.get(
"malscore", 0) >= self.options.get("min_malscore", 0):
distribution = int(self.options.get("distribution", 0))
threat_level_id = int(self.options.get("threat_level_id", 4))
analysis = int(self.options.get("analysis", 0))
tag = self.options.get("tag") or "CAPEv2"
info = self.options.get("title", "")
upload_sample = self.options.get("upload_sample")
malfamily = ""
if results.get("detections", ""):
malfamily = results["detections"]
event = MISPEvent()
response = self.misp.search(
"attributes",
value=results["target"]["file"]["sha256"],
return_format="json")
if response.get("Attribute", []):
res = self.misp.get_event(
response["Attribute"][0]["event_id"])["Event"]
event.load(res)
else:
event.distribution = distribution
event.threat_level_id = threat_level_id
event.analysis = analysis
event.info = "{} {} - {}".format(
info, malfamily,
results.get('info', {}).get('id'))
event = self.misp.add_event(event, pythonify=True)
# Add a specific tag to flag Cuckoo's event
if tag:
self.misp.tag(event["uuid"], tag)
#ToDo?
self.signature(results, event)
self.sample_hashes(results, event)
self.all_network(results, event)
self.dropped_files(results, event)
#DeprecationWarning: Call to deprecated method upload_samplelist. (Use MISPEvent.add_attribute with the expand='binary' key)
if upload_sample:
target = results.get("target", {})
f = target.get("file", {})
if target.get("category") == "file" and f:
with open(f["path"], 'rb') as f:
event.add_attribute(
'malware-sample',
value=os.path.basename(f["path"]),
data=BytesIO(f.read()),
expand='binary',
comment="Sample run",
)
if results.get("target", {}).get(
"url",
"") and results["target"]["url"] not in whitelist:
self.misp.add_named_attribute(event, 'url',
[results["target"]["url"]])
# ToDo migth be outdated!
#if self.options.get("ids_files", False) and "suricata" in results.keys():
# for surifile in results["suricata"]["files"]:
# if "file_info" in surifile.keys():
# self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]})
# self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]})
# self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]})
if self.options.get(
"mutexes", False
) and "behavior" in results and "summary" in results[
"behavior"]:
if "mutexes" in results.get("behavior",
{}).get("summary", {}):
for mutex in results["behavior"]["summary"]["mutexes"]:
if mutex not in whitelist:
self.misp.add_mutex(event, mutex)
if self.options.get(
"registry", False
) and "behavior" in results and "summary" in results[
"behavior"]:
if "read_keys" in results["behavior"].get("summary", {}):
for regkey in results["behavior"]["summary"][
"read_keys"]:
self.misp.add_regkey(event, regkey)
#Make event public
if self.options.get("published", True):
event.published = True
self.misp.publish(event)
except Exception as e:
log.error("Failed to generate JSON report: %s" % e, exc_info=True)
def __init__(self):
self.misp = pymisp.ExpandedPyMISP(MISP_URL, MISP_KEY, False)
