def virus_total_obj(api_key, indicators):
'''Build our VirusTotal report object, File object, and AV signature objects
and link them appropriately.
indicator: Indicator hash to search in VT for
'''
vtr_report = VTReportObject(api_key["virustotal"]["key"], indicators)
report_objects = []
report_objects.append(vtr_report)
av_report = vtr_report._report
if vtr_report._resource_type == 'file':
file_object = pymisp.MISPObject(name="file")
file_object.add_attribute("md5", value=raw_report["md5"])
file_object.add_attribute("sha1", value=raw_report["sha1"])
file_object.add_attribute("sha256", value=raw_report["sha256"])
vtr_report.add_reference(referenced_uuid=file_object.uuid, relationship_type="report of")
report_objects.append(file_object)
elif vtr_report._resource_type == "url":
parsed = urlsplit(indicator)
url_object = pymisp.MISPObject(name="url")
url_object.add_attribute("url", value=parsed.geturl())
url_object.add_attribute("host", value=parsed.hostname)
url_object.add_attribute("scheme", value=parsed.scheme)
url_object.add_attribute("port", value=parsed.port)
vtr_report.add_reference(referenced_uuid=url_object.uuid, relationship_type="report of")
report_objects.append(url_object)
for antivirus in av_report["scans"]:
if av_report["scans"][antivirus]["detected"]:
av_object = pymisp.MISPObject(name="av-signature")
av_object.add_attribute("software", value=antivirus)
signature_name = raw_report["scans"][antivirus]["result"]
av_object.add_attribute("signature", value=signature_name, disable_correlation=True)
vtr_report.add_reference(referenced_uuid=av_object.uuid, relationship_type="included-in")
report_objects.append(av_object)
def parse_object(self, o, labels):
object_type = self.get_misp_type(labels)
object_category = self.get_misp_category(labels)
stix_type = o._type
misp_object = pymisp.MISPObject(object_type)
misp_object['meta-category'] = object_category
if stix_type == 'indicator':
pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
pattern[0] = pattern[0][2:]
pattern[-1] = pattern[-1][:-2]
attributes = objects_mapping[object_type]['pattern'](pattern)
if stix_type == 'observed-data':
observable = o.get('objects')
attributes = objects_mapping[object_type]['observable'](observable)
for attribute in attributes:
misp_object.add_attribute(**attribute)
misp_object.to_ids = bool(labels[1].split('=')[1])
self.misp_event.add_object(**misp_object)
def parse_course_of_action(self, o):
misp_object = pymisp.MISPObject('course-of-action')
if 'name' in o:
attribute = {
'type': 'text',
'object_relation': 'name',
'value': o.get('name')
}
misp_object.add_attribute(**attribute)
else:
return
if 'description' in o:
attribute = {
'type': 'text',
'object_relation': 'description',
'value': o.get('description')
}
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
def misp_send(self, strMISPEventID, strInput, strInfo, strUsername):
# Establish communication with MISP
# The main processing block.
try:
objects = []
#get comments and tags from string input
str_comment, tags = self.get_comm_and_tags(strInput)
if tags == None:
self.misp_logger.info('Irate not in Tags: %s equals None' %
tags)
response = None
return response
#setup misp objects
mispobj_email = pymisp.MISPObject(name="email")
mispobj_file = pymisp.MISPObject(name="file")
mispobj_files = {}
mispobj_domainip = pymisp.MISPObject(name="domain-ip")
url_no = 0
file_no = 0
mispobj_urls = {}
#process input
for line in strInput.splitlines():
if ("domain:" in line.lower()
): #Catch domain and add to domain/IP object
mispobj_domainip = pymisp.MISPObject(name="domain-ip")
vals = line.split(":", 1)
mispobj_domainip.add_attribute("domain",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_domainip)
elif ("ip:" in line.lower()) or ("ip-dst:" in line.lower(
)) or ("ip-src:"
in line.lower()): #Catch IP and add to domain/IP object
if "domain:" in strInput.splitlines():
mispobj_domainip = pymisp.MISPObject(name="domain-ip")
vals = line.split(":", 1)
mispobj_domainip.add_attribute("ip",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_domainip)
else:
mispobj_network_connection = pymisp.MISPObject(
name="network-connection")
vals = line.split(":", 1)
if ("ip:" in line.lower()) or ("ip-dst:"
in line.lower()):
mispobj_network_connection.add_attribute(
"ip-dst",
type="ip-dst",
value=vals[1].strip(),
comment=str_comment)
else:
mispobj_network_connection.add_attribute(
"ip-src",
type="ip-src",
value=vals[1].strip(),
comment=str_comment)
objects.append(mispobj_network_connection)
elif ("source-email:"
in line.lower()) or ("email-source" in line.lower()) or (
"from:" in line.lower()
): #Catch email and add to email object
vals = line.split(":", 1)
mispobj_email.add_attribute("from",
value=vals[1].strip(),
comment=str_comment)
elif ("url:" in line.lower()) or (
('kit:' in line.lower() or ('creds:' in line.lower())) and
(('hxxp' in line.lower()) or ('http' in line.lower()))
): #Catch URL and add to URL object
vals = line.split(":", 1)
url = vals[1].strip()
url = refang(url)
parsed = urlparse(url)
mispobj_url = pymisp.MISPObject(name="url")
mispobj_url.add_attribute("url",
value=parsed.geturl(),
category="Payload delivery",
comment=str_comment)
if parsed.hostname:
mispobj_url.add_attribute("host",
value=parsed.hostname,
comment=str_comment)
if parsed.scheme:
mispobj_url.add_attribute("scheme",
value=parsed.scheme,
comment=str_comment)
if parsed.port:
mispobj_url.add_attribute("port",
value=parsed.port,
comment=str_comment)
mispobj_urls[url_no] = mispobj_url
url_no += 1
#Catch different hashes and add to file object
elif ("sha1:" in line.lower()) or ("SHA1:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("sha1",
value=vals[1].strip(),
comment=str_comment)
elif ("sha256:" in line.lower()) or ("SHA256:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("sha256",
value=vals[1].strip(),
comment=str_comment)
elif ("md5:" in line.lower()) or ("MD5:" in line):
vals = line.split(":", 1)
mispobj_file.add_attribute("md5",
value=vals[1].strip(),
comment=str_comment)
elif (
"subject:" in line.lower()
): #or ("subject:" in line): #Catch subject and add to email object
self.misp_logger.info('adding subject')
vals = line.split(":", 1)
mispobj_email.add_attribute("subject",
value=vals[1].strip(),
comment=str_comment)
elif ("hash|filename:" in line.lower()
): #catch hash|filename pair and add to file object
vals = line.split(":", 1)
val = vals[1].split("|")
l_hash = val[0]
l_filename = val[1]
l_mispobj_file = pymisp.MISPObject(name="file")
if len(re.findall(r"\b[a-fA-F\d]{32}\b", l_hash)) > 0:
l_mispobj_file.add_attribute("md5",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
elif len(re.findall(r'\b[0-9a-f]{40}\b', l_hash)) > 0:
l_mispobj_file.add_attribute("sha1",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
elif len(re.findall(r'\b[A-Fa-f0-9]{64}\b', l_hash)) > 0:
l_mispobj_file.add_attribute("sha256",
value=l_hash.strip(),
comment=str_comment)
l_mispobj_file.add_attribute("filename",
value=l_filename.strip(),
comment=str_comment)
mispobj_files[file_no] = l_mispobj_file
file_no += 1
#add all misp objects to List to be processed and submitted to MISP server as one.
if len(mispobj_file.attributes) > 0:
objects.append(mispobj_file)
if len(mispobj_email.attributes) > 0:
objects.append(mispobj_email)
for u_key, u_value in mispobj_urls.items():
if len(u_value.attributes) > 0:
objects.append(u_value)
for f_key, f_value in mispobj_files.items():
if len(f_value.attributes) > 0:
objects.append(f_value)
# Update timestamp and event
except Exception as e:
error = traceback.format_exc()
response = "Error occured when converting string to misp objects:\n %s" % error
self.misp_logger.error(response)
return response
if self.check_object_length(objects) != True:
self.misp_logger.error(
'Input from %s did not contain accepted tags.\n Input: \n%s' %
(strUsername, strInput))
return "Error in the tags you entered. Please see the guide for accepted tags."
try:
event = self.misp.new_event(info=strInfo,
distribution='0',
analysis='2',
threat_level_id='3',
published=True)
misp_event = MISPEvent()
misp_event.load(event)
a, b = self.submit_to_misp(self.misp, misp_event, objects)
for tag in tags:
self.misp.tag(misp_event.uuid, tag)
self.misp.add_internal_comment(misp_event.id,
reference="Author: " + strUsername,
comment=str_comment)
self.misp.fast_publish(misp_event.id, alert=False)
misp_event = self.misp.get_event(misp_event.id)
response = misp_event
#for response in misp_event:
if ('errors' in response and response['errors'] != None):
return ("Submission error: " + repr(response['errors']))
else:
if response['Event']['RelatedEvent']:
e_related = ""
for each in response['Event']['RelatedEvent']:
e_related = e_related + each['Event']['id'] + ", "
return "Created ID: " + str(
response['Event']
['id']) + "\nRelated Events: " + ''.join(e_related)
else:
return "Created ID: " + str(response['Event']['id'])
except Exception as e:
error = traceback.format_exc()
response = "Error occured when submitting to misp:\n %s" % error
self.misp_logger.error(response)
return response
def parse(self, analysis_link, result):
"""
Parse the analysis result into a MISP event.
:param str analysis_link: the analysis link
:param dict[str, any] result: the JSON returned by the analysis client.
:rtype: MISPEvent
:return: some results that can be consumed by MIPS.
"""
self.misp_event = pymisp.MISPEvent()
# Add analysis subject info
if "url" in result["analysis_subject"]:
o = pymisp.MISPObject("url")
o.add_attribute("url", result["analysis_subject"]["url"])
else:
o = pymisp.MISPObject("file")
o.add_attribute("md5", type="md5", value=result["analysis_subject"]["md5"])
o.add_attribute("sha1", type="sha1", value=result["analysis_subject"]["sha1"])
o.add_attribute("sha256", type="sha256", value=result["analysis_subject"]["sha256"])
o.add_attribute(
"mimetype",
type="mime-type",
value=result["analysis_subject"]["mime_type"]
)
self.misp_event.add_object(o)
# Add HTTP requests from url analyses
network_dict = result.get("report", {}).get("analysis", {}).get("network", {})
for request in network_dict.get("requests", []):
parsed_uri = parse.urlparse(request["url"])
o = pymisp.MISPObject(name='http-request')
o.add_attribute('host', parsed_uri.netloc)
o.add_attribute('method', "GET")
o.add_attribute('uri', request["url"])
o.add_attribute("ip", request["ip"])
self.misp_event.add_object(o)
# Add network behaviors from files
for subject in result.get("report", {}).get("analysis_subjects", []):
# Add DNS requests
for dns_query in subject.get("dns_queries", []):
hostname = dns_query.get("hostname")
# Skip if it is an IP address
try:
if hostname == "wpad":
continue
_ = ipaddress.ip_address(hostname)
continue
except ValueError:
pass
o = pymisp.MISPObject(name='dns-record')
o.add_attribute('queried-domain', hostname)
self.misp_event.add_object(o)
# Add HTTP conversations (as network connection and as http request)
for http_conversation in subject.get("http_conversations", []):
o = pymisp.MISPObject(name="network-connection")
o.add_attribute("ip-src", http_conversation["src_ip"])
o.add_attribute("ip-dst", http_conversation["dst_ip"])
o.add_attribute("src-port", http_conversation["src_port"])
o.add_attribute("dst-port", http_conversation["dst_port"])
o.add_attribute("hostname-dst", http_conversation["dst_host"])
o.add_attribute("layer3-protocol", "IP")
o.add_attribute("layer4-protocol", "TCP")
o.add_attribute("layer7-protocol", "HTTP")
self.misp_event.add_object(o)
method, path, http_version = http_conversation["url"].split(" ")
if http_conversation["dst_port"] == 80:
uri = "http://{}{}".format(http_conversation["dst_host"], path)
else:
uri = "http://{}:{}{}".format(
http_conversation["dst_host"],
http_conversation["dst_port"],
path
)
o = pymisp.MISPObject(name='http-request')
o.add_attribute('host', http_conversation["dst_host"])
o.add_attribute('method', method)
o.add_attribute('uri', uri)
o.add_attribute('ip', http_conversation["dst_ip"])
self.misp_event.add_object(o)
# Add sandbox info like score and sandbox type
o = pymisp.MISPObject(name="sandbox-report")
sandbox_type = "saas" if is_analysis_hosted(analysis_link) else "on-premise"
o.add_attribute("score", result["score"])
o.add_attribute("sandbox-type", sandbox_type)
o.add_attribute("{}-sandbox".format(sandbox_type), "lastline")
o.add_attribute("permalink", analysis_link)
self.misp_event.add_object(o)
# Add behaviors
o = pymisp.MISPObject(name="sb-signature")
o.add_attribute("software", "Lastline")
for activity in result.get("malicious_activity", []):
a = pymisp.MISPAttribute()
a.from_dict(type="text", value=activity)
o.add_attribute("signature", **a)
self.misp_event.add_object(o)
# Add mitre techniques
for technique in self._get_mitre_techniques(result):
self.misp_event.add_tag(technique)
def main(args):
logging.info('Passing through misp_exporter')
with open(args.fingerprint) as file:
try:
data = json.load(file)
except Exception:
logging.error('Fingerprint not compatible.')
return
misp = pymisp.ExpandedPyMISP(args.misp_url,
args.misp_key,
ssl=False,
debug=True)
event = pymisp.MISPEvent(strict_validation=True)
event.info = args.event_info
event.distribution = args.distribution
event.threat_level_id = args.threat_level
event.analysis = args.analysis_level
event.add_tag(tag='validated')
ddos = pymisp.MISPObject('ddos', strict=True)
logging.info('Fingerprint processed: {}'.format(args.fingerprint))
if 'attack_vector' in data:
av = data['attack_vector'][0]
snort = {
'protocol': [],
'src_ip': [],
'src_port': [],
'dst_port': [],
'options': [],
}
df = find_ips(av)
logging.info('IPs found: {}'.format(len(df['ip'])))
subnets = utils.smart_aggregate(df)
logging.info('The IPs were summarized in: {} subnets'.format(
len(subnets)))
if args.subnets:
add_attributes(event, subnets, 'ip-src', 'attacker subnet')
ddos.add_attributes('ip-src', *subnets)
snort['src_ip'] += subnets
else:
add_attributes(event, df['ip'], 'ip-src', 'attacker ip')
ddos.add_attributes('ip-src', *df['ip'])
snort['src_ip'] += df['ip'].tolist()
if 'srcport' in av:
sport = find_attr(av['srcport'], 'srcport')
add_attributes(event, sport['srcport'], 'port',
'source port of attack')
ddos.add_attributes('src-port', *sport['srcport'])
snort['src_port'] += sport['srcport'].tolist()
if 'dstport' in av:
dport = find_attr(av['dstport'], 'dstport')
add_attributes(event, dport['dstport'], 'port',
'destination port of attack')
ddos.add_attributes('dst-port', *sport['dstport'])
snort['dst_port'] += sport['dstport']
if 'ip_proto' in av:
proto4 = find_attr(av['ip_proto'], 'ip_proto')
add_attributes(event, proto4['ip_proto'], 'other',
'4 level protocol of attack')
ddos.add_attributes('protocol', *proto4['ip_proto'])
snort['protocol'] += proto4['ip_proto'].tolist()
if 'dns_qry_name' in av:
snort['options'] += snort_content(av['dns_qry_name'])
if 'http_request' in av:
snort['options'] += snort_content(av['http_request'])
if 'http_response' in av:
snort['options'] += snort_content(av['http_response'])
if 'http_user_agent' in av:
snort['options'] += snort_content(av['http_user_agent'])
if 'icmp_type' in av:
snort['options'] += snort_itype(av['icmp_type'])
if 'icmp_code' in av:
snort['options'] += snort_icode(av['icmp_code'])
if 'ip_ttl' in av:
snort['options'] += snort_ttl(av['ip_ttl'])
if 'ntp_priv_reqcode' in av:
snort['options'] += snort_content(av['ntp_priv_reqcode'])
if 'tcp_flags' in av:
snort['options'] += snort_flags(av['tcp_flags'])
if 'tags' in av:
for tag in av['tags']:
event.add_tag(tag=tag)
if 'duration_sec' in av:
add_attribute_comment(event, 'duration_sec', av['duration_sec'])
if 'total_dst_ports' in av:
add_attribute_comment(event, 'total_dst_ports',
av['total_dst_ports'])
if 'total_ips' in av:
add_attribute_comment(event, 'total_ips', av['total_ips'])
if 'total_packets' in av:
add_attribute_comment(event, 'total_packets', av['total_packets'])
add_attribute_snort(event, snort)
event.add_tag(tag='ddos attack')
if 'amplifiers' in data:
amp = data['amplifiers'][0]
dfa = find_ips(amp)
logging.info('Amplifier IPs found: {}'.format(len(dfa['ip'])))
subnetsa = utils.smart_aggregate(dfa)
logging.info('The IPs were summarized in: {} subnets'.format(
len(subnetsa)))
if args.subnets:
add_attributes(event, subnetsa, 'ip-src', 'amplifier subnet')
for elem in subnetsa:
ddos.add_attribute('ip-src', elem, to_ids=0)
else:
add_attributes(event, dfa['ip'], 'ip-src', 'amplifier ip')
for elem in dfa['ip']:
ddos.add_attribute('ip-src', elem, to_ids=0)
event.add_tag(tag='ddos amplification attack')
p = pathlib.Path(args.fingerprint)
comment = 'DDoS fingerprint json file generated by dissector'
event.add_attribute(type='attachment',
value=p.name,
data=p,
comment=comment)
event.add_object(ddos, pythonify=True)
event = misp.add_event(event, pythonify=True)
def parse(self, analysis_link: str, result: Dict[str,
Any]) -> pymisp.MISPEvent:
"""
Parse the analysis result into a MISP event.
:param str analysis_link: the analysis link
:param dict[str, any] result: the JSON returned by the analysis client.
:rtype: pymisp.MISPEvent
:return: a MISP event
"""
misp_event = pymisp.MISPEvent()
# Add analysis subject info
if "url" in result["analysis_subject"]:
o = pymisp.MISPObject("url")
o.add_attribute("url", result["analysis_subject"]["url"])
else:
o = pymisp.MISPObject("file")
o.add_attribute("md5",
type="md5",
value=result["analysis_subject"]["md5"])
o.add_attribute("sha1",
type="sha1",
value=result["analysis_subject"]["sha1"])
o.add_attribute("sha256",
type="sha256",
value=result["analysis_subject"]["sha256"])
o.add_attribute("mimetype",
category="Payload delivery",
type="mime-type",
value=result["analysis_subject"]["mime_type"])
misp_event.add_object(o)
# Add HTTP requests from url analyses
network_dict = result.get("report", {}).get("analysis",
{}).get("network", {})
for request in network_dict.get("requests", []):
if not request["url"] and not request["ip"]:
continue
o = pymisp.MISPObject(name="http-request")
o.add_attribute("method", "GET")
if request["url"]:
parsed_uri = parse.urlparse(request["url"])
o.add_attribute("host", parsed_uri.netloc)
o.add_attribute("uri", request["url"])
if request["ip"]:
o.add_attribute("ip-dst", request["ip"])
misp_event.add_object(o)
# Add network behaviors from files
for subject in result.get("report", {}).get("analysis_subjects", []):
# Add DNS requests
for dns_query in subject.get("dns_queries", []):
hostname = dns_query.get("hostname")
# Skip if it is an IP address
try:
if hostname == "wpad" or hostname == "localhost":
continue
# Invalid hostname, e.g., hostname: ZLKKJRPY or 2.2.0.10.in-addr.arpa.
if "." not in hostname or hostname[-1] == ".":
continue
_ = ipaddress.ip_address(hostname)
continue
except ValueError:
pass
o = pymisp.MISPObject(name="domain-ip")
o.add_attribute("hostname", type="hostname", value=hostname)
for ip in dns_query.get("results", []):
o.add_attribute("ip", type="ip-dst", value=ip)
misp_event.add_object(o)
# Add HTTP conversations (as network connection and as http request)
for http_conversation in subject.get("http_conversations", []):
o = pymisp.MISPObject(name="network-connection")
o.add_attribute("ip-src", http_conversation["src_ip"])
o.add_attribute("ip-dst", http_conversation["dst_ip"])
o.add_attribute("src-port", http_conversation["src_port"])
o.add_attribute("dst-port", http_conversation["dst_port"])
o.add_attribute("hostname-dst", http_conversation["dst_host"])
o.add_attribute("layer3-protocol", "IP")
o.add_attribute("layer4-protocol", "TCP")
o.add_attribute("layer7-protocol", "HTTP")
misp_event.add_object(o)
method, path, http_version = http_conversation["url"].split(
" ")
if http_conversation["dst_port"] == 80:
uri = "http://{}{}".format(http_conversation["dst_host"],
path)
else:
uri = "http://{}:{}{}".format(
http_conversation["dst_host"],
http_conversation["dst_port"], path)
o = pymisp.MISPObject(name="http-request")
o.add_attribute("host", http_conversation["dst_host"])
o.add_attribute("method", method)
o.add_attribute("uri", uri)
o.add_attribute("ip-dst", http_conversation["dst_ip"])
misp_event.add_object(o)
# Add sandbox info like score and sandbox type
o = pymisp.MISPObject(name="sandbox-report")
sandbox_type = "saas" if tau_clients.is_task_hosted(
analysis_link) else "on-premise"
o.add_attribute("score", result["score"])
o.add_attribute("sandbox-type", sandbox_type)
o.add_attribute("{}-sandbox".format(sandbox_type),
"vmware-nsx-defender")
o.add_attribute("permalink", analysis_link)
misp_event.add_object(o)
# Add behaviors
# Check if its not empty first, as at least one attribute has to be set for sb-signature object
if result.get("malicious_activity", []):
o = pymisp.MISPObject(name="sb-signature")
o.add_attribute("software", "VMware NSX Defender")
for activity in result.get("malicious_activity", []):
a = pymisp.MISPAttribute()
a.from_dict(type="text", value=activity)
o.add_attribute("signature", **a)
misp_event.add_object(o)
# Add mitre techniques
for techniques in result.get("activity_to_mitre_techniques",
{}).values():
for technique in techniques:
for misp_technique_id, misp_technique_name in self.techniques_galaxy.items(
):
if technique["id"].casefold(
) in misp_technique_id.casefold():
# If report details a sub-technique, trust the match
# Otherwise trust it only if the MISP technique is not a sub-technique
if "." in technique[
"id"] or "." not in misp_technique_id:
misp_event.add_tag(misp_technique_name)
break
return misp_event