def load(self):
objects = []
with open(self.csv_path, newline='') as csvfile:
reader = csv.reader(csvfile)
if self.has_fieldnames:
# The file has fieldnames, we either ignore it, or validate its validity
fieldnames = [f.strip().lower() for f in reader.__next__()]
if not self.fieldnames:
self.fieldnames = fieldnames
if not self.fieldnames:
raise Exception(f'No fieldnames, impossible to create objects.')
else:
# Check if the CSV file has a header, and if it matches with the object template
tmp_object = MISPObject(self.template_name)
allowed_fieldnames = list(tmp_object._definition['attributes'].keys())
for fieldname in self.fieldnames:
if fieldname not in allowed_fieldnames:
raise Exception(f'{fieldname} is not a valid object relation for {self.template_name}: {allowed_fieldnames}')
for row in reader:
tmp_object = MISPObject(self.template_name)
for object_relation, value in zip(self.fieldnames, row):
tmp_object.add_attribute(object_relation, value=value)
objects.append(tmp_object)
return objects
def add_hashes(self):
if self.args.filename is None and self.args.md5 is None and self.args.sha1 is None and self.args.sha256 is None:
if not __sessions__.is_attached_file(True):
self.log('error', "Not attached to a file, please set the hashes manually.")
return False
file_object = MISPObject('file')
file_object.add_attribute('filename', value=__sessions__.current.file.name, comment=__sessions__.current.file.tags)
file_object.add_attribute('md5', value=__sessions__.current.file.md5, comment=__sessions__.current.file.tags)
file_object.add_attribute('sha1', value=__sessions__.current.file.sha1, comment=__sessions__.current.file.tags)
file_object.add_attribute('sha256', value=__sessions__.current.file.sha256, comment=__sessions__.current.file.tags)
__sessions__.current.misp_event.event.add_object(file_object)
else:
if self.args.filename:
if self.args.md5:
__sessions__.current.misp_event.event.add_attribute('filename|md5', '{}|{}'.format(
self.args.filename, self.args.md5))
if self.args.sha1:
__sessions__.current.misp_event.event.add_attribute('filename|sha1', '{}|{}'.format(
self.args.filename, self.args.sha1))
if self.args.sha256:
__sessions__.current.misp_event.event.add_attribute('filename|sha256', '{}|{}'.format(
self.args.filename, self.args.sha256))
else:
if self.args.md5:
__sessions__.current.misp_event.event.add_attribute('md5', self.args.md5)
if self.args.sha1:
__sessions__.current.misp_event.event.add_attribute('sha1', self.args.sha1)
if self.args.sha256:
__sessions__.current.misp_event.event.add_attribute('sha256', self.args.sha256)
self._change_event()
def add_dns(self):
"""Add DNS records"""
network = self.report.get("network", [])
dns = network.get("dns", [])
if not dns:
log.info("No DNS connection found in the report, skipping")
return False
for record in dns:
o = MISPObject(name='dns-record')
o.add_attribute('text', f"request type:{record['type']}")
o.add_attribute('queried-domain', record['request'])
for answer in record.get("answers", []):
if answer["type"] in ("A", "AAAA"):
o.add_attribute('a-record', answer['data'])
# TODO implement MX/NS
self.event.add_object(o)
def add_domain_object():
"""Adds a domain object to MISP
domain-ip description: https://www.misp-project.org/objects.html#_domain_ip
"""
template = 'domain-ip'
args = ['text', 'creation_date', 'first_seen', 'last_seen']
event_id = demisto.getArg('event_id')
domain = demisto.getArg('name')
obj = MISPObject(template)
ips = argToList(demisto.getArg('dns'))
for ip in ips:
obj.add_attribute('ip', value=ip)
obj.add_attribute('domain', value=domain)
for arg in args:
value = demisto.getArg(arg)
if value:
obj.add_attribute(arg, value=value)
add_object(event_id, obj)
def parse_course_of_action(self, o, _):
misp_object = MISPObject('course-of-action')
if 'name' in o:
attribute = {'type': 'text', 'object_relation': 'name', 'value': o.get('name')}
misp_object.add_attribute(**attribute)
else:
return
if 'description' in o:
attribute = {'type': 'text', 'object_relation': 'description', 'value': o.get('description')}
misp_object.add_attribute(**attribute)
self.misp_event.add_object(**misp_object)
def parse_vt_object(self, query_result):
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink',
type='link',
value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'],
query_result['total'])
vt_object.add_attribute('detection-ratio',
type='text',
value=detection_ratio)
self.misp_event.add_object(**vt_object)
def handle_object_case(self,
attribute_type,
attribute_value,
compl_data,
object_uuid=None):
misp_object = MISPObject(attribute_type)
if object_uuid:
misp_object.uuid = object_uuid
for attribute in attribute_value:
misp_object.add_attribute(**attribute)
if type(compl_data) is dict and "pe_uuid" in compl_data:
# if some complementary data is a dictionary containing an uuid,
# it means we are using it to add an object reference
misp_object.add_reference(compl_data['pe_uuid'], 'included-in')
self.misp_event.add_object(**misp_object)
def ip_lookup(self, ip):
sophos_object = MISPObject('SOPHOSLabs Intelix IP Category Lookup')
h = {"Authorization": f"{self.accessToken}"}
r = requests.get(f"https://{self.baseurl}/lookup/ips/v1/{ip}", headers=h)
if r.status_code == 200:
j = json.loads(r.text)
if 'category' in j:
for c in j['category']:
sophos_object.add_attribute('IP Address Categorisation', type='text', value=c)
else:
sophos_object.add_attribute('IP Address Categorisation', type='text', value='No category assocaited with IoC')
self.misp_event.add_object(**sophos_object)
def parse_url(self, url, recurse=False, uuid=None):
req = requests.get(self.base_url.format('url'),
params={
'apikey': self.apikey,
'resource': url
})
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
if not recurse:
feature = 'url'
url_object = MISPObject(feature)
url_object.add_attribute(feature, type=feature, value=url)
url_object.add_reference(vt_uuid, 'analyzed-with')
if uuid:
url_object.add_reference(uuid, 'hosted-in')
self.misp_event.add_object(**url_object)
return status_code
def __parse_capec(self, vulnerability_uuid):
attribute_type = 'text'
for capec in self.vulnerability['capec']:
capec_object = MISPObject('attack-pattern')
for feature in self.capec_features:
capec_object.add_attribute(feature, **dict(type=attribute_type, value=capec[feature]))
for related_weakness in capec['related_weakness']:
attribute = dict(type='weakness', value="CWE-{}".format(related_weakness))
capec_object.add_attribute('related-weakness', **attribute)
self.misp_event.add_object(**capec_object)
self.references[vulnerability_uuid].append(dict(referenced_uuid=capec_object.uuid,
relationship_type='targeted-by'))
def all_network(self, results, event):
"""All of the accessed URLS as per the PCAP."""
urls = set()
if self.options.get("network", False) and "network" in results.keys():
urls = set()
for req in results["network"].get("http", []):
if req.get("uri") not in whitelist:
urls.add(req["uri"])
if "user-agent" in req:
event.add_attribute("user-agent", req["user-agent"])
domains, ips = {}, set()
for domain in results.get("network", {}).get("domains", []):
if domain["domain"] not in whitelist and domain[
"ip"] not in whitelist:
domains[domain["domain"]] = domain["ip"]
ips.add(domain["ip"])
for block in results.get("network", {}).get("hosts", []):
if block["ip"] not in whitelist:
ips.add(block["ip"])
for block in results["network"].get("dns", []): # Added DNS
if block.get("request", "") and (block["request"]
not in whitelist):
if block["request"] not in domains and block[
"request"] not in whitelist:
if block["answers"]:
domains[
block["request"]] = block["answers"][0]["data"]
ips.add(domain[block["answers"][0]["data"]])
# Added CAPE Addresses
for section in results.get("CAPE", []) or []:
try:
if section.get("cape_config", {}).get("address", []) or []:
for ip in section["cape_config"]["address"]:
if ip not in ips:
ips.add(ip.split(":", 1)[0])
except Exception as e:
print(e)
for url in sorted(list(urls)):
event.add_attribute("url", url)
for ip in sorted(list(ips)):
event.add_attribute("ip-dst", ip)
for domain, ips in domains.items():
obj = MISPObject("domain-ip")
obj.add_attribute("domain", domain)
for ip in ips:
obj.add_attribute("ip", ip)
event.add_object(obj)
self.misp.update_event(event)
def __parse_misp_csv(self):
objects = {}
attribute_fields = self.header[:1] + self.header[2:8]
for line in self.data:
a_uuid, _, category, _type, value, comment, ids, timestamp, relation, tag, o_uuid, name, _ = line[:self.fields_number]
attribute = {t: v.strip('"') for t, v in zip(attribute_fields, (a_uuid, category, _type, value, comment, ids, timestamp))}
attribute['to_ids'] = True if attribute['to_ids'] == '1' else False
if tag:
attribute['Tag'] = [{'name': t.strip()} for t in tag.split(',')]
if relation:
if o_uuid not in objects:
objects[o_uuid] = MISPObject(name)
objects[o_uuid].add_attribute(relation, **attribute)
else:
self.misp_event.add_attribute(**attribute)
for uuid, misp_object in objects.items():
misp_object.uuid = uuid
self.misp_event.add_object(**misp_object)
def parse_hash(self, sample, recurse=False, uuid=None, relationship=None):
req = requests.get(self.base_url.format('file'), params={'apikey': self.apikey, 'resource': sample})
status_code = req.status_code
if req.status_code == 200:
req = req.json()
vt_uuid = self.parse_vt_object(req)
file_attributes = []
for hash_type in ('md5', 'sha1', 'sha256'):
if req.get(hash_type):
file_attributes.append({'type': hash_type, 'object_relation': hash_type,
'value': req[hash_type]})
if file_attributes:
file_object = MISPObject('file')
for attribute in file_attributes:
file_object.add_attribute(**attribute)
file_object.add_reference(vt_uuid, 'analyzed-with')
if uuid and relationship:
file_object.add_reference(uuid, relationship)
self.misp_event.add_object(**file_object)
return status_code
def export_pgp(pgp_type, pgp_value):
dict_metadata = Pgp.pgp.get_metadata(pgp_type, pgp_value)
obj = MISPObject('pgp-meta')
obj.first_seen = dict_metadata['first_seen']
obj.last_seen = dict_metadata['last_seen']
l_obj_attr = []
if pgp_type == 'key':
l_obj_attr.append(obj.add_attribute('key-id', value=pgp_value))
elif pgp_type == 'name':
#l_obj_attr.append( obj.add_attribute('key-id', value='debug') )
l_obj_attr.append(obj.add_attribute('user-id-name', value=pgp_value))
else: # mail
#l_obj_attr.append( obj.add_attribute('key-id', value='debug') )
l_obj_attr.append(obj.add_attribute('user-id-email', value=pgp_value))
return obj
def resolve_dns_objects(self):
for domain in self.dns_objects['domain']:
domain_object = self.dns_objects['domain'][domain]
ip_reference = domain_object['related']
domain_attribute = domain_object['data']
if ip_reference in self.dns_objects['ip']:
misp_object = MISPObject('passive-dns')
domain_attribute['object_relation'] = "rrname"
misp_object.add_attribute(**domain_attribute)
ip = self.dns_objects['ip'][ip_reference]['value']
ip_attribute = {"type": "text", "value": ip, "object_relation": "rdata"}
misp_object.add_attribute(**ip_attribute)
rrtype = "AAAA" if ":" in ip else "A"
rrtype_attribute = {"type": "text", "value": rrtype, "object_relation": "rrtype"}
misp_object.add_attribute(**rrtype_attribute)
self.misp_event.add_object(**misp_object)
else:
self.misp_event.add_attribute(**domain_attribute)
for ip in self.dns_objects['ip']:
if ip not in self.dns_ips:
self.misp_event.add_attribute(**self.dns_objects['ip'][ip])
def parse_report(self, query_result):
if query_result.get('asn'):
asn_mapping = {'network': ('ip-src', 'subnet-announced'),
'country': ('text', 'country')}
asn_object = MISPObject('asn')
asn_object.add_attribute('asn', type='AS', value=query_result['asn'])
for key, value in asn_mapping.items():
if query_result.get(key):
attribute_type, relation = value
asn_object.add_attribute(relation, type=attribute_type, value=query_result[key])
self.misp_event.add_object(**asn_object)
self.parse_urls(query_result)
if query_result.get('resolutions'):
self.parse_resolutions(query_result['resolutions'])
def export_cryptocurrency(crypto_type, crypto_address):
dict_metadata = Cryptocurrency.cryptocurrency.get_metadata(crypto_type, crypto_address)
obj = MISPObject('coin-address')
obj.first_seen = dict_metadata['first_seen']
obj.last_seen = dict_metadata['last_seen']
l_obj_attr = []
l_obj_attr.append( obj.add_attribute('address', value=crypto_address) )
crypto_symbol = Cryptocurrency.get_cryptocurrency_symbol(crypto_type)
if crypto_symbol:
l_obj_attr.append( obj.add_attribute('symbol', value=crypto_symbol) )
return obj
def handler(q=False):
""" the main handler function which gets a JSON dict as input and returns a results dict """
if q is False:
return False
q = json.loads(q)
if "config" not in q or "api-key" not in q["config"]:
return {"error": "Ransomcoindb API key is missing"}
if not q.get('attribute') or not check_input_attribute(q['attribute'], requirements=('type', 'value')):
return {'error': f'{standard_error_message}, {checking_error}.'}
if q['attribute']['type'] not in mispattributes['input']:
return {'error': 'Unsupported attribute type.'}
api_key = q["config"]["api-key"]
r = {"results": []}
""" the "q" query coming in should look something like this:
{'config': {'api-key': '<api key here>'},
'md5': 'md5 or sha1 or sha256 or btc',
'module': 'ransomcoindb',
'persistent': 1}
"""
attribute = q['attribute']
answer = ransomcoindb.get_data_by('BTC', attribute['type'], attribute['value'], api_key)
""" The results data type should be:
r = { 'results': [ {'types': 'md5', 'values': [ a list of all md5s or all binaries related to this btc address ] } ] }
"""
if attribute['type'] in ['md5', 'sha1', 'sha256']:
r['results'].append({'types': 'btc', 'values': [a['btc'] for a in answer]})
elif attribute['type'] == 'btc':
# better: create a MISP object
files = []
for a in answer:
obj = MISPObject('file')
obj.add_attribute('md5', a['md5'])
obj.add_attribute('sha1', a['sha1'])
obj.add_attribute('sha256', a['sha256'])
files.append(obj)
r['results'] = {'Object': [json.loads(f.to_json()) for f in files]}
return r
def parse_vt_object(self, query_result):
if query_result['response_code'] == 1:
vt_object = MISPObject('virustotal-report')
vt_object.add_attribute('permalink',
type='link',
value=query_result['permalink'])
detection_ratio = '{}/{}'.format(query_result['positives'],
query_result['total'])
vt_object.add_attribute('detection-ratio',
type='text',
value=detection_ratio,
disable_correlation=True)
self.misp_event.add_object(**vt_object)
return vt_object.uuid
def export_decoded(sha1_string):
decoded_metadata = Decoded.get_decoded_metadata(sha1_string, tag=True)
obj = MISPObject('file')
obj.first_seen = decoded_metadata['first_seen']
obj.last_seen = decoded_metadata['last_seen']
l_obj_attr = []
l_obj_attr.append( obj.add_attribute('sha1', value=sha1_string) )
l_obj_attr.append( obj.add_attribute('mimetype', value=Decoded.get_decoded_item_type(sha1_string)) )
l_obj_attr.append( obj.add_attribute('malware-sample', value=sha1_string, data=Decoded.get_decoded_file_content(sha1_string)) )
# add tags
if decoded_metadata['tags']:
tag_misp_object_attributes(l_obj_attr, decoded_metadata['tags'])
return obj
def _create_file_object(self, file_info):
file_object = MISPObject('file')
filename_attribute = {'type': 'filename'}
filename_attribute.update(self.attribute)
if file_info['classification'] != "UNCLASSIFIED":
tag = {'Tag': [{'name': file_info['classification'].lower()}]}
filename_attribute.update(tag)
for feature, attribute in self._file_mapping.items():
attribute.update(tag)
file_object.add_attribute(value=file_info[feature],
**attribute)
return filename_attribute, file_object
for feature, attribute in self._file_mapping.items():
file_object.add_attribute(value=file_info[feature], **attribute)
return filename_attribute, file_object
def to_misp_object(self, tag: bool) -> MISPObject:
obj = MISPObject(name="email")
if self.sender:
classifications = classifications_to_str(self.classifications)
attr = obj.add_attribute("from",
value=self.sender,
to_ids=self.is_ioc,
comment=classifications)
if tag:
self.tag_artifact_attribute(attr)
if self.subject:
obj.add_attribute("subject", value=self.subject, to_ids=False)
for recipient in self.recipients:
obj.add_attribute("to", value=recipient, to_ids=False)
return obj
def export_domain(domain):
domain_obj = Domain.Domain(domain)
dict_metadata = domain_obj.get_domain_metadata(tags=True)
dict_metadata['ports'] = ['80', '223', '443']
# create domain-ip obj
obj = MISPObject('domain-ip', standalone=True)
obj.first_seen = dict_metadata['first_seen']
obj.last_seen = dict_metadata['last_check']
l_obj_attr = []
l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['first_seen']) )
l_obj_attr.append( obj.add_attribute('last-seen', value=dict_metadata['last_check']) )
l_obj_attr.append( obj.add_attribute('domain', value=domain) )
for port in dict_metadata['ports']:
l_obj_attr.append( obj.add_attribute('port', value=port) )
# add tags
if dict_metadata['tags']:
tag_misp_object_attributes(l_obj_attr, dict_metadata['tags'])
#print(obj.to_json())
return obj
def __parse_weakness(self, vulnerability_uuid):
attribute_type = 'text'
cwe_string, cwe_id = self.vulnerability['cwe'].split('-')
cwes = requests.get(self.api_url.replace('/cve/', '/cwe'))
if cwes.status_code == 200:
for cwe in cwes.json():
if cwe['id'] == cwe_id:
weakness_object = MISPObject('weakness')
weakness_object.add_attribute('id', **dict(type=attribute_type, value='-'.join([cwe_string, cwe_id])))
for feature, relation in self.weakness_mapping.items():
if cwe.get(feature):
weakness_object.add_attribute(relation, **dict(type=attribute_type, value=cwe[feature]))
self.misp_event.add_object(**weakness_object)
self.references[vulnerability_uuid].append(dict(referenced_uuid=weakness_object.uuid,
relationship_type='weakened-by'))
break
def parse(self, value):
try:
results = self.pdns.query(self.attribute.value)
except Exception:
self.result = {'error': 'There is an authentication error, please make sure you supply correct credentials.'}
return
mapping = {'count': 'counter', 'origin': 'text',
'time_first': 'datetime', 'rrtype': 'text',
'rrname': 'text', 'rdata': 'text',
'time_last': 'datetime'}
for result in results:
pdns_object = MISPObject('passive-dns')
for relation, attribute_type in mapping.items():
pdns_object.add_attribute(relation, type=attribute_type, value=result[relation])
pdns_object.add_reference(self.attribute.uuid, 'associated-to')
self.misp_event.add_object(**pdns_object)
def handler(q=False):
""" the main handler function which gets a JSON dict as input and returns a results dict """
if q is False:
return False
q = json.loads(q)
api_key = q["config"]["api-key"]
r = {"results": []}
""" the "q" query coming in should look something like this:
{'config': {'api-key': '<api key here>'},
'md5': 'md5 or sha1 or sha256 or btc',
'module': 'ransomcoindb',
'persistent': 1}
"""
attribute = q['attribute']
answer = ransomcoindb.get_data_by('BTC', attribute['type'],
attribute['value'], api_key)
""" The results data type should be:
r = { 'results': [ {'types': 'md5', 'values': [ a list of all md5s or all binaries related to this btc address ] } ] }
"""
if attribute['type'] in ['md5', 'sha1', 'sha256']:
r['results'].append({
'types': 'btc',
'values': [a['btc'] for a in answer]
})
elif attribute['type'] == 'btc':
# better: create a MISP object
files = []
for a in answer:
obj = MISPObject('file')
obj.add_attribute('md5', a['md5'])
obj.add_attribute('sha1', a['sha1'])
obj.add_attribute('sha256', a['sha256'])
files.append(obj)
r['results'] = {'Object': [json.loads(f.to_json()) for f in files]}
return r
def parse_dropped_files(self):
droppedinfo = self.data['droppedinfo']
if droppedinfo:
for droppedfile in droppedinfo['hash']:
file_object = MISPObject('file')
for key, mapping in dropped_file_mapping.items():
attribute_type, object_relation = mapping
file_object.add_attribute(
object_relation, **{
'type': attribute_type,
'value': droppedfile[key],
'to_ids': False
})
if droppedfile['@malicious'] == 'true':
file_object.add_attribute(
'state', **{
'type': 'text',
'value': 'Malicious',
'to_ids': False
})
for h in droppedfile['value']:
hash_type = dropped_hash_mapping[h['@algo']]
file_object.add_attribute(
hash_type, **{
'type': hash_type,
'value': h['$'],
'to_ids': False
})
self.misp_event.add_object(**file_object)
self.references[self.process_references[(
int(droppedfile['@targetid']),
droppedfile['@process'])]].append({
'referenced_uuid':
file_object.uuid,
'relationship_type':
'drops'
})
def parse_result(attribute, values):
event = MISPEvent()
initial_attribute = MISPAttribute()
initial_attribute.from_dict(**attribute)
event.add_attribute(**initial_attribute)
mapping = {'asn': ('AS', 'asn'), 'prefix': ('ip-src', 'subnet-announced')}
print(values)
for last_seen, response in values['response'].items():
asn = MISPObject('asn')
asn.add_attribute('last-seen', **{
'type': 'datetime',
'value': last_seen
})
for feature, attribute_fields in mapping.items():
attribute_type, object_relation = attribute_fields
asn.add_attribute(
object_relation, **{
'type': attribute_type,
'value': response[feature]
})
asn.add_reference(initial_attribute.uuid, 'related-to')
event.add_object(**asn)
event = json.loads(event.to_json())
return {key: event[key] for key in ('Attribute', 'Object')}
def parse_ip(self, ip, recurse=False):
req = requests.get(self.base_url.format('ip-address'), params={'apikey': self.apikey, 'ip': ip})
if req.status_code != 200:
return req.status_code
req = req.json()
if req.get('asn'):
asn_mapping = {'network': ('ip-src', 'subnet-announced'),
'country': ('text', 'country')}
asn_object = MISPObject('asn')
asn_object.add_attribute('asn', type='AS', value=req['asn'])
for key, value in asn_mapping.items():
if req.get(key):
attribute_type, relation = value
asn_object.add_attribute(relation, type=attribute_type, value=req[key])
self.misp_event.add_object(**asn_object)
uuid = self.parse_resolutions(req['resolutions']) if req.get('resolutions') else None
return self.parse_related_urls(req, recurse, uuid)
def add_original_file(self, original_filename):
with open(self.filename, 'rb') as f:
sample = b64encode(f.read()).decode('utf-8')
original_file = MISPObject('original-imported-file')
original_file.add_attribute(
**{
'type': 'attachment',
'value': original_filename,
'object_relation': 'imported-sample',
'data': sample
})
original_file.add_attribute(
**{
'type': 'text',
'object_relation': 'format',
'value': self.stix_version
})
self.misp_event.add_object(**original_file)
def parse_object(self, o, labels):
object_type = self.get_misp_type(labels)
name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type
object_category = self.get_misp_category(labels)
stix_type = o._type
misp_object = MISPObject(name)
misp_object['meta-category'] = object_category
if stix_type == 'indicator':
pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
pattern[0] = pattern[0][1:]
pattern[-1] = pattern[-1][:-1]
attributes = self.objects_mapping[object_type]['pattern'](pattern)
if stix_type == 'observed-data':
observable = o.get('objects')
attributes = self.objects_mapping[object_type]['observable'](observable)
if isinstance(attributes, tuple):
attributes, pe_uuid = attributes
misp_object.add_reference(pe_uuid, 'included-in')
for attribute in attributes:
misp_object.add_attribute(**attribute)
misp_object.to_ids = bool(labels[1].split('=')[1])
self.misp_event.add_object(**misp_object)
def parse_system_behavior(self):
system = self.data['behavior']['system']
if system.get('processes'):
process_activities = {
'fileactivities': self.parse_fileactivities,
'registryactivities': self.parse_registryactivities
}
for process in system['processes']['process']:
no_correlation = False
general = process['general']
if general['name'] in disable_correlations:
no_correlation = True
process_object = MISPObject('process')
for feature, relation in process_object_fields.items():
process_object.add_attribute(
relation, **{
'type': 'text',
'value': general[feature],
'disable_correlation': no_correlation,
'to_ids': False
})
start_time = datetime.strptime(
'{} {}'.format(general['date'], general['time']),
'%d/%m/%Y %H:%M:%S')
process_object.add_attribute(
'start-time', **{
'type': 'datetime',
'value': start_time,
'disable_correlation': no_correlation,
'to_ids': False
})
self.misp_event.add_object(**process_object)
for field, to_call in process_activities.items():
if process.get(field):
to_call(process_object.uuid, process[field])
self.references[self.analysisinfo_uuid].append(
dict(referenced_uuid=process_object.uuid,
relationship_type='calls'))
self.process_references[(
general['targetid'],
general['path'])] = process_object.uuid
def check_hashes(self):
if self.offline_mode:
self.log('error', 'Offline mode, unable to query VirusTotal')
return
event_id = self._get_eventid()
if event_id is None:
return
event = self.misp.get(event_id)
if self._has_error_message(event):
return
misp_event = MISPEvent()
misp_event.load(event)
hashes_to_expand = {}
hashes_expanded = [] # Thoses hashes are known and already processed
local_samples_hashes = []
partial_objects = {}
for o in misp_event.Object:
if o.name != 'file':
continue
if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']):
# This object has all the hashes we care about
tmphashes = []
tmphashes += [h.value for h in o.get_attributes_by_relation('md5')]
tmphashes += [h.value for h in o.get_attributes_by_relation('sha1')]
tmphashes += [h.value for h in o.get_attributes_by_relation('sha256')]
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[o.get_attributes_by_relation('sha256')[0].value] = o.get_attributes_by_relation('sha256')[0]
if o.has_attributes_by_relation(['malware-sample']):
# ... and it has a malware sample
local_samples_hashes += tmphashes
hashes_expanded += tmphashes
elif o.has_attributes_by_relation(['malware-sample']):
# This object has a malware sample, but is missing hashes. We can expand locally.
# get the MD5 from the malware-sample attribute
malware_sample = o.get_attributes_by_relation('malware-sample')[0] # at most one sample/file object
local_samples_hashes.append(malware_sample.value.split('|')[1])
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('md5')]
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha1')]
local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha256')]
if self.args.populate:
# The object is missing hashes, keeping track of it for expansion if it isn't already done.
partial_objects[o.uuid] = malware_sample
else:
sha256 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha256')}
sha1 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha1')}
md5 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('md5')}
if sha256:
hashes_to_expand.update(sha256)
elif sha1:
hashes_to_expand.update(sha1)
elif md5:
hashes_to_expand.update(md5)
for ref_uuid, sample in partial_objects.items():
if sample.value.split('|')[1] in hashes_expanded:
# Already expanded in an other object
continue
new_obj, hashes = self._expand_local_sample(pseudofile=sample.malware_binary,
filename=sample.value.split('|')[0],
refobj=ref_uuid,
default_attributes_paramaters=sample)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = sample
hashes_expanded += local_samples_hashes
for a in misp_event.attributes:
if a.type == 'malware-sample' and a.value.split('|')[1] not in hashes_expanded:
new_obj, hashes = self._expand_local_sample(pseudofile=a.malware_binary,
filename=a.value.split('|')[0],
default_attributes_paramaters=a)
misp_event.Object += new_obj
local_samples_hashes += hashes
# Make sure to query VT for the sha256, even if expanded locally
hashes_to_expand[hashes[0]] = a
elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
fname, hashval = a.value.split('|')
hashes_to_expand[hashval] = a
elif a.type in ('md5', 'sha1', 'sha256'):
# We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
hashes_to_expand[a.value] = a
unk_vt_hashes = []
if cfg.virustotal.virustotal_has_private_key is False:
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
hashes_expanded += local_samples_hashes
processed_on_vt = []
# Make sure to start getting reports for the longest possible hashes (reduce risks of collisions)
for to_expand in sorted(list(set(hashes_to_expand)), key=len):
if to_expand in processed_on_vt:
# Always run VT, once per sample
continue
original_attribute = hashes_to_expand[to_expand]
if original_attribute.get('object_id'):
original_object_id = original_attribute.get('object_id')
vt_object = self._make_VT_object(to_expand, original_attribute)
if not vt_object:
unk_vt_hashes.append(to_expand)
continue
result = vt_object.get_report()
md5 = result['md5']
sha1 = result['sha1']
sha256 = result['sha256']
processed_on_vt += [sha256, sha1, md5]
if all(h in local_samples_hashes for h in [md5, sha1, sha256]):
self.log('success', 'Sample available in MISP:')
else:
self.log('success', 'Sample available in VT:')
self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5, sha1, sha256))
if self.args.populate:
if not all(h in hashes_expanded for h in [md5, sha1, sha256]):
# If all the "new" expanded hashes are in the hashes_expanded list, skip
file_object = MISPObject('file', default_attributes_paramaters=original_attribute)
file_object.add_attribute('md5', value=md5)
file_object.add_attribute('sha1', value=sha1)
file_object.add_attribute('sha256', value=sha256)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(file_object)
hashes_expanded += [md5, sha1, sha256]
else:
if not original_object_id or original_object_id == '0':
# Not an object, but the hashes are in an other object, skipping
continue
else:
# We already have a MISP object, adding the link to the new VT object
file_object = misp_event.get_object_by_id(original_object_id)
file_object.add_reference(vt_object.uuid, 'analysed-with')
misp_event.Object.append(vt_object)
if cfg.virustotal.virustotal_has_private_key is False:
if quota > 0:
quota -= 1
else:
waiting_time = (timeout - datetime.datetime.now()).seconds
if waiting_time > 0:
self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time))
time.sleep(waiting_time)
quota = 4
timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
if self.args.populate:
self._populate(misp_event)
if len(unk_vt_hashes) > 0:
self.log('error', 'Unknown on VT:')
for h in unk_vt_hashes:
self.log('item', '{}'.format(h))
