def build_generic_object(template_name: str, args: List[dict]) -> GenericObjectGenerator:
"""
Args:
template_name: template name as described in
args: arguments to create the generic object
Returns:
GenericObjectGenerator: object created in MISP
Example:
args should look like:
[{'analysis_submitted_at': '2018-06-15T06:40:27'},
{'threat_score': {value=95, to_ids=False}},
{'permalink': 'https://panacea.threatgrid.com/mask/samples/2e445ef5389d8b'},
{'heuristic_raw_score': 7.8385159793597}, {'heuristic_score': 96},
{'original_filename': 'juice.exe'}, {'id': '2e445ef5389d8b'}] # guardrails-disable-line
"""
misp_object = GenericObjectGenerator(template_name)
misp_object.generate_attributes(args)
return misp_object
def cuckoo2misp_thread(self, iocs, event):
event_id = event['Event']['id']
while iocs:
ioc = iocs.pop()
if ioc.get("md5"):
self.misp.add_hashes(event,
md5=ioc["md5"],
sha1=ioc["sha1"],
sha256=ioc["sha256"]
)
elif ioc.get("file_obj", ""):
self.add_file(event_id, ioc["path"], ioc["filename"], ioc["file_obj"])
elif ioc.get("domain_obj", ""):
template_id = [x['ObjectTemplate']['id'] for x in self.misp.get_object_templates_list() if x['ObjectTemplate']['name'] == 'domain-ip'][0]
misp_object = GenericObjectGenerator('domain-ip')
misp_object.generate_attributes(json.loads(ioc['domain_obj']))
self.misp.add_object(event_id, template_id, misp_object)
self.slaves_id.append(misp_object.uuid)
elif ioc.get("uri_obj", ""):
template_id = [x['ObjectTemplate']['id'] for x in self.misp.get_object_templates_list() if x['ObjectTemplate']['name'] == 'http-request'][0]
misp_object = GenericObjectGenerator('http-request')
misp_object.generate_attributes(json.loads(ioc['uri_obj']))
self.misp.add_object(event_id, template_id, misp_object)
self.slaves_id.append(misp_object.uuid)
elif ioc.get("uri", ""):
self.misp.add_url(event, ioc["uri"])
elif ioc.get("mutex", ""):
self.misp.add_mutex(event, ioc["mutex"])
elif ioc.get("regkey", ""):
self.misp.add_regkey(event, ioc["regkey"])
def _add_exploit_poc_template(self, uuid, exploit):
"""
Adds a new exploit-poc object template to a given event.
:param uuid (str): The uuid of the event.
:param exploit (dict): Dictionary of an exploit. It needs the following
keys: 'file', 'author' and 'description'.
"""
gen_obj = GenericObjectGenerator('exploit-poc')
attributes = []
exploit_file = exploit['file']
with open(os.path.join(self.local_git, exploit_file)) as f:
poc = f.read()
attributes.append({
'poc':
poc,
'references':
'{}/tree/master/{}'.format(self.remote_git, exploit_file),
'author':
exploit['author'],
'description':
exploit['description']
})
gen_obj.generate_attributes(attributes)
self.misp.add_object(uuid, gen_obj.template_uuid, gen_obj)
def cuckoo2misp_thread(self, iocs, event):
event_id = event['Event']['id']
while iocs:
ioc = iocs.pop()
if ioc.get("md5"):
self.misp.add_hashes(event,
md5=ioc["md5"],
sha1=ioc["sha1"],
sha256=ioc["sha256"])
elif ioc.get("file_obj", ""):
self.add_file(event_id, ioc["path"], ioc["filename"],
ioc["file_obj"])
elif ioc.get("domain_obj", ""):
template_id = [
x['ObjectTemplate']['id']
for x in self.misp.get_object_templates_list()
if x['ObjectTemplate']['name'] == 'domain-ip'
][0]
misp_object = GenericObjectGenerator('domain-ip')
misp_object.generate_attributes(json.loads(ioc['domain_obj']))
self.misp.add_object(event_id, template_id, misp_object)
self.slaves_id.append(misp_object.uuid)
elif ioc.get("uri_obj", ""):
template_id = [
x['ObjectTemplate']['id']
for x in self.misp.get_object_templates_list()
if x['ObjectTemplate']['name'] == 'http-request'
][0]
misp_object = GenericObjectGenerator('http-request')
misp_object.generate_attributes(json.loads(ioc['uri_obj']))
self.misp.add_object(event_id, template_id, misp_object)
self.slaves_id.append(misp_object.uuid)
elif ioc.get("uri", ""):
self.misp.add_url(event, ioc["uri"])
elif ioc.get("mutex", ""):
self.misp.add_mutex(event, ioc["mutex"])
elif ioc.get("regkey", ""):
self.misp.add_regkey(event, ioc["regkey"])
def addSampleByHash(hashes, event):
hash=""
if ( type(hashes) is list):
hash=hashes[0]
elif ( type(hashes) is str):
hash=hashes
hashes=[hash]
sample_json = _getSampleJson(hash)
if (sample_json is None):
return
sampl = GenericObjectGenerator('file')
sampl.add_attribute("md5", value=sample_json['md5_hash'], to_ids=True)
sampl.add_attribute("filename", value=sample_json['file_name'], to_ids=False, disable_correlation=True)
sampl.add_attribute("sha1", value=sample_json['sha1_hash'], to_ids=True)
sampl.add_attribute("sha256", value=sample_json['sha256_hash'], to_ids=True)
sampl.add_attribute("ssdeep", value=sample_json['ssdeep'], to_ids=True)
sampl.add_attribute("size-in-bytes", value=sample_json['file_size'], to_ids=False, disable_correlation=True)
sampl.add_attribute("state", value="Malicious", to_ids=False, disable_correlation=True)
sampl.add_attribute("mimetype", value=sample_json['file_type_mime'].replace('\\',''), to_ids=False, disable_correlation=True)
# if it exists, add the comment
if ( 'comment' in sample_json ):
comment=sample_json['comment']
if ( comment is not None ) and (len(comment) > 0):
commattrs=pm.freetext(event, comment)
for commattr in commattrs:
if (commattr['value'] in attributes):
attr=attributes[commattr['value']]
sampl.add_reference(referenced_uuid=attr.uuid, relationship_type='related-to')
else:
attr=event.add_attribute(commattr['type'], commattr['value'])
attributes.update({commattr['value']:attr})
sampl.add_reference(referenced_uuid=attr.uuid, relationship_type='related-to')
# find and add x-references
if ( 'file_information' in sample_json):
info=sample_json['file_information']
if (info is not None):
for context_set in info:
context=context_set['context']
value=context_set['value']
print("context: {}, value: {}".format(context, value))
if ( context in API_REF_CONTEXTS ):
ref_uuid=""
addedSample=None
# if referenced sample is not already represented, recursively create it and add to the event and to samples<dict>
if (( value not in samples ) and (value not in hashes) ) :
addedSample=addSampleByHash([value]+hashes, event)
if (addedSample is not None):
ref_uuid=samples[value].uuid
if (context == "dropped_by_sha256"):
sampl.add_reference(referenced_uuid=ref_uuid, relationship_type='dropped-by')
else:
sampl.add_reference(referenced_uuid=ref_uuid, relationship_type='drops')
elif ( context.casefold() in API_LINK_CONTEXTS ):
url_ref=value.replace('\\','')
attribute = None
if ( url_ref not in attributes):
attribute = event.add_attribute('url', url_ref, to_ids=False, disable_correlation=True)
attributes.update({attribute.value : attribute})
sampl.add_reference(referenced_uuid=attribute.uuid, relationship_type='related-to')
else:
sampl.add_reference(referenced_uuid=attributes[url_ref].uuid, relationship_type='related-to')
else:
print("Lost context: {}".format(context))
attribute = None
report_url="https://bazaar.abuse.ch/sample/{}/".format(hash)
if (report_url not in attributes):
attribute = event.add_attribute("url", "https://bazaar.abuse.ch/sample/{}/".format(hash), to_ids = False, disable_correlation=True)
attributes.update({attribute.value : attribute})
else:
attribute=attributes[report_url]
sampl.add_reference(referenced_uuid=attribute.uuid, relationship_type='derived-from')
sampl=event.add_object(sampl)
samples.update({hash:sampl})
return sampl
def create_misp_events(config, results):
# print(config)
# iterate through each row, cleaning multivalue fields and then adding the attributes under same event key
# this builds the dict events
events = {}
for row in results:
# Splunk makes a bunch of dumb empty multivalue fields - we filter those out here
row = {
key: value
for key, value in row.items() if not key.startswith("__mv_")
}
# GEt the specific eventkey if define in Splunk search. Defaults to alert form got above
eventkey = config['eventkey']
if eventkey in row:
eventkey = row.pop(eventkey)
# check if building event has been initiated
# if yes simply add attribute entry otherwise collect other metadata
# remove fields _time and info from row and keep their values if this is a new event
if eventkey in events:
event = events[eventkey]
artifacts = event['attribute']
if '_time' in row:
remove = str(row.pop('_time'))
if 'info' in row:
remove = row.pop('info')
else:
event = {}
event['eo_count'] = 0
event['fo_count'] = 0
event['no_count'] = 0
artifacts = []
if '_time' in row:
event['timestamp'] = str(row.pop('_time'))
else:
event['timestamp'] = str(int(time.time()))
if 'info' in row:
event['info'] = row.pop('info')
else:
event['info'] = config['info']
# collect attribute value and build type=value entry
if 'to_ids' in row:
if str(row.pop('to_ids')) == 'True':
to_ids = True
else:
to_ids = False
else:
to_ids = False
if 'category' in row:
category = str(row.pop('category'))
else:
category = None
# now we take KV pairs starting by misp_ to add to event as single attribute(s)
for key, value in row.items():
if key.startswith("misp_") and value != "":
misp_key = str(key).replace('misp_', '').replace('_', '-')
artifacts.append(
store_attribute(misp_key, str(value), to_ids, category))
event['attribute'] = artifacts
# now we look for attribute belonging to a file object i.e. on the same row, field(s) start(s) with fo_
fo_list = []
for key, value in row.items():
if key.startswith("fo_") and value != "":
fo_key = str(key).replace('fo_', '').replace('_', '-')
fo_list.append(store_object_attribute(fo_key, str(value)))
if fo_list:
event['fo_count'] = event['fo_count'] + 1
my_key = 'fo_' + str(event['fo_count'])
event[my_key] = fo_list
# then we look for attribute belonging to an email object i.e. on the same row, field(s) start(s) with eo_
eo_list = []
for key, value in row.items():
if key.startswith("eo_") and value != "":
eo_key = str(key).replace('eo_', '').replace('_', '-')
eo_list.append(store_object_attribute(eo_key, str(value)))
if eo_list:
event['eo_count'] = event['eo_count'] + 1
my_key = 'eo_' + str(event['eo_count'])
event[my_key] = eo_list
# finally we look if there is a domain-ip object i.e. on the same row, 2 fields no_domain and no_ip
no_list = []
if 'no_domain' in row and 'no_ip' in row:
domain = str(row.pop('no_domain'))
ip = str(row.pop('no_ip'))
if domain != "" and ip != "":
no_list.append(store_object_attribute('domain', str(domain)))
no_list.append(store_object_attribute('ip', str(ip)))
if no_list:
event['no_count'] = event['no_count'] + 1
my_key = 'no_' + str(event['no_count'])
event[my_key] = no_list
events[eventkey] = event
# events are prepared; now create them in MISP
# print(events)
mispsrv = config['mispsrv']
mispkey = config['mispkey']
sslcheck = config['sslcheck']
# connect to misp instance using url, authkey and boolean sslcheck
pymisp = init(mispsrv, mispkey, sslcheck)
# extract from config and event the values to create events
analysis = config['analysis']
distrib = config['distribution']
threat = config['threatlevel']
#iteration in events
eventlist = {}
for key, event in events.items():
date = datetime.datetime.fromtimestamp(int(
event['timestamp'])).strftime('%Y-%m-%d')
info = event['info']
# creqte the event in misp instqnce
my_event = create_event(pymisp, distrib, threat, analysis, info, date)
# tag the event with TLP level
tlp = config['tlp']
# get UUID from new event - required for tag()
uuid = my_event['Event']['uuid']
pymisp.tag(uuid, tlp)
# add atrributes to event
# get ID from new event
eid = int(my_event['Event']['id'])
# loop for attribute entries
# please note that distribution will be force to 5 = inherit - if not provided default to your organisation
for a in event['attribute']:
updated = add_attribute(pymisp, eid, a['type'], a['value'],
a['category'], a['to_ids'])
#loop for file object entry
if event['fo_count'] > 0:
try:
template_id = [
x['ObjectTemplate']['id']
for x in pymisp.get_object_templates_list()
if x['ObjectTemplate']['name'] == 'file'
][0]
fo_record = event['fo_count']
while fo_record > 0:
misp_object = GenericObjectGenerator('file')
my_key = 'fo_' + str(fo_record)
misp_object.generate_attributes(event[my_key])
r = pymisp.add_object(eid, template_id, misp_object)
fo_record = fo_record - 1
except IndexError:
valid_types = ", ".join([
x['ObjectTemplate']['name']
for x in pymisp.get_object_templates_list()
])
print("Template for type %s not found! Valid types are: %s" %
('file', valid_types))
#loop for email object entry
if event['eo_count'] > 0:
try:
template_id = [
x['ObjectTemplate']['id']
for x in pymisp.get_object_templates_list()
if x['ObjectTemplate']['name'] == 'email'
][0]
eo_record = event['eo_count']
while eo_record > 0:
misp_object = GenericObjectGenerator('email')
my_key = 'eo_' + str(eo_record)
misp_object.generate_attributes(event[my_key])
r = pymisp.add_object(eid, template_id, misp_object)
eo_record = eo_record - 1
except IndexError:
valid_types = ", ".join([
x['ObjectTemplate']['name']
for x in pymisp.get_object_templates_list()
])
print("Template for type %s not found! Valid types are: %s" %
('file', valid_types))
#loop for domain-ip object entry
if event['no_count'] > 0:
try:
template_id = [
x['ObjectTemplate']['id']
for x in pymisp.get_object_templates_list()
if x['ObjectTemplate']['name'] == 'domain-ip'
][0]
no_record = event['no_count']
while no_record > 0:
misp_object = GenericObjectGenerator('domain-ip')
my_key = 'no_' + str(no_record)
misp_object.generate_attributes(event[my_key])
r = pymisp.add_object(eid, template_id, misp_object)
no_record = no_record - 1
except IndexError:
valid_types = ", ".join([
x['ObjectTemplate']['name']
for x in pymisp.get_object_templates_list()
])
print("Template for type %s not found! Valid types are: %s" %
('file', valid_types))
eventlist['eid'] = uuid
return eventlist
"""
Sample usage:
./add_generic_object.py -e 5065 -t email -l '[{"to": "*****@*****.**"}, {"to": "*****@*****.**"}]'
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary')
parser.add_argument("-e", "--event", required=True, help="Event ID to update")
parser.add_argument("-t", "--type", required=True, help="Type of the generic object")
parser.add_argument("-l", "--attr_list", required=True, help="List of attributes")
args = parser.parse_args()
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
template = pymisp.get_object_templates_list()
if 'response' in template.keys():
template = template['response']
try:
template_ids = [x['ObjectTemplate']['id'] for x in template if x['ObjectTemplate']['name'] == args.type]
if len(template_ids) > 0:
template_id = template_ids[0]
else:
raise IndexError
except IndexError:
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in template])
print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types))
exit()
misp_object = GenericObjectGenerator(args.type.replace("|", "-"))
misp_object.generate_attributes(json.loads(args.attr_list))
r = pymisp.add_object(args.event, template_id, misp_object)
def genFileObj(sample):
sampl = GenericObjectGenerator('file')
sampl.add_attribute("md5", value=sample['md5_hash'], to_ids=True)
sampl.add_attribute("filename",
value=sample['file_name'],
to_ids=False,
disable_correlation=True)
sampl.add_attribute("sha1", value=sample['sha1_hash'], to_ids=True)
sampl.add_attribute("sha256", value=sample['sha256_hash'], to_ids=True)
sampl.add_attribute("ssdeep", value=sample['ssdeep'], to_ids=True)
sampl.add_attribute("size-in-bytes",
value=sample['file_size'],
to_ids=False,
disable_correlation=True)
sampl.add_attribute("state",
value="Malicious",
to_ids=False,
disable_correlation=True)
sampl.add_attribute("mimetype",
value=sample['file_type_mime'].replace('\\', ''),
to_ids=False,
disable_correlation=True)
return sampl
event.distribution = args.distribution
if not args.dryrun:
new_event = pymisp.add_event(event)
if 'errors' in new_event.keys():
log.critical('Error creating the new event. {}'.format(new_event['errors'][2]))
exit(1)
# # Get the ID of the new event for later
args.event = new_event['Event']['uuid']
log.info('New event created: {}'.format(args.event))
# # Add Objects to existing Event
for i, o in enumerate(objects, 1):
misp_object = GenericObjectGenerator(o['object'], misp_objects_path_custom=args.custom_objects_path)
try:
misp_object.generate_attributes(o['attributes'])
except NewAttributeError as e:
log.critical('Error creating attributes, often this is due to custom objects being used. Error: {}'.format(e))
exit(1)
# # Add distribution if it has been set
try: misp_object.distribution = o.get('object_distribution')
except: pass
# # Add comment to object if it has been set
try: misp_object.comment = o.get('object_comment')
except: pass
# # Just print the object if --dryrun has been used
if args.dryrun:
help="Event ID to update")
parser.add_argument("-t",
"--type",
required=True,
help="Type of the generic object")
parser.add_argument("-l",
"--attr_list",
required=True,
help="List of attributes")
args = parser.parse_args()
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
try:
template_id = [
x['ObjectTemplate']['id']
for x in pymisp.get_object_templates_list()
if x['ObjectTemplate']['name'] == args.type
][0]
except IndexError:
valid_types = ", ".join([
x['ObjectTemplate']['name']
for x in pymisp.get_object_templates_list()
])
print("Template for type %s not found! Valid types are: %s" %
(args.type, valid_types))
exit()
misp_object = GenericObjectGenerator(args.type.replace("|", "-"))
misp_object.generate_attributes(json.loads(args.attr_list))
r = pymisp.add_object(args.event, template_id, misp_object)
raw_report = getFileDetail(apikey=api_key, resource=sample.id)
scandate = datetime.datetime.strptime(raw_report["scan_date"],
'%Y-%m-%d %H:%M:%S')
me = MISPEvent()
this_event_name = misp_event_name.format(scandate.strftime("%Y-%m-%d"))
search = pm.search(controller='events', eventinfo=this_event_name)
if (len(search) == 1):
me.load(search[0])
else:
me.info = this_event_name
pm.add_event(me)
vtreport = GenericObjectGenerator('virustotal-report')
vtreport.add_attribute("last-submission",
value=raw_report["scan_date"])
vtreport.add_attribute("permalink", value=raw_report["permalink"])
ratio = "{}/{}".format(raw_report["positives"], raw_report["total"])
vtreport.add_attribute("detection-ratio", value=ratio)
file_object = GenericObjectGenerator('file')
file_object.add_attribute("md5", value=raw_report["md5"])
file_object.add_attribute("sha1", value=raw_report["sha1"])
file_object.add_attribute("sha256", value=raw_report["sha256"])
file_object.add_attribute("ssdeep", value=raw_report["ssdeep"])
file_object.add_attribute("authentihash",
value=raw_report["authentihash"])
file_object.add_attribute("size-in-bytes", value=raw_report["size"])
me = MISPEvent()
if (this_event_name in events):
me = events[this_event_name]
else:
me.info = this_event_name
#Geolocation
geo = None
cc = data['geo_country']
country = pycountry.countries.get(alpha_2=cc)
if (country is not None):
if (me.info + cc in places):
geo = places[me.info + cc]
else:
geo = GenericObjectGenerator('geolocation')
geo.add_attribute(
"country",
value=pycountry.countries.get(alpha_2=cc).name)
me.add_object(geo)
places.update({me.info + cc: geo})
#x509
xfive = None
if ("issuer" in data):
if (len(data["issuer"]) > 2):
xfive = GenericObjectGenerator('x509')
xfive.add_attribute("issuer",
value=data["issuer"],
to_ids=True)
if (len(data["subject"]) > 0):
