def prepare_alert(helper, app_name):
instance = helper.get_param("misp_instance")
sessionKey = helper.settings['session_key']
splunkService = client.connect(token=sessionKey)
storage = splunkService.storage_passwords
config_args = prepare_config(helper, app_name, instance, storage)
if config_args is None:
return None
alert_args = dict()
# Get string values from alert form
alert_args['mode'] = str(helper.get_param("mode"))
alert_args['type'] = int(helper.get_param("type"))
alert_args['source'] = str(helper.get_param("source"))
if not helper.get_param("unique"):
alert_args['unique'] = "no_timestamp_field"
else:
alert_args['unique'] = str(helper.get_param("unique"))
config_args.update(alert_args)
return config_args
def prepare_alert(helper, app_name):
# Get parameters for calling prepare_config
instance = helper.get_param("misp_instance")
sessionKey = helper.settings['session_key']
splunkService = client.connect(token=sessionKey)
storage = splunkService.storage_passwords
helper.log_debug("[AL-PA-001] successfully retrieved storage object")
config_args = prepare_config(helper, app_name, instance, storage, sessionKey)
if config_args is None:
return None
alert_args = dict()
# Get string values from alert form
alert_args['tlp'] = str(helper.get_param("tlp").replace('_', ':'))
alert_args['pap'] = str(helper.get_param("pap").replace('_', ':'))
if not helper.get_param("eventid"):
alert_args['eventid'] = "0"
else:
alert_args['eventid'] = str(helper.get_param("eventid"))
if not helper.get_param("unique"):
alert_args['eventkey'] = "oneEvent"
else:
alert_args['eventkey'] = str(helper.get_param("unique"))
if not helper.get_param("info"):
alert_args['info'] = "notable event"
else:
alert_args['info'] = str(helper.get_param("info"))
published = helper.get_param("publish_on_creation")
if published == "1":
alert_args['published'] = True
else:
alert_args['published'] = False
if not helper.get_param("tags"):
alert_args['tags'] = None
else:
alert_args['tags'] = str(helper.get_param("tags"))
# Get numeric values from alert form
alert_args['analysis'] = int(helper.get_param("analysis"))
alert_args['threatlevel'] = int(helper.get_param("threatlevel"))
alert_args['distribution'] = int(helper.get_param("distribution"))
config_args.update(alert_args)
return config_args
def stream(self, records):
# self.self.logging.debug('mispgetioc.reduce')
# Generate args
my_args = prepare_config(self, 'misp42splunk')
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
search_url = my_args['misp_url'] + '/attributes/restSearch'
sight_url = my_args['misp_url'] + \
'/sightings/restSearch/attribute'
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
search_dict = {"returnFormat": "json"}
search_dict['value'] = str(value)
search_dict['withAttachments'] = "false",
search_body = json.dumps(search_dict)
sight_dict = {"returnFormat": "json"}
misp_value = ''
misp_fp = False
misp_fp_ts = 0
misp_fp_id = ''
ms_seen = False
ms = {
'count': 0,
'first': 0,
'f_id': 0,
'last': 0,
'l_id': 0
}
# search
logging.debug('mispsight request body: %s', search_body)
rs = requests.post(search_url,
headers=headers,
data=search_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
rs.raise_for_status()
# response is 200 by this point or we would
# have thrown an exception
response = rs.json()
logging.info(
"MISP REST API %s has got a response with status code 200",
search_url)
if 'response' in response:
if 'Attribute' in response['response']:
r_number = len(response['response']['Attribute'])
logging.debug(
"MISP REST API %s: response: with %s records" %
(search_url, str(r_number)))
for a in response['response']['Attribute']:
if misp_value == '':
misp_value = str(a['value'])
if misp_fp is False:
sight_dict['id'] = str(a['id'])
sight_body = json.dumps(sight_dict)
rt = requests.post(
sight_url,
headers=headers,
data=sight_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything
# other than 200; throw an exception
rt.raise_for_status()
# response is 200 by this point or we
# would have thrown an exception
sight = rt.json()
logging.info(
"MISP REST API %s has got a response with status code 200",
sight_url)
logging.debug(
"MISP REST API %s has got a response: with %s records"
% (sight_url, len(sight)))
if 'response' in sight:
for s in sight['response']:
if 'Sighting' in s:
# true sighting
ty = s['Sighting']['type']
ds = int(s['Sighting']
['date_sighting'])
ev = str(
s['Sighting']['event_id'])
if int(ty) == 0:
ms_seen = True
ms['count'] = \
ms['count'] + 1
if ms['first'] == 0 or \
ms['first'] > ds:
ms['first'] = ds
ms['f_id'] = ev
if ms['last'] < int(ds):
ms['last'] = int(ds)
ms['l_id'] = ev
# false positive
elif int(ty) == 1:
misp_fp = True
misp_fp_ts = ds
misp_fp_id = ev
if misp_fp is True:
record['misp_value'] = misp_value
record['misp_fp'] = "True"
record['misp_fp_timestamp'] = str(misp_fp_ts)
record['misp_fp_event_id'] = str(misp_fp_id)
if ms_seen is True:
record['misp_value'] = misp_value
record['misp_count'] = str(ms['count'])
record['misp_first'] = str(ms['first'])
record['misp_first_event_id'] = str(ms['f_id'])
record['misp_last'] = str(ms['last'])
record['misp_last_event_id'] = str(ms['l_id'])
yield record
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self, 'misp42splunk')
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getioc is True:
getioc = True
else:
getioc = False
if self.pipesplit is True:
pipesplit = True
else:
pipesplit = False
if self.output is not None:
output = self.output
else:
output = "default"
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.error('MispGetEventCommand request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if output == "raw":
if 'response' in response:
for r_item in response['response']:
if 'Event' in r_item:
for e in list(r_item.values()):
yield e
else:
results = []
# add colums for each type in results
typelist = []
format_output_table(response, results, typelist, getioc, pipesplit)
logging.info('typelist is %s', json.dumps(typelist))
# relevant_cat = ['Artifacts dropped', 'Financial fraud',
# 'Network activity','Payload delivery','Payload installation']
# logging.debug('results is %s', json.dumps(results))
logging.debug('results contains %s records', str(len(results)))
if getioc is False:
for e in results:
# logging.debug('event is %s', json.dumps(e))
yield e
else:
output_dict = {}
for e in results:
if 'Attribute' in e:
for r in e['Attribute']:
if int(r['misp_object_id']) == 0: # not an object
key = str(e['misp_event_id']) + '_' \
+ r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(e['misp_event_id']) + \
'_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = init_misp_output(e, r)
for t in typelist:
misp_t = 'misp_' \
+ t.replace('-', '_')\
.replace('|', '_p_')
v[misp_t] = []
if t == r['misp_type']:
v[misp_t].append(r['misp_value'])
v['misp_to_ids'] = []
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'] = []
v['misp_category'].append(r['misp_category'])
a_tag = []
if r['misp_attribute_tag'] is not None:
for t in r['misp_attribute_tag']:
a_tag.append(t)
v['misp_attribute_tag'] = a_tag
comment = []
comment.append(r['misp_comment'])
v['misp_comment'] = comment
v['misp_attribute_uuid'] = []
v['misp_attribute_uuid']\
.append(r['misp_attribute_uuid'])
v['misp_attribute_id'] = []
v['misp_attribute_id']\
.append(r['misp_attribute_id'])
if is_object_member is True:
v['misp_type'] = v['misp_object_name']
v['misp_value'] = v['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type']\
.replace('-', '_')
misp_value = v[misp_t]
misp_value.append(r['misp_value'])
v[misp_t] = misp_value
to_ids = v['misp_to_ids']
if r['misp_to_ids'] not in to_ids:
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = v['misp_category']
# append
if r['misp_category'] not in category:
category.append(r['misp_category'])
v['misp_category'] = category
v['misp_attribute_uuid']\
.append(r['misp_attribute_uuid'])
v['misp_attribute_id']\
.append(r['misp_attribute_id'])
if r['misp_attribute_tag'] is not None:
a_tag = v['misp_attribute_tag']
for t in r['misp_attribute_tag']:
if t not in a_tag:
a_tag.append(t)
v['misp_attribute_tag'] = a_tag
comment = v['misp_comment']
if r['misp_comment'] not in comment:
comment.append(r['misp_comment'])
v['misp_comment'] = comment
if is_object_member is False:
misp_type = r['misp_type'] \
+ '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + \
'|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
if output_dict is not None:
for k, v in list(output_dict.items()):
yield v
def stream(self, records):
# Generate args
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
if self.gettag is True:
get_tag = True
else:
get_tag = False
pagination = True
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 1000
if self.page is not None:
page = int(self.page)
else:
page = 1
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
if 'page' in body_dict:
page = body_dict['page']
pagination = False
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
if self.onlyids is True:
body_dict['to_ids'] = "True"
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
if self.last is not None:
body_dict['last'] = self.last
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict['value'] = str(value)
misp_category = []
misp_event_id = []
misp_event_uuid = []
misp_orgc_id = []
misp_to_ids = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispsearch request body: %s', body)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
# print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json()
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
if 'Event' in a:
if a['Event'][
'uuid'] not in misp_event_uuid:
misp_event_uuid.append(
str(a['Event']['uuid']))
if a['Event'][
'orgc_id'] not in misp_orgc_id:
misp_orgc_id.append(
str(a['Event']['orgc_id']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
record['misp_event_uuid'] = misp_event_uuid
record['misp_orgc_id'] = misp_orgc_id
record['misp_tag'] = misp_tag
yield record
def generate(self):
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
if self.target not in [None, '']:
my_args['misp_url'] = my_args['misp_url'] + self.target
if self.json_request not in [None, '']:
body_dict = json.loads(self.json_request)
logging.debug('[MR-201] body_dict is {}'.format(body_dict))
else:
body_dict = {}
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
if self.method == "GET":
r = requests.get(my_args['misp_url'],
headers=headers,
params=body_dict,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
elif self.method == "POST":
r = requests.post(my_args['misp_url'],
headers=headers,
data=json.dumps(body_dict),
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
elif self.method == "DELETE":
r = requests.delete(my_args['misp_url'],
headers=headers,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
else:
raise Exception(
"Sorry, no valid method provided (GET/POST//DELETE)."
" it was {}.".format(self.method))
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
logging.info("[RE301] INFO mispcollect successful. "
"url={}, HTTP status={}".format(
my_args['misp_url'], r.status_code))
else:
logging.error("[RE302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}".format(
my_args['misp_url'], body_dict, r.status_code,
r.text))
raise Exception(
"[RE302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}".format(
my_args['misp_url'], body_dict, r.status_code, r.text))
# response is 200 by this point or we would have thrown an exception
data = {'_time': time.time(), '_raw': json.dumps(r.json())}
yield data
def stream(self, records):
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception("Sorry, no configuration for misp_instance={}".format(misp_instance))
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
search_url = my_args['misp_url'] + '/attributes/restSearch'
sight_url = my_args['misp_url'] + \
'/sightings/restSearch/attribute'
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
search_dict = {"returnFormat": "json"}
search_dict['value'] = str(value)
search_dict['withAttachments'] = "false",
search_body = json.dumps(search_dict)
sight_dict = {"returnFormat": "json"}
misp_value = ''
misp_fp = False
misp_fp_ts = 0
misp_fp_id = ''
ms_seen = False
ms = {
'count': 0,
'first': 0,
'f_id': 0,
'last': 0,
'l_id': 0
}
# search
rs = requests.post(
search_url,
headers=headers,
data=search_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies']
)
# check if status is anything other than 200;
# throw an exception if it is
# check if status is anything other than 200;
# throw an exception if it is
if rs.status_code in (200, 201, 204):
logging.info(
"[SI301] INFO mispsight part 1 successful. "
"url={}, HTTP status={}".format(my_args['misp_url'], rs.status_code)
)
else:
logging.error(
"[SI302] ERROR mispsight part 1 failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], search_body, rs.status_code, rs.text)
)
raise Exception(
"[SI302] ERROR mispsight part 1 failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], search_body, rs.status_code, rs.text)
)
# response is 200 by this point or we would
# have thrown an exception
response = rs.json()
if 'response' in response:
if 'Attribute' in response['response']:
r_number = len(response['response']['Attribute'])
logging.info(
"MISP REST API %s: response: with %s records"
% (search_url, str(r_number))
)
for a in response['response']['Attribute']:
if misp_value == '':
misp_value = str(a['value'])
if misp_fp is False:
sight_dict['id'] = str(a['id'])
sight_body = json.dumps(sight_dict)
rt = requests.post(
sight_url,
headers=headers,
data=sight_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies']
)
# check if status is anything
# other than 200; throw an exception
if rt.status_code in (200, 201, 204):
logging.info(
"[SI301] INFO mispsight part 2 successful. "
"url={}, HTTP status={}".format(my_args['misp_url'], rt.status_code)
)
else:
logging.error(
"[SI302] ERROR mispsight part 2 failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], sight_body, rt.status_code, rt.text)
)
raise Exception(
"[SI302] ERROR mispsight part 2 failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], sight_body, rt.status_code, rt.text)
)
# response is 200 by this point or we
# would have thrown an exception
sight = rt.json()
if 'response' in sight:
for s in sight['response']:
if 'Sighting' in s:
# true sighting
ty = s['Sighting']['type']
ds = int(
s['Sighting']
['date_sighting']
)
ev = str(
s['Sighting']
['event_id']
)
if int(ty) == 0:
ms_seen = True
ms['count'] = \
ms['count'] + 1
if ms['first'] == 0 or \
ms['first'] > ds:
ms['first'] = ds
ms['f_id'] = ev
if ms['last'] < int(ds):
ms['last'] = int(ds)
ms['l_id'] = ev
# false positive
elif int(ty) == 1:
misp_fp = True
misp_fp_ts = ds
misp_fp_id = ev
if misp_fp is True:
record['misp_value'] = misp_value
record['misp_fp'] = "True"
record['misp_fp_timestamp'] = str(
misp_fp_ts
)
record['misp_fp_event_id'] = str(
misp_fp_id
)
if ms_seen is True:
record['misp_value'] = misp_value
record['misp_count'] = str(ms['count'])
record['misp_first'] = str(ms['first'])
record['misp_first_event_id'] = str(
ms['f_id']
)
record['misp_last'] = str(ms['last'])
record['misp_last_event_id'] = str(
ms['l_id']
)
yield record
def stream(self, records):
# loggging
self.set_log_level()
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
search_url = my_args['misp_url'] + '/attributes/restSearch'
sight_url = my_args['misp_url'] + \
'/sightings/restSearch/attribute'
# iterate through records from SPL
for record in records:
# if a record contains the field passed to mispsight command
# search by value in MISP to get attribute local ID.
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
search_dict = dict(returnFormat='json',
value=str(value),
withAttachments="false")
search_body = json.dumps(search_dict)
# search
rs = requests.post(search_url,
headers=headers,
data=search_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
# check if status is anything other than 200;
# throw an exception if it is
if rs.status_code in (200, 201, 204):
self.log_info(
"[SI301] INFO mispsight part 1 successful. url={}, HTTP status={}"
.format(my_args['misp_url'], rs.status_code))
else:
self.log_error(
"[SI302] ERROR mispsight part 1 failed. url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], search_body,
rs.status_code, rs.text))
raise Exception(
"[SI302] ERROR mispsight part 1 failed for url={} with HTTP Error={}. Check search.log for details"
.format(my_args['misp_url'], rs.status_code))
# response is 200 by this point or we would
# have thrown an exception
response = rs.json()
if 'response' in response:
if 'Attribute' in response['response']:
# MISP API returned a JSON response
r_number = len(response['response']['Attribute'])
self.log_info(
"MISP REST API {}: response: with {} records".
format(search_url, str(r_number)))
sightings = dict()
sight_types = ['t0', 't1', 't2']
metrics = [
'et', 'lt', 'first_a_id', 'last_a_id',
'first_e_id', 'last_e_id', 'first_org_id',
'last_org_id', 'first_source', 'last_source'
]
sight_counter = {
'misp_value': ''
# 'misp_sightings': []
}
for stype in sight_types:
skey = 'misp_sight_' + stype + '_count'
sight_counter[skey] = 0
for metric in metrics:
skey = 'misp_sight_' + stype + '_' + metric
sight_counter[skey] = ''
# iterate through results to get sighting counters
for a in response['response']['Attribute']:
misp_value = str(a['value'])
if misp_value in sightings:
a_sight = sightings[misp_value]
else:
a_sight = sight_counter.copy()
a_sight['misp_value'] = misp_value
sight_dict = {"returnFormat": "json"}
sight_dict['id'] = str(a['id'])
sight_body = json.dumps(sight_dict)
rt = requests.post(
sight_url,
headers=headers,
data=sight_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything
# other than 200; throw an exception
if rt.status_code in (200, 201, 204):
self.log_info(
"[SI301] INFO mispsight part 2 successful. url={}, HTTP status={}"
.format(my_args['misp_url'],
rt.status_code))
else:
self.log_error(
"[SI302] ERROR mispsight part 2 failed. url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'],
sight_body, rt.status_code,
rt.text))
raise Exception(
"[SI302] ERROR mispsight part 2 failed for url={} with HTTP Error={}. Check search.log for details"
.format(my_args['misp_url'],
rt.status_code))
# response is 200 by this point or we
# would have thrown an exception
sight = rt.json()
if 'response' in sight:
for s in sight['response']:
if 'Sighting' in s:
# true sighting
sr = s['Sighting']
ty = int(sr['type'])
if ty == 0:
MispSightCommand._sight_metric(
a_sight, 't0', sr)
elif ty == 1:
MispSightCommand._sight_metric(
a_sight, 't1', sr)
elif ty == 2:
MispSightCommand._sight_metric(
a_sight, 't2', sr)
sightings[misp_value] = a_sight
init_record = True
for srec in sightings.values():
self.log_debug("[SI401] srec={}".format(
json.dumps(srec)))
if init_record is True:
for key, value in sorted(srec.items()):
record[key] = [value]
init_record = False
else:
for key, value in sorted(srec.items()):
record[key].append(value)
yield record
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# build search JSON object
body_dict = {"returnFormat": "json",
"withAttachments": False,
"deleted": False
}
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date_from:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "eventid", "last" or "date_from" argument')
raise Exception('Missing "eventid", "last" or "date_from" argument')
elif mandatory_arg > 1:
logging.error('Options "eventid", "last" and "date_from" are mutually exclusive')
raise Exception('Options "eventid", "last" and "date_from" are mutually exclusive')
# Only ONE combination was provided
if self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set')
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', body_dict['last'])
else:
body_dict['from'] = self.date_from
logging.info('Option "date_from" set with %s', body_dict['from'])
if self.date_to:
body_dict['to'] = self.date_to
logging.info('Option "date_to" set with %s', body_dict['to'])
else:
logging.info('Option "date_to" will be set to now().')
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
other_page = True
page = 1
page_length = 0
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 10000
# Search parameters: boolean and filter
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.category is not None:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
if self.type is not None:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if self.threat_level_id is not None:
body_dict['threat_level_id'] = int(self.threat_level_id)
# output filter parameters
if self.getuuid is True:
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.getorg is True:
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
if self.add_description is True:
my_args['add_desc'] = True
else:
my_args['add_desc'] = False
results = []
# add colums for each type in results
typelist = []
while other_page:
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispgetioc request body: %s', body)
# search
r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
page_lenth = len(response['response']['Attribute'])
for a in response['response']['Attribute']:
v = {}
v['misp_category'] = str(a['category'])
v['misp_attribute_id'] = str(a['id'])
v['misp_event_id'] = str(a['event_id'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_to_ids'] = str(a['to_ids'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
# include ID of the organisation that created the attribute if requested
# in previous version this was the ORG name ==> create lookup
if 'Event' in a and my_args['getorg']:
v['misp_orgc_id'] = str(a['Event']['orgc_id'])
# include attribute UUID if requested
if my_args['getuuid']:
v['misp_attribute_uuid'] = str(a['uuid'])
# handle object and multivalue attributes
v['misp_object_id'] = str(a['object_id'])
if my_args['add_desc'] is True:
if int(a['object_id']) == 0:
v['misp_description'] = 'MISP e' + str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' + str(a['type']) \
+ '" in category "' + str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) + ')'
else:
v['misp_description'] = 'MISP e' + str(a['event_id']) \
+ ' attribute ' + str(a['uuid']) + ' of type "' \
+ str(a['type']) + '" in category "' \
+ str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) \
+ ' - o' + str(a['object_id']) + ' )'
current_type = str(a['type'])
# combined: not part of an object AND multivalue attribute QND to be split
if int(a['object_id']) == 0 and '|' in current_type and my_args['pipe'] is True:
mv_type_list = current_type.split('|')
mv_value_list = str(a['value']).split('|')
left_v = v.copy()
left_v['misp_type'] = mv_type_list.pop()
left_v['misp_value'] = mv_value_list.pop()
results.append(left_v)
if left_v['misp_type'] not in typelist:
typelist.append(left_v['misp_type'])
right_v = v.copy()
right_v['misp_type'] = mv_type_list.pop()
right_v['misp_value'] = mv_value_list.pop()
results.append(right_v)
if right_v['misp_type'] not in typelist:
typelist.append(right_v['misp_type'])
else:
v['misp_type'] = current_type
v['misp_value'] = str(a['value'])
results.append(v)
if current_type not in typelist:
typelist.append(current_type)
if pagination is True:
if page_length < limit:
other_page = False
else:
page = page + 1
else:
other_page = False
logging.info(json.dumps(typelist))
output_dict = {}
# relevant_cat = ['Artifacts dropped', 'Financial fraud', 'Network activity','Payload delivery','Payload installation']
for r in results:
if int(r['misp_object_id']) == 0: # not an object
key = str(r['misp_event_id']) + '_' + r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(r['misp_event_id']) + '_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
to_ids = []
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = []
category.append(r['misp_category'])
v['misp_category'] = category
if my_args['add_desc'] is True:
description = []
description.append(r['misp_description'])
v['misp_description'] = description
if my_args['getuuid'] is True:
attribute_uuid = []
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is True:
v['misp_type'] = 'misp_object'
v['misp_value'] = r['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace('-', '_')
v[misp_t] = r['misp_value'] # set value for relevant type
to_ids = v['misp_to_ids']
if r['misp_to_ids'] not in to_ids:
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = v['misp_category']
if r['misp_category'] not in category: # append category
category.append(r['misp_category'])
v['misp_category'] = category
if my_args['add_desc'] is True:
description = v['misp_description']
if r['misp_description'] not in description:
description.append(r['misp_description'])
v['misp_description'] = description
if my_args['getuuid'] is True:
attribute_uuid = v['misp_attribute_uuid']
if r['misp_attribute_uuid'] not in attribute_uuid:
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is False:
misp_type = r['misp_type'] + '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + '|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
for k, v in output_dict.items():
yield v
def generate(self):
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception("Sorry, no configuration for misp_instance={}".format(misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
raise Exception('Missing "json_request", "eventid", "last" or "date" argument')
elif mandatory_arg > 1:
raise Exception('Options "json_request", "eventid", "last" and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
# manage to_ids and enforceWarninglist
# to avoid FP enforceWarninglist is set to True if
# to_ids is set to True (search criterion)
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.endpoint == 'events':
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
else:
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.keep_related is True:
keep_related = True
else:
keep_related = False
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispgetioc request body: %s', body)
# search
r = requests.post(my_args['misp_url'], headers=headers, data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
logging.info(
"[CO301] INFO mispcollect successful. "
"url={}, HTTP status={}".format(my_args['misp_url'], r.status_code)
)
else:
logging.error(
"[CO302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code, r.text)
)
raise Exception(
"[CO302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code, r.text)
)
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if self.endpoint == "events":
if 'response' in response:
for r_item in response['response']:
if 'Event' in r_item:
attribute_names = []
serial_number = 0
for e in list(r_item.values()):
if keep_related is False:
e.pop('RelatedEvent', None)
if serial_number == 0:
for k in list(e.keys()):
attribute_names.append(k)
yield MispCollectCommand._record(
serial_number, e['timestamp'], my_args['host'],
e, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
if 'response' in response:
if 'Attribute' in response['response']:
attribute_names = []
serial_number = 0
for a in response['response']['Attribute']:
if serial_number == 0:
for k in list(a.keys()):
attribute_names.append(k)
yield MispCollectCommand._record(
serial_number, a['timestamp'], my_args['host'],
a, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
def generate(self):
# Phase 1: Preparation
my_args = prepare_config(self, 'misp42splunk')
my_args['host'] = my_args['misp_url'].replace('https://', '')
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
# manage to_ids and enforceWarninglist
# to avoid FP enforceWarninglist is set to True if
# to_ids is set to True (search criterion)
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getuuid is True:
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.getorg is True:
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
if self.add_description is True:
my_args['add_desc'] = True
else:
my_args['add_desc'] = False
if self.output is not None:
my_args['output'] = self.output
else:
my_args['output'] = "default"
results = []
# add colums for each type in results
typelist = []
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispgetioc request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if my_args['output'] == "raw":
if 'response' in response:
if 'Attribute' in response['response']:
attribute_names = []
serial_number = 0
for a in response['response']['Attribute']:
yield MispGetIocCommand._record(
serial_number, a['timestamp'], my_args['host'], a,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
v = {}
v['misp_category'] = str(a['category'])
v['misp_attribute_id'] = str(a['id'])
v['misp_event_id'] = str(a['event_id'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_to_ids'] = str(a['to_ids'])
v['misp_comment'] = str(a['comment'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
# include ID of the organisation that
# created the attribute if requested
if 'Event' in a:
v['misp_event_uuid'] = str(a['Event']['uuid'])
if my_args['getorg']:
v['misp_orgc_id'] = str(a['Event']['orgc_id'])
if my_args['add_desc'] is True:
v['misp_event_info'] = str(a['Event']['info'])
# include attribute UUID if requested
if my_args['getuuid']:
v['misp_attribute_uuid'] = str(a['uuid'])
# handle object and multivalue attributes
v['misp_object_id'] = str(a['object_id'])
if my_args['add_desc'] is True:
if int(a['object_id']) == 0:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) \
+ '" in category "' + str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) + ')'
else:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) + '" in category "' \
+ str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) \
+ ' - o' + str(a['object_id']) + ' )'
current_type = str(a['type'])
# combined: not part of an object
# AND multivalue attribute AND to be split
# logging.debug('misp_event: %s', json.dumps(v))
if int(a['object_id']) == 0 and '|' in current_type \
and my_args['pipe'] is True:
mv_type_list = current_type.split('|')
mv_value_list = str(a['value']).split('|')
left_v = v.copy()
left_v['misp_type'] = mv_type_list.pop()
left_v['misp_value'] = mv_value_list.pop()
results.append(left_v)
if left_v['misp_type'] not in typelist:
typelist.append(left_v['misp_type'])
right_v = v.copy()
right_v['misp_type'] = mv_type_list.pop()
right_v['misp_value'] = mv_value_list.pop()
results.append(right_v)
if right_v['misp_type'] not in typelist:
typelist.append(right_v['misp_type'])
else:
v['misp_type'] = current_type
v['misp_value'] = str(a['value'])
results.append(v)
if current_type not in typelist:
typelist.append(current_type)
logging.info(json.dumps(typelist))
output_dict = {}
attribute_names = [
'misp_attribute_id', 'misp_attribute_uuid', 'misp_category',
'misp_comment', 'misp_description', 'misp_event_id',
'misp_tag', 'misp_timestamp', 'misp_to_ids', 'misp_type',
'misp_value'
]
if my_args['add_desc'] is True:
attribute_names.append('misp_event_info')
for t in typelist:
misp_t = 'misp_' + \
t.replace('-', '_').replace('|', '_p_')
if misp_t not in attribute_names:
attribute_names.append(misp_t)
for r in results:
if int(r['misp_object_id']) == 0: # not an object
key = str(r['misp_event_id']) + \
'_' + r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(r['misp_event_id']) \
+ '_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_')\
.replace('|', '_p_')
v[misp_t] = []
if t == r['misp_type']:
v[misp_t].append(r['misp_value'])
v['misp_to_ids'] = []
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'] = []
v['misp_category'].append(r['misp_category'])
v['misp_attribute_id'] = []
v['misp_attribute_id']\
.append(r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid'] = []
v['misp_attribute_uuid']\
.append(r['misp_attribute_uuid'])
if my_args['add_desc'] is True:
description = []
description.append(r['misp_description'])
v['misp_description'] = description
if is_object_member is True:
v['misp_type'] = 'misp_object'
v['misp_value'] = r['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace('-', '_')
v[misp_t].append(r['misp_value']) # set value for type
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'].append(r['misp_category'])
tag_list = v['misp_tag']
for tag in r['misp_tag']:
if tag not in tag_list:
tag_list.append(tag)
v['misp_tag'] = tag_list
if my_args['add_desc'] is True:
description = v['misp_description']
if r['misp_description'] not in description:
description.append(r['misp_description'])
v['misp_description'] = description
v['misp_attribute_id']\
.append(r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid']\
.append(r['misp_attribute_uuid'])
if is_object_member is False:
misp_type = r['misp_type'] + '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + '|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
serial_number = 0
logging.debug(json.dumps(attribute_names))
for k, v in list(output_dict.items()):
yield MispGetIocCommand._record(serial_number,
v['misp_timestamp'],
my_args['host'], v,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
def stream(self, records):
# self.logger.debug('mispgetioc.reduce')
# Generate args
my_args = prepare_config(self)
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
search_url = my_args['misp_url'] + '/attributes/restSearch'
search_dict = {"returnFormat": "json"}
search_dict['value'] = str(value)
search_dict['withAttachments'] = "false",
search_body = json.dumps(search_dict)
sight_url = my_args[
'misp_url'] + '/sightings/restSearch/attribute'
sight_dict = {"returnFormat": "json"}
misp_value = ''
misp_fp = False
misp_fp_timestamp = 0
misp_fp_event_id = ''
misp_sight_seen = False
misp_sight = {
'count': 0,
'first': 0,
'first_event_id': 0,
'last': 0,
'last_event_id': 0
}
# search
r = requests.post(search_url,
headers=headers,
data=search_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
logging.info("INFO MISP REST API %s has response: %s" %
(search_url, r.json()))
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if misp_value == '':
misp_value = str(a['value'])
if misp_fp == False:
sight_dict['id'] = str(a['id'])
sight_body = json.dumps(sight_dict)
s = requests.post(
sight_url,
headers=headers,
data=sight_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
s.raise_for_status()
# response is 200 by this point or we would have thrown an exception
sight = s.json()
logging.info(
"INFO MISP REST API %s has response: %s"
% (sight_url, s.json()))
if 'response' in sight:
for se in sight['response']:
if 'Sighting' in se:
if int(se['Sighting']['type']
) == 0: #true sighting
misp_sight_seen = True
misp_sight[
'count'] = misp_sight[
'count'] + 1
if misp_sight['first'] == 0 or \
misp_sight['first'] > int(se['Sighting']['date_sighting']):
misp_sight[
'first'] = int(
se['Sighting']
['date_sighting']
)
misp_sight[
'first_event_id'] = se[
'Sighting'][
'event_id']
if misp_sight['last'] < int(
se['Sighting']
['date_sighting']):
misp_sight['last'] = int(
se['Sighting']
['date_sighting'])
misp_sight[
'last_event_id'] = se[
'Sighting'][
'event_id']
elif int(
se['Sighting']['type']
) == 1: #false positive
misp_fp = True
misp_fp_timestamp = int(
se['Sighting']
['date_sighting'])
misp_fp_event_id = se[
'Sighting']['event_id']
if misp_fp == True:
record['misp_value'] = misp_value
record['misp_fp'] = "True"
record['misp_fp_timestamp'] = str(
misp_fp_timestamp)
record['misp_fp_event_id'] = str(
misp_fp_event_id)
if misp_sight_seen == True:
record['misp_value'] = misp_value
record['misp_sight_count'] = str(
misp_sight['count'])
record['misp_sight_first'] = str(
misp_sight['first'])
record['misp_sight_first_event_id'] = str(
misp_sight['first_event_id'])
record['misp_sight_last'] = str(
misp_sight['last'])
record['misp_sight_last_event_id'] = str(
misp_sight['last_event_id'])
yield record
def generate(self):
# Phase 1: Preparation
my_args = prepare_config(self, 'misp42splunk')
my_args['host'] = my_args['misp_url'].replace('https://', '')
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
# Search parameters: boolean and filter
# manage enforceWarninglist
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getioc is True:
getioc = True
else:
getioc = False
if self.pipesplit is True:
pipesplit = True
else:
pipesplit = False
if self.output is not None:
output = self.output
else:
output = "default"
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.error('MispGetEventCommand request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if output == "raw":
if 'response' in response:
attribute_names = list()
serial_number = 0
for r_item in response['response']:
if 'Event' in r_item:
for e in r_item.values():
yield MispGetEventCommand._record(
serial_number, e['timestamp'], my_args['host'],
e, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
# build output table and list of types
events = []
typelist = []
column_list = format_output_table(response, events, typelist,
getioc, pipesplit)
logging.info('typelist containss %s values', str(len(typelist)))
logging.debug('typelist is %s', json.dumps(typelist))
logging.info('results contains %s records', str(len(events)))
if getioc is False:
attribute_names = list()
init_attribute_names = True
serial_number = 0
for e in events:
# logging.debug('event is %s', json.dumps(e))
if init_attribute_names is True:
for key in e.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
logging.debug(json.dumps(attribute_names))
yield MispGetEventCommand._record(serial_number,
e['misp_timestamp'],
my_args['host'], e,
attribute_names, encoder,
True)
serial_number += 1
GeneratingCommand.flush
else:
output_dict = {}
for e in events:
if 'Attribute' in e:
for a in e['Attribute']:
if int(a['misp_object_id']) == 0: # not an object
key = str(e['misp_event_id']) + '_' \
+ str(a['misp_attribute_id'])
is_object_member = False
else: # this is a MISP object
key = str(e['misp_event_id']) + \
'_object_' + str(a['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = init_misp_output(e, a, column_list)
for t in typelist:
misp_t = 'misp_' \
+ t.replace('-', '_')\
.replace('|', '_p_')
v[misp_t] = []
if t == a['misp_type']:
v[misp_t].append(a['misp_value'])
if is_object_member is True:
v['misp_type'] = v['misp_object_name']
v['misp_value'] = v['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + a['misp_type']\
.replace('-', '_').replace('|', '_p_')
v[misp_t].append(a['misp_value'])
if a['misp_to_ids'] not in v['misp_to_ids']:
v['misp_to_ids'].append(a['misp_to_ids'])
if a['misp_category'] not in v[
'misp_category']:
v['misp_category'].append(
a['misp_category'])
v['misp_attribute_uuid']\
.append(a['misp_attribute_uuid'])
v['misp_attribute_id']\
.append(a['misp_attribute_id'])
if a['misp_attribute_tag'] is not None:
a_tag = v['misp_attribute_tag']
for t in a['misp_attribute_tag']:
if t not in a_tag:
a_tag.append(t)
v['misp_attribute_tag'] = a_tag
if a['misp_comment'] not in v['misp_comment']:
v['misp_comment'].append(a['misp_comment'])
if is_object_member is False:
misp_type = a['misp_type'] \
+ '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = a['misp_value'] + \
'|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
if output_dict is not None:
attribute_names = list()
init_attribute_names = True
serial_number = 0
for v in output_dict.values():
if init_attribute_names is True:
for key in v.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
logging.debug(json.dumps(attribute_names))
yield MispGetEventCommand._record(
serial_number, v['misp_timestamp'],
my_args['host'], v, attribute_names, encoder, True)
serial_number += 1
GeneratingCommand.flush
def generate(self):
# loggging
self.set_log_level()
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
self.log_error(
'Missing "json_request", eventid", "last" or "date" argument')
raise Exception(
'Missing "json_request", "eventid", "last" or "date" argument')
elif mandatory_arg > 1:
self.log_error(
'Options "json_request", eventid", "last" and "date" are mutually exclusive'
)
raise Exception(
'Options "json_request", "eventid", "last" and "date" are mutually exclusive'
)
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
self.log_info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
self.log_info('Option "eventid" set with {}'.format(
json.dumps(body_dict['eventid'])))
elif self.last:
body_dict['last'] = self.last
self.log_info('Option "last" set with {}'.format(
body_dict['last']))
else:
body_dict['date'] = self.date.split()
self.log_info('Option "date" set with {}'.format(
json.dumps(body_dict['date'])))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
# manage to_ids and enforceWarninglist
# to avoid FP enforceWarninglist is set to True if
# to_ids is set to True (search criterion)
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.add_description is True:
my_args['add_desc'] = True
else:
my_args['add_desc'] = False
if self.expand_object is True:
my_args['expand'] = True
else:
my_args['expand'] = False
if self.getorg is True:
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.getuuid is True:
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
if self.output is not None:
my_args['output'] = self.output
else:
my_args['output'] = "default"
# add colums for each type in results
results = []
typelist = []
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
self.log_info(
"[IO301] INFO mispgetioc successful. url={}, HTTP status={}".
format(my_args['misp_url'], r.status_code))
else:
self.log_error(
"[IO302] ERROR mispgetioc failed. url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code, r.text))
raise Exception(
"[IO302] ERROR mispgetioc failed for url={} with HTTP Error={}. Check search.log for details"
.format(my_args['misp_url'], r.status_code))
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
# if raw output, returns JSON 1st-level keys as columns
if my_args['output'] == "raw":
if 'response' in response:
if 'Attribute' in response['response']:
attribute_names = list()
serial_number = 0
for a in response['response']['Attribute']:
yield MispGetIocCommand._record(
serial_number, a['timestamp'], my_args['host'], a,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
# default output: extract some values from JSON attributes
else:
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
v = {}
v['misp_category'] = str(a['category'])
v['misp_attribute_id'] = str(a['id'])
v['misp_event_id'] = str(a['event_id'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_to_ids'] = str(a['to_ids'])
v['misp_comment'] = str(a['comment'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
# include ID of the organisation that
# created the attribute if requested
if 'Event' in a:
v['misp_event_uuid'] = str(a['Event']['uuid'])
if my_args['getorg']:
v['misp_orgc_id'] = str(a['Event']['orgc_id'])
if my_args['add_desc'] is True:
v['misp_event_info'] = str(a['Event']['info'])
# include attribute UUID if requested
if my_args['getuuid']:
v['misp_attribute_uuid'] = str(a['uuid'])
# handle object and multivalue attributes
v['misp_object_id'] = str(a['object_id'])
if my_args['add_desc'] is True:
if int(a['object_id']) == 0:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) \
+ '" in category "' + str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) + ')'
else:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) + '" in category "' \
+ str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) \
+ ' - o' + str(a['object_id']) + ' )'
current_type = str(a['type'])
# combined: not part of an object
# AND multivalue attribute AND to be split
if int(a['object_id']) == 0 and '|' in current_type \
and my_args['pipe'] is True:
mv_type_list = current_type.split('|')
mv_value_list = str(a['value']).split('|')
left_v = v.copy()
left_v['misp_type'] = mv_type_list.pop()
left_v['misp_value'] = mv_value_list.pop()
results.append(left_v)
if left_v['misp_type'] not in typelist:
typelist.append(left_v['misp_type'])
right_v = v.copy()
right_v['misp_type'] = mv_type_list.pop()
right_v['misp_value'] = mv_value_list.pop()
results.append(right_v)
if right_v['misp_type'] not in typelist:
typelist.append(right_v['misp_type'])
else:
v['misp_type'] = current_type
v['misp_value'] = str(a['value'])
results.append(v)
if current_type not in typelist:
typelist.append(current_type)
self.log_info(json.dumps(typelist))
# consolidate attribute values under output table
if my_args['expand'] is True:
output_dict = {}
for r in results:
key = str(r['misp_event_id']) + \
'_' + str(r['misp_attribute_id'])
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace(
'|', '_p_')
v[misp_t] = []
if t == r['misp_type']:
v[misp_t].append(r['misp_value'])
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace(
'-', '_').replace('|', '_p_')
v[misp_t].append(r['misp_value']) # set value for type
misp_type = r['misp_type'] + '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = str(r['misp_value']) + '|' + str(
v['misp_value'])
v['misp_value'] = misp_value
output_dict[key] = dict(v)
else:
output_dict = {}
for r in results:
if int(r['misp_object_id']) == 0: # not an object
key = str(r['misp_event_id']) + \
'_' + str(r['misp_attribute_id'])
is_object_member = False
else: # this is a MISP object
key = str(r['misp_event_id']) \
+ '_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace(
'|', '_p_')
v[misp_t] = []
if t == r['misp_type']:
v[misp_t].append(r['misp_value'])
v['misp_to_ids'] = []
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'] = []
v['misp_category'].append(r['misp_category'])
v['misp_attribute_id'] = []
v['misp_attribute_id'].append(r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid'] = []
v['misp_attribute_uuid'].append(
r['misp_attribute_uuid'])
if my_args['add_desc'] is True:
v['misp_description'] = []
v['misp_description'].append(r['misp_description'])
v['misp_type'] = []
v['misp_type'].append(r['misp_type'])
v['misp_value'] = []
v['misp_value'].append(str(r['misp_value']))
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace(
'-', '_').replace('|', '_p_')
v[misp_t].append(r['misp_value']) # set value for type
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'].append(r['misp_category'])
tag_list = v['misp_tag']
for tag in r['misp_tag']:
if tag not in tag_list:
tag_list.append(tag)
v['misp_tag'] = tag_list
if my_args['add_desc'] is True:
description = v['misp_description']
if r['misp_description'] not in description:
description.append(r['misp_description'])
v['misp_description'] = description
if is_object_member is False:
if r['misp_attribute_id'] not in v[
'misp_attribute_id']:
v['misp_attribute_id'].append(
r['misp_attribute_id'])
if my_args['getuuid'] is True:
if r['misp_attribute_uuid'] not in v[
'misp_attribute_uuid']:
v['misp_attribute_uuid'].append(
r['misp_attribute_uuid'])
misp_type = []
misp_type.append(r['misp_type'] + '|' +
v['misp_type'][0])
v['misp_type'] = misp_type
misp_value = []
misp_value.append(
str(r['misp_value']) + '|' +
str(v['misp_value'][0]))
v['misp_value'] = misp_value
else:
v['misp_attribute_id'].append(
r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid'].append(
r['misp_attribute_uuid'])
v['misp_type'].append(r['misp_type'])
v['misp_value'].append(r['misp_value'])
output_dict[key] = dict(v)
# return output table
attribute_names = list()
init_attribute_names = True
serial_number = 0
for v in output_dict.values():
if init_attribute_names is True:
for key in v.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
yield MispGetIocCommand._record(serial_number,
v['misp_timestamp'],
my_args['host'], v,
attribute_names, encoder, True)
serial_number += 1
GeneratingCommand.flush
def stream(self, records):
# loggging
self.set_log_level()
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
pagination = True
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 1000
if self.page is not None:
page = int(self.page)
else:
page = 1
if self.json_request is not None:
body_dict = json.loads(self.json_request)
self.log_info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
if 'page' in body_dict:
page = body_dict['page']
pagination = False
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
if self.to_ids is True:
body_dict['to_ids'] = "True"
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
if self.last is not None:
body_dict['last'] = self.last
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict['value'] = str(value)
misp_category = []
misp_event_id = []
misp_event_uuid = []
misp_orgc_id = []
misp_to_ids = []
misp_comment = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
self.log_info(
"[SE301] INFO mispsearch successful. url={}, HTTP status={}"
.format(my_args['misp_url'], r.status_code))
else:
self.log_error(
"[SE302] ERROR mispsearch failed. url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code,
r.text))
raise Exception(
"[SE302] ERROR mispsearch failed for url={} with HTTP Error={}. Check search.log for details"
.format(my_args['misp_url'], r.status_code))
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['comment']) not in misp_comment:
misp_comment.append(str(a['comment']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
if 'Event' in a:
if a['Event']['uuid'] \
not in misp_event_uuid:
misp_event_uuid.append(
str(a['Event']['uuid']))
if a['Event']['orgc_id'] \
not in misp_orgc_id:
misp_orgc_id.append(
str(a['Event']['orgc_id']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_comment'] = misp_comment
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
record['misp_event_uuid'] = misp_event_uuid
record['misp_orgc_id'] = misp_orgc_id
record['misp_tag'] = misp_tag
yield record
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getioc is True:
my_args['getioc'] = True
else:
my_args['getioc'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
results = []
# add colums for each type in results
typelist = []
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.error('mispgetevent request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
for r_item in response['response']:
if 'Event' in r_item:
for a in list(r_item.values()):
v = {}
v['misp_event_id'] = str(a['id'])
v['misp_orgc_id'] = str(a['orgc_id'])
v['misp_event_date'] = str(a['date'])
v['threat_level_id'] = str(a['threat_level_id'])
v['misp_event_info'] = a['info']
v['misp_event_published'] = str(a['published'])
v['misp_event_uuid'] = str(a['uuid'])
v['misp_attribute_count'] = str(a['attribute_count'])
v['misp_analysis'] = str(a['analysis'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_distribution'] = str(a['distribution'])
v['misp_publish_timestamp'] = \
str(a['publish_timestamp'])
v['misp_sharing_group_id'] = str(a['sharing_group_id'])
v['misp_extends_uuid'] = str(a['extends_uuid'])
if 'Orgc' in a:
v['misp_orgc_name'] = str(a['Orgc']['name'])
v['misp_orgc_uuid'] = str(a['Orgc']['uuid'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
if my_args['getioc'] is True:
v['Attribute'] = list()
v['misp_attribute_count'] = 0
if 'Attribute' in a:
v['misp_attribute_count'] = \
v['misp_attribute_count'] + len(a['Attribute'])
if my_args['getioc'] is True:
for attribute in a['Attribute']:
# combined: not part of an object AND
# multivalue attribute AND to be split
if int(attribute['object_id']) == 0 \
and '|' in attribute['type'] \
and my_args['pipe'] is True:
mv_type_list = \
attribute['type'].split('|')
mv_value_list = \
str(attribute['value']).split('|')
left_a = attribute.copy()
left_a['type'] = mv_type_list.pop()
left_a['value'] = mv_value_list.pop()
v['Attribute'].append(
getioc(left_a, typelist,
my_args['pipe'],
left_a['object_id']))
right_a = attribute.copy()
right_a['type'] = mv_type_list.pop()
right_a['value'] = mv_value_list.pop()
v['Attribute'].append(
getioc(right_a, typelist,
my_args['pipe'],
right_a['object_id']))
else:
v['Attribute'].append(
getioc(attribute, typelist,
my_args['pipe'],
attribute['object_id']))
if 'Object' in a:
for misp_o in a['Object']:
if 'Attribute' in misp_o:
v['misp_attribute_count'] = \
v['misp_attribute_count'] \
+ len(misp_o['Attribute'])
if my_args['getioc'] is True:
object_id = misp_o['id']
object_name = misp_o['name']
object_comment = misp_o['comment']
for attribute in misp_o['Attribute']:
v['Attribute'].append(
getioc(attribute, typelist,
my_args['pipe'],
object_id, object_name,
object_comment))
logging.debug('event is %s', json.dumps(v))
results.append(v)
logging.info('typelist is %s', json.dumps(typelist))
# relevant_cat = ['Artifacts dropped', 'Financial fraud',
# 'Network activity','Payload delivery','Payload installation']
logging.debug('results is %s', json.dumps(results))
if my_args['getioc'] is False:
for e in results:
yield e
else:
output_dict = {}
for e in results:
if 'Attribute' in e:
for r in e['Attribute']:
if int(r['misp_object_id']) == 0: # not an object
key = str(e['misp_event_id']) + '_' \
+ r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(e['misp_event_id']) + \
'_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = init_misp_output(e, r)
for t in typelist:
misp_t = 'misp_' \
+ t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
to_ids = []
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = []
category.append(r['misp_category'])
v['misp_category'] = category
attribute_uuid = []
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is True:
v['misp_type'] = v['misp_object_name']
v['misp_value'] = v['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace('-', '_')
v[misp_t] = r['misp_value'] # set value for type
to_ids = v['misp_to_ids']
if r['misp_to_ids'] not in to_ids:
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = v['misp_category']
# append
if r['misp_category'] not in category:
category.append(r['misp_category'])
v['misp_category'] = category
attribute_uuid = v['misp_attribute_uuid']
if r['misp_attribute_uuid'] not in attribute_uuid:
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is False:
misp_type = r['misp_type'] \
+ '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + \
'|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
for k, v in list(output_dict.items()):
yield v
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
jsonmode = False
if self.mode is not None:
if 'j' in self.mode and self.json_request is not None:
jsonmode = True
if jsonmode is True:
pagination = True
other_page = True
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
else:
limit = 10000
if 'page' in body_dict:
page = body_dict['page']
else:
page = 1
page_length = 0
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
# add provided parameters to JSON request body
# specific formats
if self.last is not None:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', body_dict['last'])
if self.date_from is not None:
body_dict['from'] = self.date_from
logging.info('Option "date_from" set with %s',
body_dict['from'])
if self.date_to is not None:
body_dict['to'] = self.date_to
logging.info('Option "date_to" set with %s',
body_dict['to'])
else:
logging.info('Option "date_to" will be set to now().')
if self.threat_level_id is not None:
body_dict['threat_level_id'] = self.threat_level_id
logging.info('Option "threat_level_id" set with %s',
body_dict['threat_level_id'])
if self.org is not None:
body_dict['org'] = self.org
logging.info('Option "org" set')
if self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set')
if self.value is not None:
body_dict['value'] = self.value
logging.info('Option "value" set')
if self.eventinfo is not None:
body_dict['eventinfo'] = self.eventinfo
logging.info('Option "eventinfo" set')
# CSV strings
if self.category is not None:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
if self.type is not None:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if self.uuid is not None:
uuid_criteria = {}
uuid_list = self.uuid.split(",")
uuid_criteria['OR'] = uuid_list
body_dict['uuid'] = uuid_criteria
# Booleans
if self.to_ids is not None:
body_dict['to_ids'] = self.to_ids
logging.info('Option "to_ids" set with %s',
body_dict['to_ids'])
if self.enforceWarninglist is not None:
body_dict['enforceWarninglist'] = self.enforceWarninglist
logging.info('Option "enforceWarninglist" set with %s',
body_dict['enforceWarninglist'])
if self.deleted is not None:
body_dict['deleted'] = self.deleted
logging.info('Option "deleted" set with %s',
body_dict['deleted'])
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
logging.info('Option "includeEventUuid" set with %s',
body_dict['includeEventUuid'])
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
logging.info('Option "includeEventTags" set with %s',
body_dict['includeEventTags'])
# Search pagination
pagination = True
other_page = True
if self.page:
page = self.page
else:
page = 1
page_length = 0
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 10000
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
results = []
# add colums for each type in results
while other_page:
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispapireport request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
page_length = len(response['response']['Attribute'])
for a in response['response']['Attribute']:
v = {}
v['misp_Object'] = "-"
if self.includeEventTags is True:
v['misp_tag'] = "-"
for ak, av in a.items():
if ak == 'Event':
json_event = a['Event']
for ek, ev in json_event.items():
key = 'misp_event_' + ek
v[key] = str(ev)
elif ak == 'Tag':
tag_list = []
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
else:
vkey = 'misp_' + ak
v[vkey] = av
results.append(v)
if pagination is True:
if page_length < limit:
other_page = False
else:
page = page + 1
else:
other_page = False
# add colums for each type in results
typelist = []
for r in results:
if r['misp_type'] not in typelist:
typelist.append(r['misp_type'])
output_dict = {}
increment = 1
for r in results:
key = str(r['misp_event_id']) + '_' + str(increment)
increment = increment + 1
v = r
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
output_dict[key] = v
for k, v in output_dict.items():
yield v
def stream(self, records):
# Generate args
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
if self.onlyids is True:
to_ids = True
else:
to_ids = False
if self.gettag is True:
get_tag = True
else:
get_tag = False
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict = {"returnFormat": "json"}
body_dict['value'] = str(value)
body_dict['withAttachments'] = "false"
if to_ids:
body_dict['to_ids'] = "True"
body = json.dumps(body_dict)
misp_category = []
misp_event_id = []
misp_to_ids = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
logging.debug('mispsearch request body: %s', body)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
# print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json()
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if get_tag and 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
if get_tag:
record['misp_tag'] = misp_tag
yield record
