def test_match(self):
validator = validators.Match('social security number', r'\d{3}-\d{2}-\d{4}')
self.assertEqual(validator.__call__('123-45-6789'), '123-45-6789')
self.assertRaises(ValueError, validator.__call__, 'foo')
self.assertEqual(validator.__call__(None), None)
self.assertEqual(validator.format(None), None)
self.assertEqual(validator.format('123-45-6789'), '123-45-6789')
class mispapireport(ReportingCommand):
""" MISP API wrapper for endpoint /attributes/restSearch.
return format is JSON for the momemnt
##Syntax
use paramater names to set values in the POST request body below.
.. code-block::
| mispapireport misp_instance=<input> page=<int> limit=<int> value=string type=CSVstring category=CSVstring org=string
tags=CSVstring not_tags=CSVstrings date_from=date_string date_to=date_string last=<int>(d|h|m)
eventid=CSVint uuid=CSVuuid_string enforceWarninglist=True|False
to_ids=True|False deleted=True|False includeEventUuid=True|False includeEventTags==True|False
threat_level_id=<int> eventinfo=string
forced parameters:
"returnFormat": "json"
withAttachments: False
not handled parameters:
"publish_timestamp": "optional",
"timestamp": "optional",
"event_timestamp": "optional",
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional"
}
# status for mode=p
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": param,
"type": param, CSV string,
"category": param, CSV string,
"org": param, CSV string,
"tags": param with not_tags,
"from": param,
"to": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": param,
"publish_timestamp": not managed,
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": param,
"includeEventUuid": param,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": param,
"eventinfo": param,
"includeProposals": not managed
}
"""
# Superseede MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as described in local/inputs.conf.''',
require=True)
# mode: p - give parameters one by one / j provide a complete JSON string
# default is mode=p
mode = Option(doc='''
**Syntax:** **mode=***p|j<AUTH_KEY>*
**Description:**mode to build the JSON request.''',
require=False,
validate=validators.Match("mode", r"^(p|j)$"))
# if mode=j a complete JSON request has to be provided
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
# specific formats
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:**publication duration in day(s), hour(s) or minute(s).''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date_from = Option(doc='''
**Syntax:** **date_from=***date_string"*
**Description:**starting date.''',
require=False)
date_to = Option(doc='''
**Syntax:** **date_to=***date_string"*
**Description:**(optional)ending date in searches with date_from. if not set default is now''',
require=False)
threat_level_id = Option(doc='''
**Syntax:** **threat_level_id=***1-4*
**Description:**Threat level.''',
require=False,
validate=validators.Match("threat_level_id",
r"^[1-4]$"))
org = Option(doc='''
**Syntax:** **org=***CSV string*
**Description:**Comma(,)-separated string of org name(s), id(s), uuid(s).''',
require=False)
# CSV numeric list
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9,]+$"))
# strings
value = Option(doc='''
**Syntax:** **value=***string*
**Description:**value.''',
require=False)
eventinfo = Option(doc='''
**Syntax:** **eventinfo=***string*
**Description:**eventinfo string''',
require=False)
# numeric values
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; default 10000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page of result to get.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
# CSV strings
uuid = Option(doc='''
**Syntax:** **uuid=***id1(,id2,...)*
**Description:**list of event UUID(s).''',
require=False)
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''',
require=False)
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''',
require=False)
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''',
require=False)
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude from results. Wildcard is %.''',
require=False)
# Booleans
to_ids = Option(doc='''
**Syntax:** **to_ids=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to search only attributes with the flag "to_ids" set to true.''',
require=False,
validate=validators.Boolean())
enforceWarninglist = Option(doc='''
**Syntax:** **enforceWarninglist=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to apply warning lists to results.''',
require=False,
validate=validators.Boolean())
deleted = Option(doc='''
**Syntax:** **deleted=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include deleted attributes to results.''',
require=False,
validate=validators.Boolean())
includeEventUuid = Option(doc='''
**Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
includeEventTags = Option(doc='''
**Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes into 2 attributes.''',
require=False,
validate=validators.Boolean())
@Configuration()
def map(self, records):
# self.logger.debug('mispgetioc.map')
return records
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
jsonmode = False
if self.mode is not None:
if 'j' in self.mode and self.json_request is not None:
jsonmode = True
if jsonmode is True:
pagination = True
other_page = True
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
else:
limit = 10000
if 'page' in body_dict:
page = body_dict['page']
else:
page = 1
page_length = 0
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
# add provided parameters to JSON request body
# specific formats
if self.last is not None:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', body_dict['last'])
if self.date_from is not None:
body_dict['from'] = self.date_from
logging.info('Option "date_from" set with %s',
body_dict['from'])
if self.date_to is not None:
body_dict['to'] = self.date_to
logging.info('Option "date_to" set with %s',
body_dict['to'])
else:
logging.info('Option "date_to" will be set to now().')
if self.threat_level_id is not None:
body_dict['threat_level_id'] = self.threat_level_id
logging.info('Option "threat_level_id" set with %s',
body_dict['threat_level_id'])
if self.org is not None:
body_dict['org'] = self.org
logging.info('Option "org" set')
if self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set')
if self.value is not None:
body_dict['value'] = self.value
logging.info('Option "value" set')
if self.eventinfo is not None:
body_dict['eventinfo'] = self.eventinfo
logging.info('Option "eventinfo" set')
# CSV strings
if self.category is not None:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
if self.type is not None:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if self.uuid is not None:
uuid_criteria = {}
uuid_list = self.uuid.split(",")
uuid_criteria['OR'] = uuid_list
body_dict['uuid'] = uuid_criteria
# Booleans
if self.to_ids is not None:
body_dict['to_ids'] = self.to_ids
logging.info('Option "to_ids" set with %s',
body_dict['to_ids'])
if self.enforceWarninglist is not None:
body_dict['enforceWarninglist'] = self.enforceWarninglist
logging.info('Option "enforceWarninglist" set with %s',
body_dict['enforceWarninglist'])
if self.deleted is not None:
body_dict['deleted'] = self.deleted
logging.info('Option "deleted" set with %s',
body_dict['deleted'])
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
logging.info('Option "includeEventUuid" set with %s',
body_dict['includeEventUuid'])
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
logging.info('Option "includeEventTags" set with %s',
body_dict['includeEventTags'])
# Search pagination
pagination = True
other_page = True
if self.page:
page = self.page
else:
page = 1
page_length = 0
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 10000
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
results = []
# add colums for each type in results
while other_page:
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispapireport request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
page_length = len(response['response']['Attribute'])
for a in response['response']['Attribute']:
v = {}
v['misp_Object'] = "-"
if self.includeEventTags is True:
v['misp_tag'] = "-"
for ak, av in a.items():
if ak == 'Event':
json_event = a['Event']
for ek, ev in json_event.items():
key = 'misp_event_' + ek
v[key] = str(ev)
elif ak == 'Tag':
tag_list = []
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
else:
vkey = 'misp_' + ak
v[vkey] = av
results.append(v)
if pagination is True:
if page_length < limit:
other_page = False
else:
page = page + 1
else:
other_page = False
# add colums for each type in results
typelist = []
for r in results:
if r['misp_type'] not in typelist:
typelist.append(r['misp_type'])
output_dict = {}
increment = 1
for r in results:
key = str(r['misp_event_id']) + '_' + str(increment)
increment = increment + 1
v = r
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
output_dict[key] = v
for k, v in output_dict.items():
yield v
class MispCollectCommand(GeneratingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetioc misp_instance=<input> last=<int>(d|h|m)
| mispgetioc misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional",
"includeDecayScore": "optional",
"includeFullModel": "optional",
"decayingModel": "optional",
"excludeDecayed": "optional",
"score": "optional"
}
# status
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": forced to False,
"includeEventUuid": set to True,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": not managed,
"eventinfo": not managed,
"includeProposals": not managed
"includeDecayScore": not managed,
"includeFullModel": not managed,
"decayingModel": not managed,
"excludeDecayed": not managed,
"score": not managed
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(
doc='''
**Syntax:** **misp_instance=instance_name*
**Description:** MISP instance parameters
as described in local/misp42splunk_instances.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(
doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(
doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(
doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(
doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date.
**eventid**, **last** and **date** are mutually exclusive''',
require=False)
# Other params
category = Option(
doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
endpoint = Option(
doc='''
**Syntax:** **endpoint=***<events|attributes>*
**Description:**selection of MISP API restSearch endpoint.
**default**: /attributes/restSearch''',
require=False, validate=validators.Match("endpoint", r"(events|attributes)"))
geteventtag = Option(
doc='''
**Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean includeEventTags. By default only
attribute tag(s) are returned.''',
require=False, validate=validators.Boolean())
keep_related = Option(
doc='''
**Syntax:** **keep_related=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to keep related events.
default is to drop RelatedEvents to reduce volume.''',
require=False, validate=validators.Boolean())
limit = Option(
doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search;
default 1000. 0 = no pagination.''',
require=False, validate=validators.Match("limit", r"^[0-9]+$"))
not_tags = Option(
doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
page = Option(
doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False, validate=validators.Match("page", r"^[0-9]+$"))
tags = Option(
doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
to_ids = Option(
doc='''
**Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to search only attributes with the flag
"to_ids" set to true.''',
require=False, validate=validators.Boolean())
type = Option(
doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
warning_list = Option(
doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False, validate=validators.Boolean())
@staticmethod
def _record(serial_number, time_stamp, host, attributes, attribute_names, encoder):
raw = encoder.encode(attributes)
# Formulate record
fields = dict()
for f in attribute_names:
if f in attributes:
fields[f] = attributes[f]
if serial_number > 0:
fields['_serial'] = serial_number
fields['_time'] = time_stamp
fields['_raw'] = raw
fields['host'] = host
return fields
record = OrderedDict(chain(
(('_serial', serial_number), ('_time', time_stamp),
('_raw', raw), ('host', host)),
map(lambda name: (name, fields.get(name, '')), attribute_names)))
return record
def generate(self):
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception("Sorry, no configuration for misp_instance={}".format(misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
raise Exception('Missing "json_request", "eventid", "last" or "date" argument')
elif mandatory_arg > 1:
raise Exception('Options "json_request", "eventid", "last" and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
# manage to_ids and enforceWarninglist
# to avoid FP enforceWarninglist is set to True if
# to_ids is set to True (search criterion)
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.endpoint == 'events':
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
else:
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.keep_related is True:
keep_related = True
else:
keep_related = False
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispgetioc request body: %s', body)
# search
r = requests.post(my_args['misp_url'], headers=headers, data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
logging.info(
"[CO301] INFO mispcollect successful. "
"url={}, HTTP status={}".format(my_args['misp_url'], r.status_code)
)
else:
logging.error(
"[CO302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code, r.text)
)
raise Exception(
"[CO302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code, r.text)
)
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if self.endpoint == "events":
if 'response' in response:
for r_item in response['response']:
if 'Event' in r_item:
attribute_names = []
serial_number = 0
for e in list(r_item.values()):
if keep_related is False:
e.pop('RelatedEvent', None)
if serial_number == 0:
for k in list(e.keys()):
attribute_names.append(k)
yield MispCollectCommand._record(
serial_number, e['timestamp'], my_args['host'],
e, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
if 'response' in response:
if 'Attribute' in response['response']:
attribute_names = []
serial_number = 0
for a in response['response']['Attribute']:
if serial_number == 0:
for k in list(a.keys()):
attribute_names.append(k)
yield MispCollectCommand._record(
serial_number, a['timestamp'], my_args['host'],
a, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
class mispgetioc(ReportingCommand):
mispsrv = Option(require=False,
validate=validators.Match(
"mispsrv", r"^https?:\/\/[0-9a-zA-Z\.]+(?:\:\d+)?$"))
mispkey = Option(require=False,
validate=validators.Match("mispkey",
r"^[0-9a-zA-Z]{40}$"))
sslcheck = Option(require=False,
validate=validators.Match("sslcheck", r"^[yYnN01]$"))
eventid = Option(require=False,
validate=validators.Match("eventid", r"^[0-9]+$"))
last = Option(require=False,
validate=validators.Match("last", r"^[0-9]+[hdwm]$"))
onlyids = Option(require=False,
validate=validators.Match("onlyids", r"^[yYnN01]+$"))
getuuid = Option(require=False,
validate=validators.Match("getuuid", r"^[yYnN01]+$"))
getorg = Option(require=False,
validate=validators.Match("getorg", r"^[yYnN01]+$"))
category = Option(require=False)
type = Option(require=False)
tags = Option(require=False)
not_tags = Option(require=False)
@Configuration()
def map(self, records):
self.logger.debug('mispgetioc.map')
yield {}
return
def reduce(self, records):
self.logger.debug('mispgetioc.reduce')
if self.sslcheck == None:
self.sslcheck = 'n'
_SPLUNK_PATH = os.environ['SPLUNK_HOME']
# open misp.conf
config_file = _SPLUNK_PATH + '/etc/apps/misp42splunk/local/misp.conf'
mispconf = ConfigParser.RawConfigParser()
mispconf.read(config_file)
# Generate args
my_args = {}
#MISP instance parameters
if self.mispsrv:
my_args['mispsrv'] = self.mispsrv
else:
my_args['mispsrv'] = mispconf.get('mispsetup', 'mispsrv')
if self.mispkey:
my_args['mispkey'] = self.mispkey
else:
my_args['mispkey'] = mispconf.get('mispsetup', 'mispkey')
if self.sslcheck:
if self.sslcheck == 'Y' or self.sslcheck == 'y' or self.sslcheck == '1':
my_args['sslcheck'] = True
else:
my_args['sslcheck'] = False
else:
my_args['sslcheck'] = mispconf.getboolean('mispsetup', 'sslcheck')
#Search parameters: boolean and filter
if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1':
my_args['onlyids'] = True
else:
my_args['onlyids'] = False
if self.getuuid == 'Y' or self.getuuid == 'y' or self.getuuid == '1':
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.getorg == 'Y' or self.getorg == 'y' or self.getorg == '1':
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.category != None:
my_args['category'] = self.category
else:
my_args['category'] = None
if self.type != None:
my_args['type'] = self.type
else:
my_args['type'] = None
if self.tags != None:
my_args['tags'] = self.tags
else:
my_args['tags'] = None
if self.not_tags != None:
my_args['not_tags'] = self.not_tags
else:
my_args['not_tags'] = None
#check that ONE of mandatory fields is present
if self.eventid and self.last:
print('DEBUG Options "eventid" and "last" are mutually exclusive')
exit(2)
elif self.eventid:
my_args['eventid'] = self.eventid
elif self.last:
my_args['last'] = self.last
else:
print('DEBUG Missing "eventid" or "last" argument')
exit(1)
#path to main components either use default values or set ones
if mispconf.has_option('mispsetup', 'P3_PATH'):
_NEW_PYTHON_PATH = mispconf.get('mispsetup', 'P3_PATH')
else:
_NEW_PYTHON_PATH = '/usr/bin/python3'
if mispconf.has_option('mispsetup', 'TMP_PATH'):
_TMP_PATH = mispconf.get('mispsetup', 'TMP_PATH')
else:
_TMP_PATH = '/tmp'
_SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH
my_process = _SPLUNK_PATH + '/etc/apps/misp42splunk/bin/pymisp_getioc.py'
# Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues
env = dict(os.environ)
del env['LD_LIBRARY_PATH']
FNULL = open(os.devnull, 'w')
#use pickle
swap_file = _TMP_PATH + '/mispgetioc_config'
pickle.dump(my_args, open(swap_file, "wb"), protocol=2)
env_file = _TMP_PATH + '/mispgetioc_env'
pickle.dump(env, open(env_file, "wb"), protocol=2)
p = subprocess.Popen([_NEW_PYTHON_PATH, my_process, swap_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
stdout, stderr = p.communicate()
# if stderr:
# print('DEBUG error in pymisp_getioc.py')
# exit(1)
output = {}
output = pickle.load(open(swap_file, "rb"))
if output:
for v in output:
yield v
class ipinfo(GeneratingCommand):
url = Option(require=False,
validate=validators.Match('https url', '^https:\/\/'))
headers = Option(require=False)
method = Option(require=False, default='POST')
ip = Option(require=True)
def generate(self):
url = "https://ipinfo.io/batch?token="
headers = self.parseHeaders("{'Content-type': 'application/json'}")
method = "post"
data = self.parseData(self.ip)
record = {}
storage_passwords = self.service.storage_passwords
local_conf = splunk_lib_util.make_splunkhome_path(
["etc", "apps", "ipinfo_app", "local", "ip_info_setup.conf"])
default_conf = splunk_lib_util.make_splunkhome_path(
["etc", "apps", "ipinfo_app", "default", "ip_info_setup.conf"])
config = ConfigParser()
config.read([default_conf, local_conf])
url = config.get("ip_info_configuration", "api_url")
token = config.get("ip_info_configuration", "api_token")
enable = config.get("ip_info_configuration", "proxy_enable")
proxy_url = config.get("ip_info_configuration", "proxy_url")
disable_ssl = config.get("ip_info_configuration", "disable_ssl")
cert_path = splunk_lib_util.make_splunkhome_path([
"etc", "apps", "ipinfo_app", "appserver", "static", "ipinfo.cert"
])
if (os.path.exists(cert_path)):
cert_exists = True
else:
cert_exists = False
if (disable_ssl != ""):
disable_ssl_request = False
else:
disable_ssl_request = True
if (disable_ssl_request == True and cert_exists == True):
disable_ssl_request = cert_path
response = ""
param = {"token": token}
try:
if enable == "No":
response = requests.request("post",
url + "batch?token=" + token,
headers=headers,
verify=disable_ssl_request,
data=data)
else:
proxies = {'https': proxy_url}
response = requests.request("post",
url + "batch?token=" + token,
headers=headers,
verify=disable_ssl_request,
data=data,
proxies=proxies)
except Exception as e:
logger.info(e)
#url = url+token
#response = requests.request(method, url, headers=headers , data=data )
records = response.json()
for key, value in records.items():
record = value
yield record
def parseHeaders(self, headers):
# Replace single quotes with double quotes for valid json
return json.loads(headers.replace('\'', '"'))
def parseData(self, data):
data = "[\"" + data + "\"]"
return (data.replace(',', '","'))
class MispSearchCommand(StreamingCommand):
""" search in MISP for attributes matching the value of field.
##Syntax
code-block::
mispsearch field=<field> onlyids=y|n
##Description
body = {
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional"
}
##Example
Search in MISP for value of fieldname r_ip (remote IP in proxy logs).
code-block::
* | mispsearch field=r_ip
"""
field = Option(doc='''
**Syntax:** **field=***<fieldname>*
**Description:**Name of the field containing the value to search for.''',
require=True,
validate=validators.Fieldname())
onlyids = Option(doc='''
**Syntax:** **onlyids=***<y|n>*
**Description:** Boolean to search only attributes with to_ids set''',
require=False,
validate=validators.Boolean())
gettag = Option(doc='''
**Syntax:** **gettag=***<y|n>*
**Description:** Boolean to return attribute tags''',
require=False,
validate=validators.Boolean())
# Superseede MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as decibed in lookup/misp_instances.csv.''',
require=False)
misp_url = Option(doc='''
**Syntax:** **misp_url=***<MISP URL>*
**Description:**URL of MISP instance.''',
require=False,
validate=validators.Match(
"misp_url",
r"^https?:\/\/[0-9a-zA-Z\-\.]+(?:\:\d+)?$"))
misp_key = Option(doc='''
**Syntax:** **misp_key=***<AUTH_KEY>*
**Description:**MISP API AUTH KEY.''',
require=False,
validate=validators.Match("misp_key",
r"^[0-9a-zA-Z]{40}$"))
misp_verifycert = Option(doc='''
**Syntax:** **misp_verifycert=***<y|n>*
**Description:**Verify or not MISP certificate.''',
require=False,
validate=validators.Boolean())
def stream(self, records):
# Generate args
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
if self.onlyids is True:
to_ids = True
else:
to_ids = False
if self.gettag is True:
get_tag = True
else:
get_tag = False
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict = {"returnFormat": "json"}
body_dict['value'] = str(value)
body_dict['withAttachments'] = "false"
if to_ids:
body_dict['to_ids'] = "True"
body = json.dumps(body_dict)
misp_category = []
misp_event_id = []
misp_to_ids = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
logging.info('INFO MISP REST API REQUEST: %s', body)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
# print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json()
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if get_tag and 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
if get_tag:
record['misp_tag'] = misp_tag
yield record
class mispgetevent(ReportingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetevent misp_instance=<input> last=<int>(d|h|m)
| mispgetevent misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetevent misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tag": "optional",
"tags": "optional",
"searchall": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"metadata": "optional",
"uuid": "optional",
"published": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"sgReferenceOnly": "optional",
"eventinfo": "optional",
"excludeLocalTags": "optional"
}
# status
"tag": "optional",
"searchall": "optional",
"metadata": "optional",
"published": "optional",
"sgReferenceOnly": "optional",
"eventinfo": "optional",
"excludeLocalTags": "optional"
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": not managed,
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as described
in local/inputs.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date. **eventid**, **last**
and **date** are mutually exclusive''',
require=False)
# Other params
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; default 1000.
0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
published = Option(doc='''
**Syntax:** **published=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**select only published events (for option from to) .''',
require=False,
validate=validators.Boolean())
getioc = Option(doc='''
**Syntax:** **getioc=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return the list of attributes
together with the event.''',
require=False,
validate=validators.Boolean())
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes.''',
require=False,
validate=validators.Boolean())
@Configuration()
def map(self, records):
# self.logger.debug('mispevent.map')
return records
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getioc is True:
my_args['getioc'] = True
else:
my_args['getioc'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
results = []
# add colums for each type in results
typelist = []
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.error('mispgetevent request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
for r_item in response['response']:
if 'Event' in r_item:
for a in list(r_item.values()):
v = {}
v['misp_event_id'] = str(a['id'])
v['misp_orgc_id'] = str(a['orgc_id'])
v['misp_event_date'] = str(a['date'])
v['threat_level_id'] = str(a['threat_level_id'])
v['misp_event_info'] = a['info']
v['misp_event_published'] = str(a['published'])
v['misp_event_uuid'] = str(a['uuid'])
v['misp_attribute_count'] = str(a['attribute_count'])
v['misp_analysis'] = str(a['analysis'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_distribution'] = str(a['distribution'])
v['misp_publish_timestamp'] = \
str(a['publish_timestamp'])
v['misp_sharing_group_id'] = str(a['sharing_group_id'])
v['misp_extends_uuid'] = str(a['extends_uuid'])
if 'Orgc' in a:
v['misp_orgc_name'] = str(a['Orgc']['name'])
v['misp_orgc_uuid'] = str(a['Orgc']['uuid'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
if my_args['getioc'] is True:
v['Attribute'] = list()
v['misp_attribute_count'] = 0
if 'Attribute' in a:
v['misp_attribute_count'] = \
v['misp_attribute_count'] + len(a['Attribute'])
if my_args['getioc'] is True:
for attribute in a['Attribute']:
# combined: not part of an object AND
# multivalue attribute AND to be split
if int(attribute['object_id']) == 0 \
and '|' in attribute['type'] \
and my_args['pipe'] is True:
mv_type_list = \
attribute['type'].split('|')
mv_value_list = \
str(attribute['value']).split('|')
left_a = attribute.copy()
left_a['type'] = mv_type_list.pop()
left_a['value'] = mv_value_list.pop()
v['Attribute'].append(
getioc(left_a, typelist,
my_args['pipe'],
left_a['object_id']))
right_a = attribute.copy()
right_a['type'] = mv_type_list.pop()
right_a['value'] = mv_value_list.pop()
v['Attribute'].append(
getioc(right_a, typelist,
my_args['pipe'],
right_a['object_id']))
else:
v['Attribute'].append(
getioc(attribute, typelist,
my_args['pipe'],
attribute['object_id']))
if 'Object' in a:
for misp_o in a['Object']:
if 'Attribute' in misp_o:
v['misp_attribute_count'] = \
v['misp_attribute_count'] \
+ len(misp_o['Attribute'])
if my_args['getioc'] is True:
object_id = misp_o['id']
object_name = misp_o['name']
object_comment = misp_o['comment']
for attribute in misp_o['Attribute']:
v['Attribute'].append(
getioc(attribute, typelist,
my_args['pipe'],
object_id, object_name,
object_comment))
logging.debug('event is %s', json.dumps(v))
results.append(v)
logging.info('typelist is %s', json.dumps(typelist))
# relevant_cat = ['Artifacts dropped', 'Financial fraud',
# 'Network activity','Payload delivery','Payload installation']
logging.debug('results is %s', json.dumps(results))
if my_args['getioc'] is False:
for e in results:
yield e
else:
output_dict = {}
for e in results:
if 'Attribute' in e:
for r in e['Attribute']:
if int(r['misp_object_id']) == 0: # not an object
key = str(e['misp_event_id']) + '_' \
+ r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(e['misp_event_id']) + \
'_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = init_misp_output(e, r)
for t in typelist:
misp_t = 'misp_' \
+ t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
to_ids = []
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = []
category.append(r['misp_category'])
v['misp_category'] = category
attribute_uuid = []
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is True:
v['misp_type'] = v['misp_object_name']
v['misp_value'] = v['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace('-', '_')
v[misp_t] = r['misp_value'] # set value for type
to_ids = v['misp_to_ids']
if r['misp_to_ids'] not in to_ids:
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = v['misp_category']
# append
if r['misp_category'] not in category:
category.append(r['misp_category'])
v['misp_category'] = category
attribute_uuid = v['misp_attribute_uuid']
if r['misp_attribute_uuid'] not in attribute_uuid:
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is False:
misp_type = r['misp_type'] \
+ '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + \
'|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
for k, v in list(output_dict.items()):
yield v
class GenerateTextCommand(GeneratingCommand):
account = Option(
doc='''
**Syntax:** **account=****
**Description:** JIRA account to be used, if unspecified the first account configured will be taken into account.''',
require=False, default=None)
method = Option(
doc='''
**Syntax:** **method=****
**Description:** method to use for API target. DELETE GET POST PUT are supported.''',
require=False, validate=validators.Match("method", r"^(DELETE|GET|POST|PUT)$"))
json_request = Option(
doc='''
**Syntax:** **json_request=***JSON request*
**Description:** JSON-formatted json_request.''',
require=False, validate=validators.Match("json_request", r"^{.+}$"))
target = Option(require=True)
def generate(self):
storage_passwords = self.service.storage_passwords
# global configuration
conf_file = "ta_service_desk_simple_addon_settings"
confs = self.service.confs[str(conf_file)]
jira_passthrough_mode = None
proxy_enabled = "0"
proxy_url = None
proxy_dict = None
proxy_username = None
for stanza in confs:
if stanza.name == "advanced_configuration":
for key, value in stanza.content.items():
if key == "jira_passthrough_mode":
jira_passthrough_mode = value
if stanza.name == "proxy":
for key, value in stanza.content.items():
if key == "proxy_enabled":
proxy_enabled = value
if key == "proxy_port":
proxy_port = value
if key == "proxy_rdns":
proxy_rdns = value
if key == "proxy_type":
proxy_type = value
if key == "proxy_url":
proxy_url = value
if key == "proxy_username":
proxy_username = value
if proxy_enabled == "1":
# get proxy password
if proxy_username:
proxy_password = None
# get proxy password, if any
credential_realm = '__REST_CREDENTIAL__#TA-jira-service-desk-simple-addon#configs/conf-ta_service_desk_simple_addon_settings'
for credential in storage_passwords:
if credential.content.get('realm') == str(credential_realm) \
and credential.content.get('clear_password').find('proxy_password') > 0:
proxy_password = json.loads(credential.content.get('clear_password')).get('proxy_password')
break
if proxy_type == 'http':
proxy_dict= {
"http" : "http://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port,
"https" : "https://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port
}
else:
proxy_dict= {
"http" : str(proxy_type) + "://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port,
"https" : str(proxy_type) + "://" + proxy_username + ":" + proxy_password + "@" + proxy_url + ":" + proxy_port
}
else:
proxy_dict= {
"http" : proxy_url + ":" + proxy_port,
"https" : proxy_url + ":" + proxy_port
}
# get all acounts
accounts = []
conf_file = "ta_service_desk_simple_addon_account"
confs = self.service.confs[str(conf_file)]
for stanza in confs:
# get all accounts
for name in stanza.name:
accounts.append(stanza.name)
break
# define the account target
if not self.account or self.account == '_any':
account = str(accounts[0])
else:
account = str(self.account)
# account configuration
isfound = False
jira_ssl_certificate_validation = None
jira_ssl_certificate_path = None
username = None
password = None
conf_file = "ta_service_desk_simple_addon_account"
confs = self.service.confs[str(conf_file)]
for stanza in confs:
if stanza.name == str(account):
isfound = True
for key, value in stanza.content.items():
if key == "jira_url":
jira_url = value
if key == "jira_ssl_certificate_validation":
jira_ssl_certificate_validation = value
if key == "jira_ssl_certificate_path":
jira_ssl_certificate_path = value
if key == 'auth_type':
auth_type = value
if key == 'jira_auth_mode':
jira_auth_mode = value
if key == 'username':
username = value
# end of get configuration
# Stop here if we cannot find the submitted account
if not isfound:
self.logger.fatal('This acount has not been configured on this instance, cannot proceed!: %s', self)
# else get the password
else:
credential_username = str(account) + '``splunk_cred_sep``1'
credential_realm = '__REST_CREDENTIAL__#TA-jira-service-desk-simple-addon#configs/conf-ta_service_desk_simple_addon_account'
for credential in storage_passwords:
if credential.content.get('username') == str(credential_username) \
and credential.content.get('realm') == str(credential_realm) \
and credential.content.get('clear_password').find('password') > 0:
password = json.loads(credential.content.get('clear_password')).get('password')
break
# Build the authentication header for JIRA
if str(jira_auth_mode) == 'basic':
authorization = username + ':' + password
b64_auth = base64.b64encode(authorization.encode()).decode()
jira_headers = {
'Authorization': 'Basic %s' % b64_auth,
'Content-Type': 'application/json',
}
elif str(jira_auth_mode) == 'pat':
jira_headers = {
'Authorization': 'Bearer %s' % str(password),
'Content-Type': 'application/json',
}
# verify the url
if not jira_url.startswith("https://"):
jira_url = "https://" + str(jira_url)
# handle SSL verification and bundle
if jira_ssl_certificate_validation:
if jira_ssl_certificate_validation == '0':
ssl_verify = False
elif jira_ssl_certificate_validation == '1' and jira_ssl_certificate_path and os.path.isfile(jira_ssl_certificate_path):
ssl_verify = str(jira_ssl_certificate_path)
elif jira_ssl_certificate_validation == '1':
ssl_verify = True
# verify the method
if self.method:
jira_method = self.method
else:
jira_method = "GET"
if self.json_request:
body_dict = json.loads(self.json_request)
else:
if jira_method == "POST" or jira_method == "PUT":
raise Exception("jirarest: method {} requires a valid json_request. It is empty".format(jira_method))
if self.target:
# set proper headers
if jira_method == "GET":
jira_fields_response = requests.get(
url=str(jira_url) + '/' + str(self.target),
headers=jira_headers,
verify=ssl_verify,
proxies=proxy_dict
)
elif jira_method == "DELETE":
jira_fields_response = requests.delete(
url=str(jira_url) + '/' + str(self.target),
headers=jira_headers,
verify=ssl_verify,
proxies=proxy_dict
)
elif jira_method == "POST":
jira_fields_response = requests.post(
url=str(jira_url) + '/' + str(self.target),
data=json.dumps(body_dict).encode('utf-8'),
headers=jira_headers,
verify=ssl_verify,
proxies=proxy_dict
)
elif jira_method == "PUT":
jira_fields_response = requests.put(
url=str(jira_url) + '/' + str(self.target),
data=json.dumps(body_dict).encode('utf-8'),
headers=jira_headers,
verify=ssl_verify,
proxies=proxy_dict
)
# Attenpt to get a JSON response, and render in Splunk
try:
json_response = jira_fields_response.json()
data = {'_time': time.time(), '_raw': json.dumps(json_response)}
yield data
except Exception as e:
# Build a custom response for Splunk dynamically
# Create an action field, convenient to quickly understanding when things go wrong
if jira_fields_response.status_code in (200, 201, 204):
response_action = "success"
else:
response_action = "failure"
# render
if jira_fields_response.text:
json_response = "{\"action\": \"" + str(response_action) + "\", \"status_code\": \"" + str(jira_fields_response.status_code) + "\", \"text\": \"" + str(jira_fields_response.text) + "\"}"
else:
json_response = "{\"action\": \"" + str(response_action) + "\", \"status_code\": \"" + str(jira_fields_response.status_code) + "\"}"
data = {'_time': time.time(), '_raw': str(json.dumps(json.loads(json_response, strict=False), indent=4))}
yield data
class mispgetioc(ReportingCommand):
mispsrv = Option(require=False,
validate=validators.Match(
"mispsrv", r"^https?:\/\/[0-9a-zA-Z\.]+(?:\:\d+)?$"))
mispkey = Option(require=False,
validate=validators.Match("mispkey",
r"^[0-9a-zA-Z]{40}$"))
sslcheck = Option(require=False,
validate=validators.Match("sslcheck", r"^[yYnN01]$"))
eventid = Option(require=False,
validate=validators.Match("eventid", r"^[0-9]+$"))
last = Option(require=False,
validate=validators.Match("last", r"^[0-9]+[hdwm]$"))
onlyids = Option(require=False,
validate=validators.Match("onlyids", r"^[yYnN01]+$"))
getuuid = Option(require=False,
validate=validators.Match("getuuid", r"^[yYnN01]+$"))
getorg = Option(require=False,
validate=validators.Match("getuuid", r"^[yYnN01]+$"))
category = Option(require=False)
type = Option(require=False)
@Configuration()
def map(self, records):
self.logger.debug('mispgetioc.map')
yield {}
return
def reduce(self, records):
self.logger.debug('mispgetioc.reduce')
if self.sslcheck == None:
self.sslcheck = 'n'
# open misp.conf
config_file = '/opt/splunk/etc/apps/misp42splunk/local/misp.conf'
config = ConfigParser.RawConfigParser()
config.read(config_file)
# Generate args
my_args = {}
if self.mispsrv:
my_args['mispsrv'] = self.mispsrv
else:
my_args['mispsrv'] = config.get('mispsetup', 'mispsrv')
if self.mispkey:
my_args['mispkey'] = self.mispkey
else:
my_args['mispkey'] = config.get('mispsetup', 'mispkey')
if self.sslcheck:
if self.sslcheck == 'Y' or self.sslcheck == 'y' or self.sslcheck == '1':
my_args['sslcheck'] = True
else:
my_args['sslcheck'] = False
else:
my_args['sslcheck'] = config.getboolean('mispsetup', 'sslcheck')
if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1':
onlyids = True
else:
onlyids = False
if self.getuuid == 'Y' or self.getuuid == 'y' or self.getuuid == '1':
getuuid = True
else:
getuuid = False
if self.getorg == 'Y' or self.getorg == 'y' or self.getorg == '1':
getorg = True
else:
getorg = False
if self.eventid and self.last:
print('DEBUG Options "eventid" and "last" are mutually exclusive')
exit(2)
if self.eventid:
my_args['eventid'] = self.eventid
elif self.last:
my_args['last'] = self.last
else:
print('DEBUG Missing "eventid" or "last" argument')
exit(1)
_SPLUNK_PATH = '/opt/splunk'
_NEW_PYTHON_PATH = '/usr/bin/python3'
_SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH
my_process = _SPLUNK_PATH + '/etc/apps/misp42splunk/bin/pymisp_getioc.py'
# Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues
env = dict(os.environ)
del env['LD_LIBRARY_PATH']
FNULL = open(os.devnull, 'w')
p = subprocess.Popen([_NEW_PYTHON_PATH, my_process,
str(my_args)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=FNULL,
env=env)
output = p.communicate()[0]
results = {}
for v in eval(output):
# Do not display deleted attributes
if v['deleted'] == False:
# If specified, do not display attributes with the non-ids flag set to False
if onlyids == True and v['to_ids'] == False:
continue
if self.category != None and self.category != v['category']:
continue
if self.type != None and self.type != v['type']:
continue
if getuuid == True:
results['uuid'] = v['uuid']
if getorg == True:
results['orgc'] = v['orgc']
results['eventid'] = v['event_id']
results['value'] = v['value']
results['category'] = v['category']
results['type'] = v['type']
results['to_ids'] = str(v['to_ids'])
yield results
class MispSearchCommand(StreamingCommand):
"""
search in MISP for attributes matching the value of field.
##Syntax
code-block::
mispsearch field=<field> to_ids=y|n
##Description
body = {
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional"
}
##Example
Search in MISP for value of fieldname r_ip (remote IP in proxy logs).
code-block::
* | mispsearch field=r_ip
"""
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as \
described in local/misp42splunk_instances.conf''',
require=True)
field = Option(doc='''
**Syntax:** **field=***<fieldname>*
**Description:**Name of the field containing \
the value to search for.''',
require=True,
validate=validators.Fieldname())
to_ids = Option(doc='''
**Syntax:** **to_ids=***<y|n>*
**Description:** Boolean to search only attributes with to_ids set''',
require=False,
validate=validators.Boolean())
includeEventUuid = Option(doc='''
**Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
includeEventTags = Option(doc='''
**Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include Event Tags to results.''',
require=False,
validate=validators.Boolean())
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:**Publication duration in day(s), hour(s) or minute(s)
to limit search scope only to published events in last X timerange.''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; \
default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("page", r"^[0-9]+$"))
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
def log_error(self, msg):
logging.error(msg)
def log_info(self, msg):
logging.info(msg)
def log_debug(self, msg):
logging.debug(msg)
def log_warn(self, msg):
logging.warning(msg)
def set_log_level(self):
logging.root
loglevel = logging_level('misp42splunk')
logging.root.setLevel(loglevel)
logging.error('[SE-101] logging level is set to %s', loglevel)
logging.error('[SE-102] PYTHON VERSION: ' + sys.version)
def stream(self, records):
# loggging
self.set_log_level()
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
pagination = True
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 1000
if self.page is not None:
page = int(self.page)
else:
page = 1
if self.json_request is not None:
body_dict = json.loads(self.json_request)
self.log_info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
if 'page' in body_dict:
page = body_dict['page']
pagination = False
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
if self.to_ids is True:
body_dict['to_ids'] = "True"
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
if self.last is not None:
body_dict['last'] = self.last
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict['value'] = str(value)
misp_category = []
misp_event_id = []
misp_event_uuid = []
misp_orgc_id = []
misp_to_ids = []
misp_comment = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
self.log_info(
"[SE301] INFO mispsearch successful. url={}, HTTP status={}"
.format(my_args['misp_url'], r.status_code))
else:
self.log_error(
"[SE302] ERROR mispsearch failed. url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code,
r.text))
raise Exception(
"[SE302] ERROR mispsearch failed for url={} with HTTP Error={}. Check search.log for details"
.format(my_args['misp_url'], r.status_code))
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['comment']) not in misp_comment:
misp_comment.append(str(a['comment']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
if 'Event' in a:
if a['Event']['uuid'] \
not in misp_event_uuid:
misp_event_uuid.append(
str(a['Event']['uuid']))
if a['Event']['orgc_id'] \
not in misp_orgc_id:
misp_orgc_id.append(
str(a['Event']['orgc_id']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_comment'] = misp_comment
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
record['misp_event_uuid'] = misp_event_uuid
record['misp_orgc_id'] = misp_orgc_id
record['misp_tag'] = misp_tag
yield record
class mispgetioc(ReportingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetioc misp_instance=<input> last=<int>(d|h|m)
| mispgetioc misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc misp_instance=<input> date_from=<<YYYY-MM-DD> (date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional"
}
# status
"returnFormat": forced to json,
"page": not managed,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param,
"from": param,
"to": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": not managed,
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": forced to False,
"includeEventUuid": param,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": param,
"eventinfo": not managed,
"includeProposals": not managed
}
"""
# Superseede MISP instance for this search
misp_instance = Option(
doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as described in local/inputs.conf.''',
require=True)
# MANDATORY: eventid XOR last XOR date_from
eventid = Option(
doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s). **eventid**, **last** and **date_from** are mutually exclusive''',
require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(
doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:**publication duration in day(s), hour(s) or minute(s). **eventid**, **last** and **date_from** are mutually exclusive''',
require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date_from = Option(
doc='''
**Syntax:** **date_from=***date_string"*
**Description:**starting date. **eventid**, **last** and **date_from** are mutually exclusive''',
require=False)
date_to = Option(
doc='''
**Syntax:** **date_to=***date_string"*
**Description:**(optional)ending date in searches with date_from. if not set default is now''',
require=False)
to_ids = Option(
doc='''
**Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to search only attributes with the flag "to_ids" set to true.''',
require=False, validate=validators.Boolean())
published = Option(
doc='''
**Syntax:** **published=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**select only published events (for option from to) .''',
require=False, validate=validators.Boolean())
category = Option(
doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''',
require=False)
type = Option(
doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''',
require=False)
tags = Option(
doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''',
require=False)
not_tags = Option(
doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude from results. Wildcard is %.''',
require=False)
threat_level_id = Option(
doc='''
**Syntax:** **threat_level=***<int>*
**Description:**define the threat_level_id''',
require=False, validate=validators.Match("threat_level_id", r"^[1-4]$"))
limit = Option(
doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; default 10000. 0 = no pagination.''',
require=False, validate=validators.Match("limit", r"^[0-9]+$"))
getuuid = Option(
doc='''
**Syntax:** **getuuid=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return attribute UUID.''',
require=False, validate=validators.Boolean())
getorg = Option(
doc='''
**Syntax:** **getorg=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return the ID of the organisation that created the event.''',
require=False, validate=validators.Boolean())
geteventtag = Option(
doc='''
**Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return also event tag(s). By default only attribute tag(s) are returned.''',
require=False, validate=validators.Boolean())
pipesplit = Option(
doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes into 2 attributes.''',
require=False, validate=validators.Boolean())
add_description = Option(
doc='''
**Syntax:** **add_description=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return misp_description.''',
require=False, validate=validators.Boolean())
warning_list = Option(
doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False, validate=validators.Boolean())
@Configuration()
def map(self, records):
# self.logger.debug('mispgetioc.map')
return records
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# build search JSON object
body_dict = {"returnFormat": "json",
"withAttachments": False,
"deleted": False
}
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date_from:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "eventid", "last" or "date_from" argument')
raise Exception('Missing "eventid", "last" or "date_from" argument')
elif mandatory_arg > 1:
logging.error('Options "eventid", "last" and "date_from" are mutually exclusive')
raise Exception('Options "eventid", "last" and "date_from" are mutually exclusive')
# Only ONE combination was provided
if self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set')
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', body_dict['last'])
else:
body_dict['from'] = self.date_from
logging.info('Option "date_from" set with %s', body_dict['from'])
if self.date_to:
body_dict['to'] = self.date_to
logging.info('Option "date_to" set with %s', body_dict['to'])
else:
logging.info('Option "date_to" will be set to now().')
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
other_page = True
page = 1
page_length = 0
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 10000
# Search parameters: boolean and filter
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.category is not None:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
if self.type is not None:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if self.threat_level_id is not None:
body_dict['threat_level_id'] = int(self.threat_level_id)
# output filter parameters
if self.getuuid is True:
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.getorg is True:
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
if self.add_description is True:
my_args['add_desc'] = True
else:
my_args['add_desc'] = False
results = []
# add colums for each type in results
typelist = []
while other_page:
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispgetioc request body: %s', body)
# search
r = requests.post(my_args['misp_url'], headers=headers, data=body, verify=my_args['misp_verifycert'], cert=my_args['client_cert_full_path'], proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
page_lenth = len(response['response']['Attribute'])
for a in response['response']['Attribute']:
v = {}
v['misp_category'] = str(a['category'])
v['misp_attribute_id'] = str(a['id'])
v['misp_event_id'] = str(a['event_id'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_to_ids'] = str(a['to_ids'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
# include ID of the organisation that created the attribute if requested
# in previous version this was the ORG name ==> create lookup
if 'Event' in a and my_args['getorg']:
v['misp_orgc_id'] = str(a['Event']['orgc_id'])
# include attribute UUID if requested
if my_args['getuuid']:
v['misp_attribute_uuid'] = str(a['uuid'])
# handle object and multivalue attributes
v['misp_object_id'] = str(a['object_id'])
if my_args['add_desc'] is True:
if int(a['object_id']) == 0:
v['misp_description'] = 'MISP e' + str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' + str(a['type']) \
+ '" in category "' + str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) + ')'
else:
v['misp_description'] = 'MISP e' + str(a['event_id']) \
+ ' attribute ' + str(a['uuid']) + ' of type "' \
+ str(a['type']) + '" in category "' \
+ str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) \
+ ' - o' + str(a['object_id']) + ' )'
current_type = str(a['type'])
# combined: not part of an object AND multivalue attribute QND to be split
if int(a['object_id']) == 0 and '|' in current_type and my_args['pipe'] is True:
mv_type_list = current_type.split('|')
mv_value_list = str(a['value']).split('|')
left_v = v.copy()
left_v['misp_type'] = mv_type_list.pop()
left_v['misp_value'] = mv_value_list.pop()
results.append(left_v)
if left_v['misp_type'] not in typelist:
typelist.append(left_v['misp_type'])
right_v = v.copy()
right_v['misp_type'] = mv_type_list.pop()
right_v['misp_value'] = mv_value_list.pop()
results.append(right_v)
if right_v['misp_type'] not in typelist:
typelist.append(right_v['misp_type'])
else:
v['misp_type'] = current_type
v['misp_value'] = str(a['value'])
results.append(v)
if current_type not in typelist:
typelist.append(current_type)
if pagination is True:
if page_length < limit:
other_page = False
else:
page = page + 1
else:
other_page = False
logging.info(json.dumps(typelist))
output_dict = {}
# relevant_cat = ['Artifacts dropped', 'Financial fraud', 'Network activity','Payload delivery','Payload installation']
for r in results:
if int(r['misp_object_id']) == 0: # not an object
key = str(r['misp_event_id']) + '_' + r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(r['misp_event_id']) + '_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
to_ids = []
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = []
category.append(r['misp_category'])
v['misp_category'] = category
if my_args['add_desc'] is True:
description = []
description.append(r['misp_description'])
v['misp_description'] = description
if my_args['getuuid'] is True:
attribute_uuid = []
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is True:
v['misp_type'] = 'misp_object'
v['misp_value'] = r['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace('-', '_')
v[misp_t] = r['misp_value'] # set value for relevant type
to_ids = v['misp_to_ids']
if r['misp_to_ids'] not in to_ids:
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = v['misp_category']
if r['misp_category'] not in category: # append category
category.append(r['misp_category'])
v['misp_category'] = category
if my_args['add_desc'] is True:
description = v['misp_description']
if r['misp_description'] not in description:
description.append(r['misp_description'])
v['misp_description'] = description
if my_args['getuuid'] is True:
attribute_uuid = v['misp_attribute_uuid']
if r['misp_attribute_uuid'] not in attribute_uuid:
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is False:
misp_type = r['misp_type'] + '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + '|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
for k, v in output_dict.items():
yield v
class getmispioc(ReportingCommand):
'''
Extract IOC's from MISP
'''
server = Option(doc="",
require=False,
validate=validators.Match(
"server", r"^https?:\/\/[0-9a-zA-Z\.]+(?:\:\d+)?$"))
authkey = Option(doc="",
require=False,
validate=validators.Match("authkey",
r"^[0-9a-zA-Z]{40}$"))
sslcheck = Option(doc="",
require=False,
validate=validators.Match("sslcheck", r"^[yYnN01]$"))
eventid = Option(doc="",
require=False,
validate=validators.Match("eventid", r"^[0-9]+$"))
last = Option(doc="",
require=False,
validate=validators.Match("last", r"^[0-9]+[hdwm]$"))
onlyids = Option(doc="",
require=False,
validate=validators.Match("onlyids", r"^[yYnN01]+$"))
category = Option(doc="", require=False)
type = Option(doc="", require=False)
@Configuration()
def map(self, records):
self.logger.debug('getmispioc.map')
yield {}
return
def reduce(self, records):
self.logger.debug('getmispioc.reduce')
if self.sslcheck == None:
self.sslcheck = 'n'
# Generate args
my_args = {}
if self.server:
my_args['server'] = self.server
if self.authkey:
my_args['authkey'] = self.authkey
if self.sslcheck:
my_args['sslcheck'] = self.sslcheck
if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1':
onlyids = True
else:
onlyids = False
if self.eventid and self.last:
print('Options "eventid" and "last" are mutually exclusive')
exit(1)
if self.eventid:
my_args['eventid'] = self.eventid
elif self.last:
my_args['last'] = self.last
else:
print('Missing "eventid" or "last" argument')
exit(1)
_NEW_PYTHON_PATH = '/bin/python3'
_SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH
my_process = '/usr/local/bin/get-misp-ioc.py'
# Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues
env = dict(os.environ)
del env['LD_LIBRARY_PATH']
FNULL = open(os.devnull, 'w')
p = subprocess.Popen(
[os.environ['PYTHONPATH'], my_process,
str(my_args)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=FNULL,
env=env)
output = p.communicate()[0]
results = {}
for v in eval(output):
# Do not display deleted attributes
if v['deleted'] == False:
# If wpecified, do not display attributed with the non-ids flag set to False
if onlyids == True and v['to_ids'] == False:
continue
if self.category != None and self.category != v['category']:
continue
if self.type != None and self.type != v['type']:
continue
results['value'] = v['value']
results['category'] = v['category']
results['type'] = v['type']
results['to_ids'] = str(v['to_ids'])
yield results
class HiveCollectCommand(GeneratingCommand):
""" get the attributes from a TheHive instance.
##Syntax
.. code-block::
| mispgetioc hive_instance=<input> last=<int>(d|h|m)
| mispgetioc hive_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc hive_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
"""
# MANDATORY TheHive instance for this search
hive_instance = Option(doc='''
**Syntax:** **hive_instance=instance_name*
**Description:** TheHive instance parameters
as described in lookup/thehive_instance_list.csv.''',
require=True)
# MANDATORY: json_request XOR alertid XOR last XOR date
# json_request = Option(
# doc='''
# **Syntax:** **json_request=***valid JSON request*
# **Description:**Valid JSON request''',
# require=False)
objectid = Option(doc='''
**Syntax:** **objectid=***id1(,id2,...)*
**Description:**ID.''',
require=False,
validate=validators.Match("objectid",
r"^([0-9a-f]|\w{20})+$"))
endpoint = Option(doc='''
**Syntax:** **endpoint=***alert|case*
**Description:**endpoint of TheHive API''',
require=False,
validate=validators.Match("endpoint", r"^(alert|case)$"))
range = Option(doc='''
**Syntax:** **range=***val|start_number-end_number*
**Description:**valid range to limit number of alerts returned.
for example range=all or range=10-100''',
require=False,
validate=validators.Match("range", r"^(all|\d+\-\d+)$"))
def log_error(self, msg):
logging.error(msg)
def log_info(self, msg):
logging.info(msg)
def log_debug(self, msg):
logging.debug(msg)
def log_warn(self, msg):
logging.warning(msg)
def set_log_level(self):
logging.root
loglevel = logging_level('TA_thehive_ce')
logging.root.setLevel(loglevel)
logging.error('[CO-101] logging level is set to %s', loglevel)
logging.error('[CO-102] PYTHON VERSION: ' + sys.version)
@staticmethod
def _record(serial_number, time_stamp, host, attributes, attribute_names,
encoder):
raw = encoder.encode(attributes)
# Formulate record
fields = dict()
for f in attribute_names:
if f in attributes:
fields[f] = attributes[f]
if serial_number > 0:
fields['_serial'] = serial_number
fields['_time'] = time_stamp
fields['_raw'] = raw
fields['host'] = host
return fields
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('_raw', raw), ('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
return record
def displayResponse(results, action, host):
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if action == 'list_alert' or action == 'list_case':
attribute_names = list()
serial_number = 0
for a in results:
if serial_number == 0:
for k in list(a.keys()):
attribute_names.append(k)
if action == 'list_alert':
timestamp = int(a['date'] / 1000)
elif action == 'list_case':
timestamp = int(a['startDate'] / 1000)
yield HiveCollectCommand._record(serial_number, timestamp,
host, a, attribute_names,
encoder)
serial_number += 1
GeneratingCommand.flush
elif action == 'get_an_alert' or action == 'get_a_case':
attribute_names = list()
serial_number = 0
a = results
if serial_number == 0:
for k in list(a.keys()):
attribute_names.append(k)
if action == 'get_an_alert':
timestamp = int(a['date'] / 1000)
elif action == 'get_a_case':
timestamp = int(a['startDate'] / 1000)
yield HiveCollectCommand._record(serial_number, timestamp, host, a,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
def generate(self):
# Phase 1: Preparation
logging.root
loglevel = logging_level(self, 'TA_thehive_ce')
logging.root.setLevel(loglevel)
self.log_error('[CO-101] logging level is set to {}'.format(loglevel))
storage = self.service.storage_passwords
my_args = prepare_config(self, 'TA_thehive_ce', self.hive_instance,
storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for hive_instance={}".format(
self.hive_instance))
my_args['host'] = my_args['thehive_url'].replace('https://', '')
api_action = ''
if self.endpoint == 'case':
my_args['thehive_url'] = my_args['thehive_url'] + '/api/case'
api_action = 'list_case'
else:
self.endpoint = 'alert'
my_args['thehive_url'] = my_args['thehive_url'] + '/api/alert'
api_action = 'list_alert'
if self.objectid is not None:
my_args['thehive_url'] = my_args['thehive_url'] + '/' + str(
self.objectid)
if api_action == 'list_case':
api_action = 'get_a_case'
elif api_action == 'list_alert':
api_action = 'get_an_alert'
if self.range is not None:
my_args['range'] = str(self.range)
else:
my_args['range'] = "0-10"
headers = {"Authorization": "Bearer {}".format(my_args['thehive_key'])}
params = {"range": my_args['range']}
# # Search pagination
# pagination = True
# if self.limit is not None:
# limit = int(self.limit)
# elif 'limit' in body_dict:
# limit = int(body_dict['limit'])
# else:
# limit = 1000
# if limit == 0:
# pagination = False
# if self.page is not None:
# page = int(self.page)
# elif 'page' in body_dict:
# page = body_dict['page']
# else:
# page = 1
# body = json.dumps(body_dict)
# self.log_debug('mispgetioc request body: %s', body)
# search
r = requests.get(my_args['thehive_url'],
headers=headers,
params=params,
verify=my_args['thehive_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
# HiveCollectCommand.displayResponse(response, api_action, my_args['host'])
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if api_action == 'list_alert' or api_action == 'list_case':
attribute_names = list()
serial_number = 0
for a in response:
if serial_number == 0:
for k in list(a.keys()):
attribute_names.append(k)
if api_action == 'list_alert':
timestamp = int(a['date'] / 1000)
elif api_action == 'list_case':
timestamp = int(a['startDate'] / 1000)
yield HiveCollectCommand._record(serial_number, timestamp,
my_args['host'], a,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
elif api_action == 'get_an_alert' or api_action == 'get_a_case':
attribute_names = list()
a = response
for k in list(a.keys()):
attribute_names.append(k)
if api_action == 'get_an_alert':
timestamp = int(a['date'] / 1000)
elif api_action == 'get_a_case':
timestamp = int(a['startDate'] / 1000)
yield HiveCollectCommand._record(0, timestamp, my_args['host'], a,
attribute_names, encoder)
GeneratingCommand.flush
class ToSFXCommand(EventingCommand):
"""
## Syntax
<command> | tosfx
## Description
One or more datapoints are generated for each input event's field(s) of the
form `gauge_*`, `counter_*` or `cumulative_counter_*`. The metric name in
SignalFx will be the `*` part of the field name. Any additional fields on
the event will be attached as dimensions to the generated datapoints.
"""
access_token = None
debug = Option(validate=validators.Boolean(), default=False)
dry_run = Option(validate=validators.Boolean(), default=False)
ingest_url = Option(
validate=validators.Match("https://.*", r"^https://.*"))
dp_endpoint = Option(default="/v2/datapoint")
SPLUNK_PASSWORD_REALM = "realm"
SPLUNK_PASSWORD_USER_NAME = "username"
SPLUNK_PASSWORD_CLEAR_PASSWORD = "******"
SPLUNK_KV_STORE_SFX_CONFIG_COLLECTION_NAME = "sfx_ingest_config"
SPLUNK_SFX_CONFIG_INGEST_URL_KEY = "ingest_url"
SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_REALM = "sfx_ingest_command"
SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_USER_NAME = "access_token"
def ensure_default_config(self):
if not self.ingest_url:
self.ingest_url = self.get_sfx_ingest_url()
self.logger.error("getting access token")
self.access_token = self.get_access_token()
def get_access_token(self):
try:
for credential in self.service.storage_passwords:
if (credential.content.get(self.SPLUNK_PASSWORD_REALM, None)
== self.SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_REALM
and credential.content.get(
self.SPLUNK_PASSWORD_USER_NAME, None) == self.
SPLUNK_PASSWORDS_STORAGE_SFX_ACCESS_TOKEN_USER_NAME):
return credential.content.get(
self.SPLUNK_PASSWORD_CLEAR_PASSWORD, None)
except Exception as e: # pylint:disable=broad-except
self.logger.error(
"status=error, action=get_sfx_access_token, error_msg=%s",
e,
exc_info=True)
return None
def get_sfx_ingest_url(self):
try:
sfx_api_config_collection = self.service.kvstore.get(
self.SPLUNK_KV_STORE_SFX_CONFIG_COLLECTION_NAME, None)
if sfx_api_config_collection is not None:
collection = self.service.kvstore[
self.SPLUNK_KV_STORE_SFX_CONFIG_COLLECTION_NAME]
# will return a list of settings, we should only have one.
# We'll grab the 'last' / most recent one by default
sfx_ingest_config_records = collection.data.query()
if sfx_ingest_config_records:
sfx_ingest_config_latest = sfx_ingest_config_records[-1]
return sfx_ingest_config_latest.get(
self.SPLUNK_SFX_CONFIG_INGEST_URL_KEY, None)
except Exception as e: # pylint:disable=broad-except
self.logger.error(
"status=error, action=get_sfx_ingest_url, error_msg=%s",
str(e),
exc_info=True)
return None
def transform(self, records):
self.ensure_default_config()
out = []
payload = OrderedDict()
for event in records:
add_event_to_payload(event=event, payload=payload)
if self.debug:
event["endpoint"] = self.ingest_url + self.dp_endpoint
out.append(event)
if not self.dry_run:
resp = send_payload(
payload=payload,
target_url=compose_ingest_url(self.ingest_url,
self.dp_endpoint),
token=self.access_token,
)
for event in out:
event["status"] = resp.status_code
if resp.status_code != 200:
event["response_error"] = resp.content
for event in out:
yield event
class MispRestCommand(GeneratingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetioc misp_instance=<input> last=<int>(d|h|m)
| mispgetioc misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional",
"includeDecayScore": "optional",
"includeFullModel": "optional",
"decayingModel": "optional",
"excludeDecayed": "optional",
"score": "optional"
}
# status
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": forced to False,
"includeEventUuid": set to True,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": not managed,
"eventinfo": not managed,
"includeProposals": not managed
"includeDecayScore": not managed,
"includeFullModel": not managed,
"decayingModel": not managed,
"excludeDecayed": not managed,
"score": not managed
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:** MISP instance parameters
as described in local/misp42splunk_instances.conf.''',
require=True)
method = Option(doc='''
**Syntax:** **method=****
**Description:** method to use for API target DELETE GET PATCH POST.''',
require=True,
validate=validators.Match("method",
r"^(DELETE|GET|POST)$"))
json_request = Option(doc='''
**Syntax:** **json_request=***JSON request*
**Description:** JSON-formatted json_request.''',
require=False,
validate=validators.Match("json_request", r"^{.+}$"))
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search;
default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("page", r"^[0-9]+$"))
target = Option(
doc='''
**Syntax:** **target=api_target****
**Description:**target of MISP API.''',
require=True,
validate=validators.Match(
"target",
r"^/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+$"
))
def generate(self):
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
if self.target not in [None, '']:
my_args['misp_url'] = my_args['misp_url'] + self.target
if self.json_request not in [None, '']:
body_dict = json.loads(self.json_request)
logging.debug('[MR-201] body_dict is {}'.format(body_dict))
else:
body_dict = {}
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
if self.method == "GET":
r = requests.get(my_args['misp_url'],
headers=headers,
params=body_dict,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
elif self.method == "POST":
r = requests.post(my_args['misp_url'],
headers=headers,
data=json.dumps(body_dict),
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
elif self.method == "DELETE":
r = requests.delete(my_args['misp_url'],
headers=headers,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
else:
raise Exception(
"Sorry, no valid method provided (GET/POST//DELETE)."
" it was {}.".format(self.method))
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
logging.info("[RE301] INFO mispcollect successful. "
"url={}, HTTP status={}".format(
my_args['misp_url'], r.status_code))
else:
logging.error("[RE302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}".format(
my_args['misp_url'], body_dict, r.status_code,
r.text))
raise Exception(
"[RE302] ERROR mispcollect failed. "
"url={}, data={}, HTTP Error={}, content={}".format(
my_args['misp_url'], body_dict, r.status_code, r.text))
# response is 200 by this point or we would have thrown an exception
data = {'_time': time.time(), '_raw': json.dumps(r.json())}
yield data
class MispSearchCommand(StreamingCommand):
""" search in MISP for attributes matching the value of field.
##Syntax
code-block::
mispsearch field=<field> onlyids=y|n
##Description
body = {
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional"
}
##Example
Search in MISP for value of fieldname r_ip (remote IP in proxy logs).
code-block::
* | mispsearch field=r_ip
"""
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as described in local/inputs.conf''',
require=True)
field = Option(doc='''
**Syntax:** **field=***<fieldname>*
**Description:**Name of the field containing the value to search for.''',
require=True,
validate=validators.Fieldname())
onlyids = Option(doc='''
**Syntax:** **onlyids=***<y|n>*
**Description:** Boolean to search only attributes with to_ids set''',
require=False,
validate=validators.Boolean())
gettag = Option(doc='''
**Syntax:** **gettag=***<y|n>*
**Description:** Boolean to return attribute tags''',
require=False,
validate=validators.Boolean())
includeEventUuid = Option(doc='''
**Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
includeEventTags = Option(doc='''
**Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:**publication duration in day(s), hour(s) or minute(s). **eventid**, **last** and **date_from** are mutually exclusive''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
def stream(self, records):
# Generate args
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
if self.gettag is True:
get_tag = True
else:
get_tag = False
pagination = True
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 1000
if self.page is not None:
page = int(self.page)
else:
page = 1
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
if 'page' in body_dict:
page = body_dict['page']
pagination = False
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
if self.onlyids is True:
body_dict['to_ids'] = "True"
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
if self.last is not None:
body_dict['last'] = self.last
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict['value'] = str(value)
misp_category = []
misp_event_id = []
misp_event_uuid = []
misp_orgc_id = []
misp_to_ids = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispsearch request body: %s', body)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
# print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json()
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
if 'Event' in a:
if a['Event'][
'uuid'] not in misp_event_uuid:
misp_event_uuid.append(
str(a['Event']['uuid']))
if a['Event'][
'orgc_id'] not in misp_orgc_id:
misp_orgc_id.append(
str(a['Event']['orgc_id']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
record['misp_event_uuid'] = misp_event_uuid
record['misp_orgc_id'] = misp_orgc_id
record['misp_tag'] = misp_tag
yield record
class MispGetEventCommand(GeneratingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| MispGetEventCommand misp_instance=<input> last=<int>(d|h|m)
| MispGetEventCommand misp_instance=<input> event=<id1>(,<id2>,...)
| MispGetEventCommand misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tag": "optional",
"tags": "optional",
"searchall": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"metadata": "optional",
"uuid": "optional",
"published": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"sgReferenceOnly": "optional",
"eventinfo": "optional",
"excludeLocalTags": "optional"
}
# status
"tag": "optional",
"searchall": "optional",
"metadata": "optional",
"published": "optional",
"sgReferenceOnly": "optional",
"eventinfo": "optional",
"excludeLocalTags": "optional"
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": not managed,
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as described
in local/inputs.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date. **eventid**, **last**
and **date** are mutually exclusive''',
require=False)
# Other params
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
getioc = Option(doc='''
**Syntax:** **getioc=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return the list of attributes
together with the event.''',
require=False,
validate=validators.Boolean())
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; default 1000.
0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
output = Option(doc='''
**Syntax:** **output=***<default|rawy>*
**Description:**selection between a tabular or JSON output.''',
require=False,
validate=validators.Match("output", r"(default|raw)"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes.''',
require=False,
validate=validators.Boolean())
published = Option(doc='''
**Syntax:** **published=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**select only published events (for option from to) .''',
require=False,
validate=validators.Boolean())
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
warning_list = Option(doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False,
validate=validators.Boolean())
@staticmethod
def _record(serial_number,
time_stamp,
host,
attributes,
attribute_names,
encoder,
condensed=False):
if condensed is False:
raw = encoder.encode(attributes)
# Formulate record
fields = dict()
for f in attribute_names:
if f in attributes:
fields[f] = attributes[f]
if serial_number > 0:
fields['_serial'] = serial_number
fields['_time'] = time_stamp
if condensed is False:
fields['_raw'] = raw
fields['host'] = host
return fields
if condensed is False:
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('_raw', raw), ('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
else:
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
return record
def generate(self):
# Phase 1: Preparation
my_args = prepare_config(self, 'misp42splunk')
my_args['host'] = my_args['misp_url'].replace('https://', '')
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
# Search parameters: boolean and filter
# manage enforceWarninglist
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getioc is True:
getioc = True
else:
getioc = False
if self.pipesplit is True:
pipesplit = True
else:
pipesplit = False
if self.output is not None:
output = self.output
else:
output = "default"
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.error('MispGetEventCommand request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if output == "raw":
if 'response' in response:
attribute_names = list()
serial_number = 0
for r_item in response['response']:
if 'Event' in r_item:
for e in r_item.values():
yield MispGetEventCommand._record(
serial_number, e['timestamp'], my_args['host'],
e, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
# build output table and list of types
events = []
typelist = []
column_list = format_output_table(response, events, typelist,
getioc, pipesplit)
logging.info('typelist containss %s values', str(len(typelist)))
logging.debug('typelist is %s', json.dumps(typelist))
logging.info('results contains %s records', str(len(events)))
if getioc is False:
attribute_names = list()
init_attribute_names = True
serial_number = 0
for e in events:
# logging.debug('event is %s', json.dumps(e))
if init_attribute_names is True:
for key in e.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
logging.debug(json.dumps(attribute_names))
yield MispGetEventCommand._record(serial_number,
e['misp_timestamp'],
my_args['host'], e,
attribute_names, encoder,
True)
serial_number += 1
GeneratingCommand.flush
else:
output_dict = {}
for e in events:
if 'Attribute' in e:
for a in e['Attribute']:
if int(a['misp_object_id']) == 0: # not an object
key = str(e['misp_event_id']) + '_' \
+ str(a['misp_attribute_id'])
is_object_member = False
else: # this is a MISP object
key = str(e['misp_event_id']) + \
'_object_' + str(a['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = init_misp_output(e, a, column_list)
for t in typelist:
misp_t = 'misp_' \
+ t.replace('-', '_')\
.replace('|', '_p_')
v[misp_t] = []
if t == a['misp_type']:
v[misp_t].append(a['misp_value'])
if is_object_member is True:
v['misp_type'] = v['misp_object_name']
v['misp_value'] = v['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + a['misp_type']\
.replace('-', '_').replace('|', '_p_')
v[misp_t].append(a['misp_value'])
if a['misp_to_ids'] not in v['misp_to_ids']:
v['misp_to_ids'].append(a['misp_to_ids'])
if a['misp_category'] not in v[
'misp_category']:
v['misp_category'].append(
a['misp_category'])
v['misp_attribute_uuid']\
.append(a['misp_attribute_uuid'])
v['misp_attribute_id']\
.append(a['misp_attribute_id'])
if a['misp_attribute_tag'] is not None:
a_tag = v['misp_attribute_tag']
for t in a['misp_attribute_tag']:
if t not in a_tag:
a_tag.append(t)
v['misp_attribute_tag'] = a_tag
if a['misp_comment'] not in v['misp_comment']:
v['misp_comment'].append(a['misp_comment'])
if is_object_member is False:
misp_type = a['misp_type'] \
+ '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = a['misp_value'] + \
'|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
if output_dict is not None:
attribute_names = list()
init_attribute_names = True
serial_number = 0
for v in output_dict.values():
if init_attribute_names is True:
for key in v.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
logging.debug(json.dumps(attribute_names))
yield MispGetEventCommand._record(
serial_number, v['misp_timestamp'],
my_args['host'], v, attribute_names, encoder, True)
serial_number += 1
GeneratingCommand.flush
class mispsight(StreamingCommand):
""" search in MISP for attributes matching the value of field.
##Syntax
code-block::
mispsearch field=<field> onlyids=y|n
##Description
search_body = {"returnFormat": "json",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional"
}
##Example
Search in MISP for value of fieldname r_ip (remote IP in proxy logs).
code-block::
* | mispsearch fieldname=r_ip
"""
field = Option(doc='''
**Syntax:** **field=***<fieldname>*
**Description:**Name of the field containing the value to search for.''',
require=True,
validate=validators.Fieldname())
# Superseede MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as decibed in lookup/misp_instances.csv.''',
require=False)
misp_url = Option(doc='''
**Syntax:** **misp_url=***<MISP URL>*
**Description:**URL of MISP instance.''',
require=False,
validate=validators.Match(
"misp_url",
r"^https?:\/\/[0-9a-zA-Z\-\.]+(?:\:\d+)?$"))
misp_key = Option(doc='''
**Syntax:** **misp_key=***<AUTH_KEY>*
**Description:**MISP API AUTH KEY.''',
require=False,
validate=validators.Match("misp_key",
r"^[0-9a-zA-Z]{40}$"))
misp_verifycert = Option(doc='''
**Syntax:** **misp_verifycert=***<y|n>*
**Description:**Verify or not MISP certificate.''',
require=False,
validate=validators.Boolean())
def stream(self, records):
# self.logger.debug('mispgetioc.reduce')
# Generate args
my_args = prepare_config(self)
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
search_url = my_args['misp_url'] + '/attributes/restSearch'
search_dict = {"returnFormat": "json"}
search_dict['value'] = str(value)
search_dict['withAttachments'] = "false",
search_body = json.dumps(search_dict)
sight_url = my_args[
'misp_url'] + '/sightings/restSearch/attribute'
sight_dict = {"returnFormat": "json"}
misp_value = ''
misp_fp = False
misp_fp_timestamp = 0
misp_fp_event_id = ''
misp_sight_seen = False
misp_sight = {
'count': 0,
'first': 0,
'first_event_id': 0,
'last': 0,
'last_event_id': 0
}
# search
r = requests.post(search_url,
headers=headers,
data=search_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
logging.info("INFO MISP REST API %s has response: %s" %
(search_url, r.json()))
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if misp_value == '':
misp_value = str(a['value'])
if misp_fp == False:
sight_dict['id'] = str(a['id'])
sight_body = json.dumps(sight_dict)
s = requests.post(
sight_url,
headers=headers,
data=sight_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
s.raise_for_status()
# response is 200 by this point or we would have thrown an exception
sight = s.json()
logging.info(
"INFO MISP REST API %s has response: %s"
% (sight_url, s.json()))
if 'response' in sight:
for se in sight['response']:
if 'Sighting' in se:
if int(se['Sighting']['type']
) == 0: #true sighting
misp_sight_seen = True
misp_sight[
'count'] = misp_sight[
'count'] + 1
if misp_sight['first'] == 0 or \
misp_sight['first'] > int(se['Sighting']['date_sighting']):
misp_sight[
'first'] = int(
se['Sighting']
['date_sighting']
)
misp_sight[
'first_event_id'] = se[
'Sighting'][
'event_id']
if misp_sight['last'] < int(
se['Sighting']
['date_sighting']):
misp_sight['last'] = int(
se['Sighting']
['date_sighting'])
misp_sight[
'last_event_id'] = se[
'Sighting'][
'event_id']
elif int(
se['Sighting']['type']
) == 1: #false positive
misp_fp = True
misp_fp_timestamp = int(
se['Sighting']
['date_sighting'])
misp_fp_event_id = se[
'Sighting']['event_id']
if misp_fp == True:
record['misp_value'] = misp_value
record['misp_fp'] = "True"
record['misp_fp_timestamp'] = str(
misp_fp_timestamp)
record['misp_fp_event_id'] = str(
misp_fp_event_id)
if misp_sight_seen == True:
record['misp_value'] = misp_value
record['misp_sight_count'] = str(
misp_sight['count'])
record['misp_sight_first'] = str(
misp_sight['first'])
record['misp_sight_first_event_id'] = str(
misp_sight['first_event_id'])
record['misp_sight_last'] = str(
misp_sight['last'])
record['misp_sight_last_event_id'] = str(
misp_sight['last_event_id'])
yield record
class MispGetIocCommand(GeneratingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetioc misp_instance=<input> last=<int>(d|h|m)
| mispgetioc misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional",
"includeDecayScore": "optional",
"includeFullModel": "optional",
"decayingModel": "optional",
"excludeDecayed": "optional",
"score": "optional"
}
# status
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": forced to False,
"includeEventUuid": set to True,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": not managed,
"eventinfo": not managed,
"includeProposals": not managed
"includeDecayScore": not managed,
"includeFullModel": not managed,
"decayingModel": not managed,
"excludeDecayed": not managed,
"score": not managed
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:** MISP instance parameters
as described in local/inputs.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date.
**eventid**, **last** and **date** are mutually exclusive''',
require=False)
# Other params
add_description = Option(doc='''
**Syntax:** **add_description=***<1|y|Y|t|true|True
|0|n|N|f|false|False>*
**Description:**Boolean to return misp_description.''',
require=False,
validate=validators.Boolean())
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
geteventtag = Option(doc='''
**Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean includeEventTags. By default only
attribute tag(s) are returned.''',
require=False,
validate=validators.Boolean())
getorg = Option(doc='''
**Syntax:** **getorg=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return the ID of the organisation that
created the event.''',
require=False,
validate=validators.Boolean())
getuuid = Option(doc='''
**Syntax:** **getuuid=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return attribute UUID.''',
require=False,
validate=validators.Boolean())
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search;
default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
output = Option(doc='''
**Syntax:** **output=***<default|rawy>*
**Description:**selection between the default behaviou or \
JSON output by event.''',
require=False,
validate=validators.Match("output", r"(default|raw)"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes.''',
require=False,
validate=validators.Boolean())
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
to_ids = Option(doc='''
**Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to search only attributes with the flag
"to_ids" set to true.''',
require=False,
validate=validators.Boolean())
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
warning_list = Option(doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False,
validate=validators.Boolean())
@staticmethod
def _record(serial_number, time_stamp, host, attributes, attribute_names,
encoder):
raw = encoder.encode(attributes)
# Formulate record
fields = dict()
for f in attribute_names:
if f in attributes:
fields[f] = attributes[f]
if serial_number > 0:
fields['_serial'] = serial_number
fields['_time'] = time_stamp
fields['_raw'] = raw
fields['host'] = host
return fields
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('_raw', raw), ('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
return record
def generate(self):
# Phase 1: Preparation
my_args = prepare_config(self, 'misp42splunk')
my_args['host'] = my_args['misp_url'].replace('https://', '')
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
# manage to_ids and enforceWarninglist
# to avoid FP enforceWarninglist is set to True if
# to_ids is set to True (search criterion)
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getuuid is True:
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.getorg is True:
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
if self.add_description is True:
my_args['add_desc'] = True
else:
my_args['add_desc'] = False
if self.output is not None:
my_args['output'] = self.output
else:
my_args['output'] = "default"
results = []
# add colums for each type in results
typelist = []
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispgetioc request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if my_args['output'] == "raw":
if 'response' in response:
if 'Attribute' in response['response']:
attribute_names = []
serial_number = 0
for a in response['response']['Attribute']:
yield MispGetIocCommand._record(
serial_number, a['timestamp'], my_args['host'], a,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
v = {}
v['misp_category'] = str(a['category'])
v['misp_attribute_id'] = str(a['id'])
v['misp_event_id'] = str(a['event_id'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_to_ids'] = str(a['to_ids'])
v['misp_comment'] = str(a['comment'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
# include ID of the organisation that
# created the attribute if requested
if 'Event' in a:
v['misp_event_uuid'] = str(a['Event']['uuid'])
if my_args['getorg']:
v['misp_orgc_id'] = str(a['Event']['orgc_id'])
if my_args['add_desc'] is True:
v['misp_event_info'] = str(a['Event']['info'])
# include attribute UUID if requested
if my_args['getuuid']:
v['misp_attribute_uuid'] = str(a['uuid'])
# handle object and multivalue attributes
v['misp_object_id'] = str(a['object_id'])
if my_args['add_desc'] is True:
if int(a['object_id']) == 0:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) \
+ '" in category "' + str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) + ')'
else:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) + '" in category "' \
+ str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) \
+ ' - o' + str(a['object_id']) + ' )'
current_type = str(a['type'])
# combined: not part of an object
# AND multivalue attribute AND to be split
# logging.debug('misp_event: %s', json.dumps(v))
if int(a['object_id']) == 0 and '|' in current_type \
and my_args['pipe'] is True:
mv_type_list = current_type.split('|')
mv_value_list = str(a['value']).split('|')
left_v = v.copy()
left_v['misp_type'] = mv_type_list.pop()
left_v['misp_value'] = mv_value_list.pop()
results.append(left_v)
if left_v['misp_type'] not in typelist:
typelist.append(left_v['misp_type'])
right_v = v.copy()
right_v['misp_type'] = mv_type_list.pop()
right_v['misp_value'] = mv_value_list.pop()
results.append(right_v)
if right_v['misp_type'] not in typelist:
typelist.append(right_v['misp_type'])
else:
v['misp_type'] = current_type
v['misp_value'] = str(a['value'])
results.append(v)
if current_type not in typelist:
typelist.append(current_type)
logging.info(json.dumps(typelist))
output_dict = {}
attribute_names = [
'misp_attribute_id', 'misp_attribute_uuid', 'misp_category',
'misp_comment', 'misp_description', 'misp_event_id',
'misp_tag', 'misp_timestamp', 'misp_to_ids', 'misp_type',
'misp_value'
]
if my_args['add_desc'] is True:
attribute_names.append('misp_event_info')
for t in typelist:
misp_t = 'misp_' + \
t.replace('-', '_').replace('|', '_p_')
if misp_t not in attribute_names:
attribute_names.append(misp_t)
for r in results:
if int(r['misp_object_id']) == 0: # not an object
key = str(r['misp_event_id']) + \
'_' + r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(r['misp_event_id']) \
+ '_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_')\
.replace('|', '_p_')
v[misp_t] = []
if t == r['misp_type']:
v[misp_t].append(r['misp_value'])
v['misp_to_ids'] = []
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'] = []
v['misp_category'].append(r['misp_category'])
v['misp_attribute_id'] = []
v['misp_attribute_id']\
.append(r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid'] = []
v['misp_attribute_uuid']\
.append(r['misp_attribute_uuid'])
if my_args['add_desc'] is True:
description = []
description.append(r['misp_description'])
v['misp_description'] = description
if is_object_member is True:
v['misp_type'] = 'misp_object'
v['misp_value'] = r['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace('-', '_')
v[misp_t].append(r['misp_value']) # set value for type
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'].append(r['misp_category'])
tag_list = v['misp_tag']
for tag in r['misp_tag']:
if tag not in tag_list:
tag_list.append(tag)
v['misp_tag'] = tag_list
if my_args['add_desc'] is True:
description = v['misp_description']
if r['misp_description'] not in description:
description.append(r['misp_description'])
v['misp_description'] = description
v['misp_attribute_id']\
.append(r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid']\
.append(r['misp_attribute_uuid'])
if is_object_member is False:
misp_type = r['misp_type'] + '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + '|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
serial_number = 0
logging.debug(json.dumps(attribute_names))
for k, v in list(output_dict.items()):
yield MispGetIocCommand._record(serial_number,
v['misp_timestamp'],
my_args['host'], v,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
class TestSearchCommand(SearchCommand):
boolean = Option(
doc='''
**Syntax:** **boolean=***<value>*
**Description:** A boolean value''',
validate=validators.Boolean())
required_boolean = Option(
doc='''
**Syntax:** **boolean=***<value>*
**Description:** A boolean value''',
require=True, validate=validators.Boolean())
aliased_required_boolean = Option(
doc='''
**Syntax:** **boolean=***<value>*
**Description:** A boolean value''',
name='foo', require=True, validate=validators.Boolean())
code = Option(
doc='''
**Syntax:** **code=***<value>*
**Description:** A Python expression, if mode == "eval", or statement, if mode == "exec"''',
validate=validators.Code())
required_code = Option(
doc='''
**Syntax:** **code=***<value>*
**Description:** A Python expression, if mode == "eval", or statement, if mode == "exec"''',
require=True, validate=validators.Code())
duration = Option(
doc='''
**Syntax:** **duration=***<value>*
**Description:** A length of time''',
validate=validators.Duration())
required_duration = Option(
doc='''
**Syntax:** **duration=***<value>*
**Description:** A length of time''',
require=True, validate=validators.Duration())
fieldname = Option(
doc='''
**Syntax:** **fieldname=***<value>*
**Description:** Name of a field''',
validate=validators.Fieldname())
required_fieldname = Option(
doc='''
**Syntax:** **fieldname=***<value>*
**Description:** Name of a field''',
require=True, validate=validators.Fieldname())
file = Option(
doc='''
**Syntax:** **file=***<value>*
**Description:** Name of a file''',
validate=validators.File())
required_file = Option(
doc='''
**Syntax:** **file=***<value>*
**Description:** Name of a file''',
require=True, validate=validators.File())
integer = Option(
doc='''
**Syntax:** **integer=***<value>*
**Description:** An integer value''',
validate=validators.Integer())
required_integer = Option(
doc='''
**Syntax:** **integer=***<value>*
**Description:** An integer value''',
require=True, validate=validators.Integer())
map = Option(
doc='''
**Syntax:** **map=***<value>*
**Description:** A mapping from one value to another''',
validate=validators.Map(foo=1, bar=2, test=3))
required_map = Option(
doc='''
**Syntax:** **map=***<value>*
**Description:** A mapping from one value to another''',
require=True, validate=validators.Map(foo=1, bar=2, test=3))
match = Option(
doc='''
**Syntax:** **match=***<value>*
**Description:** A value that matches a regular expression pattern''',
validate=validators.Match('social security number', r'\d{3}-\d{2}-\d{4}'))
required_match = Option(
doc='''
**Syntax:** **required_match=***<value>*
**Description:** A value that matches a regular expression pattern''',
require=True, validate=validators.Match('social security number', r'\d{3}-\d{2}-\d{4}'))
optionname = Option(
doc='''
**Syntax:** **optionname=***<value>*
**Description:** The name of an option (used internally)''',
validate=validators.OptionName())
required_optionname = Option(
doc='''
**Syntax:** **optionname=***<value>*
**Description:** The name of an option (used internally)''',
require=True, validate=validators.OptionName())
regularexpression = Option(
doc='''
**Syntax:** **regularexpression=***<value>*
**Description:** Regular expression pattern to match''',
validate=validators.RegularExpression())
required_regularexpression = Option(
doc='''
**Syntax:** **regularexpression=***<value>*
**Description:** Regular expression pattern to match''',
require=True, validate=validators.RegularExpression())
set = Option(
doc='''
**Syntax:** **set=***<value>*
**Description:** A member of a set''',
validate=validators.Set('foo', 'bar', 'test'))
required_set = Option(
doc='''
**Syntax:** **set=***<value>*
**Description:** A member of a set''',
require=True, validate=validators.Set('foo', 'bar', 'test'))
class ConfigurationSettings(SearchCommand.ConfigurationSettings):
@classmethod
def fix_up(cls, command_class):
pass
class GetJiraKv(GeneratingCommand):
verify = Option(
doc='''
**Syntax:** **verify=****
**Description:** verify the connectivity to a remote instance. True / False are supported.''',
require=False, default="False", validate=validators.Match("verify", r"^(True|False)$"))
def generate(self, **kwargs):
if self:
# Get the session key
session_key = self._metadata.searchinfo.session_key
# Get splunkd port
entity = splunk.entity.getEntity('/server', 'settings',
namespace='TA-jira-service-desk-simple-addon', sessionKey=session_key, owner='-')
splunkd_port = entity['mgmtHostPort']
# Get conf
conf_file = "ta_service_desk_simple_addon_settings"
confs = self.service.confs[str(conf_file)]
storage_passwords = self.service.storage_passwords
kvstore_instance = None
bearer_token = None
for stanza in confs:
if stanza.name == "advanced_configuration":
for key, value in stanza.content.items():
if key == "jira_passthrough_mode":
jira_passthrough_mode = value
if key == "kvstore_instance":
kvstore_instance = value
if key == "kvstore_search_filters":
kvstore_search_filters = value
if kvstore_instance:
# The bearer token is stored in the credential store
# However, likely due to the number of chars, the credential.content.get SDK command is unable to return its value in a single operation
# As a workaround, we concatenate the different values return to form a complete object, finally we use a regex approach to extract its clear text value
credential_realm = '__REST_CREDENTIAL__#TA-jira-service-desk-simple-addon#configs/conf-ta_service_desk_simple_addon_settings'
bearer_token_rawvalue = ""
for credential in storage_passwords:
if credential.content.get('realm') == str(credential_realm):
bearer_token_rawvalue = bearer_token_rawvalue + str(credential.content.clear_password)
# extract a clean json object
bearer_token_rawvalue_match = re.search('\{\"bearer_token\":\s*\"(.*)\"\}', bearer_token_rawvalue)
if bearer_token_rawvalue_match:
bearer_token = bearer_token_rawvalue_match.group(1)
else:
bearer_token = None
# the root search
search = "| inputlookup jira_failures_replay | eval uuid=_key, mtime=if(isnull(mtime), ctime, mtime)"
# If the passthrough mode is disabled, there is no distributed setup
# and the instance is the localhost
if (not kvstore_instance or not bearer_token) and str(jira_passthrough_mode) == '0':
kvstore_instance = 'localhost:' + str(splunkd_port)
header = 'Splunk ' + str(session_key)
elif str(jira_passthrough_mode) == '1':
# yield
data = {'_time': time.time(), '_raw': "{\"response\": \"" + "INFO: Passthrough mode is currently enabled in this instance, you can safety disable the alert execution for this instance.}"}
yield data
sys.exit(0)
elif kvstore_instance and not bearer_token:
# yield
data = {'_time': time.time(), '_raw': "{\"response\": \"" + "ERROR: The KVstore instance is set but not the bearer token.}"}
yield data
sys.exit(0)
elif bearer_token and not kvstore_instance:
# yield
data = {'_time': time.time(), '_raw': "{\"response\": \"" + "ERROR: The bearer token is set but not the KVstore instance.}"}
yield data
sys.exit(0)
else:
header = 'Bearer ' + str(bearer_token)
search = str(search) + " | search " + str(kvstore_search_filters)
# Define the url
url = "https://" + str(kvstore_instance) + "/services/search/jobs/export"
# Get data
output_mode = "csv"
exec_mode = "oneshot"
response = requests.post(url, headers={'Authorization': header}, verify=False, data={'search': search, 'output_mode': output_mode, 'exec_mode': exec_mode})
csv_data = response.text
if response.status_code not in (200, 201, 204):
response_error = 'JIRA Get remove KVstore has failed!. url={}, data={}, HTTP Error={}, content={}'.format(url, search, response.status_code, response.text)
self.logger.fatal(str(response_error))
data = {'_time': time.time(), '_raw': "{\"response\": \"" + str(response_error) + "\""}
yield data
sys.exit(0)
else:
if self.verify == 'True':
response_error = 'JIRA Get remove KVstore was successfull. url={}, data={}, HTTP Error={}'.format(url, search, response.status_code)
data = {'_time': time.time(), '_raw': "{\"response\": \"" + str(response_error) + "\""}
yield data
sys.exit(0)
else:
# Use the CSV dict reader
readCSV = csv.DictReader(csv_data.splitlines(True), delimiter=str(u','), quotechar=str(u'"'))
# For row in CSV, generate the _raw
for row in readCSV:
yield {'_time': time.time(), 'uuid': str(row['uuid']), 'account': str(row['account']), 'data': str(row['data']), 'status': str(row['status']), 'ctime': str(row['ctime']), 'mtime': str(row['mtime']), 'no_attempts': str(row['no_attempts'])}
else:
# yield
data = {'_time': time.time(), '_raw': "{\"response\": \"" + "Error: bad request}"}
yield data
