class Executestory(GeneratingCommand):
logger = splunk.mining.dcutils.getLogger()
story = Option(doc='''
**Syntax:** **story=***<story name>*
**Description:** Story to update.
''',
name='story',
require=True,
default=None)
mode = Option(require=True)
cron = Option(require=False)
earliest_time = Option(require=False)
latest_time = Option(require=False)
def getURL(self):
cfg = cli.getConfStanza('asx', 'settings')
self.logger.info("executestory.py - asx_conf: {0}".format(
cfg['api_url']))
return cfg['api_url']
def generate(self):
if self.earliest_time:
earliest_time = self.earliest_time
if self.latest_time:
latest_time = self.latest_time
# connect to splunk and start execution
port = splunk.getDefault('port')
service = splunklib.client.connect(
token=self._metadata.searchinfo.session_key,
port=port,
owner="nobody",
app="Splunk_ASX")
self.logger.info("executestory.py - starting ASX - {0} ".format(
self.story))
API_URL = self.getURL()
asx_lib = ASXLib(service, API_URL)
#Runnning the selected analytic story
if self.mode == "now":
#time attributes from time picker
if hasattr(self.search_results_info, 'search_et') and hasattr(
self.search_results_info, 'search_lt'):
earliest_time = self.search_results_info.search_et
latest_time = self.search_results_info.search_lt
search_name = asx_lib.run_analytics_story(self.story,
earliest_time,
latest_time)
yield {
'_time': time.time(),
'sourcetype': "_json",
'_raw': {
'analytic_story':
self.story,
'search_name':
search_name,
'mode':
self.mode,
'status':
"Successfully executed the searches in the analytic story"
}
}
#Schedule the selected analytic story if cron is selected
if self.mode == "schedule":
if self.cron:
search_name = asx_lib.schedule_analytics_story(
self.story, earliest_time, latest_time, self.cron)
yield {
'_time': time.time(),
'sourcetype': "_json",
'_raw': {
'analytic_story': self.story,
'search_name': search_name,
'mode': self.mode,
'cron_schecule': self.cron,
'status': "Successfully scheduled the analytic story"
}
}
self.logger.info("executestory.py - completed ASX - {0} ".format(
self.story))
def __init__(self):
super(Executestory, self).__init__()
class SimulateCommand(GeneratingCommand):
""" Generates a sequence of events drawn from a CSV file using repeated random sampling
##Syntax
.. code-block::
simulate csv=<path> rate=<expected_event_count> interval=<sampling_period> duration=<execution_period>
[seed=<string>]
##Description
The :code:`simulate` command uses repeated random samples of the event records in :code:`csv` for the execution
period of :code:`duration`. Sample sizes are determined for each time :code:`interval` in :code:`duration`
using a Poisson distribution with an average :code:`rate` specifying the expected event count during
:code:`interval`.
##Example
.. code-block::
| simulate csv=population.csv rate=50 interval=00:00:01
duration=00:00:05 | countmatches fieldname=word_count
pattern="\\w+" text | stats mean(word_count) stdev(word_count)
This example generates events drawn from repeated random sampling of events from :code:`tweets.csv`. Events are
drawn at an average rate of 50 per second for a duration of 5 seconds. Events are piped to the example
:code:`countmatches` command which adds a :code:`word_count` field containing the number of words in the
:code:`text` field of each event. The mean and standard deviation of the :code:`word_count` are then computed by
the builtin :code:`stats` command.
"""
csv_file = Option(doc='''**Syntax:** **csv=***<path>*
**Description:** CSV file from which repeated random samples will be
drawn''',
name='csv',
require=True,
validate=validators.File())
duration = Option(doc='''**Syntax:** **duration=***<time-interval>*
**Description:** Duration of simulation''',
require=True,
validate=validators.Duration())
interval = Option(doc='''**Syntax:** **interval=***<time-interval>*
**Description:** Sampling interval''',
require=True,
validate=validators.Duration())
rate = Option(doc='''**Syntax:** **rate=***<expected-event-count>*
**Description:** Average event count during sampling `interval`''',
require=True,
validate=validators.Integer(1))
seed = Option(doc='''**Syntax:** **seed=***<string>*
**Description:** Value for initializing the random number generator '''
)
def generate(self):
if not self.records:
if self.seed is not None:
random.seed(self.seed)
self.records = [record for record in csv.DictReader(self.csv_file)]
self.lambda_value = 1.0 / (self.rate / float(self.interval))
duration = self.duration
while duration > 0:
count = int(round(random.expovariate(self.lambda_value)))
start_time = time.clock()
for record in random.sample(self.records, count):
yield record
interval = time.clock() - start_time
if interval < self.interval:
time.sleep(self.interval - interval)
duration -= max(interval, self.interval)
def __init__(self):
super(SimulateCommand, self).__init__()
self.lambda_value = None
self.records = None
class mispgetioc(ReportingCommand):
mispsrv = Option(require=False,
validate=validators.Match(
"mispsrv", r"^https?:\/\/[0-9a-zA-Z\.]+(?:\:\d+)?$"))
mispkey = Option(require=False,
validate=validators.Match("mispkey",
r"^[0-9a-zA-Z]{40}$"))
sslcheck = Option(require=False,
validate=validators.Match("sslcheck", r"^[yYnN01]$"))
eventid = Option(require=False,
validate=validators.Match("eventid", r"^[0-9]+$"))
last = Option(require=False,
validate=validators.Match("last", r"^[0-9]+[hdwm]$"))
onlyids = Option(require=False,
validate=validators.Match("onlyids", r"^[yYnN01]+$"))
getuuid = Option(require=False,
validate=validators.Match("getuuid", r"^[yYnN01]+$"))
getorg = Option(require=False,
validate=validators.Match("getuuid", r"^[yYnN01]+$"))
category = Option(require=False)
type = Option(require=False)
@Configuration()
def map(self, records):
self.logger.debug('mispgetioc.map')
yield {}
return
def reduce(self, records):
self.logger.debug('mispgetioc.reduce')
if self.sslcheck == None:
self.sslcheck = 'n'
# open misp.conf
config_file = '/opt/splunk/etc/apps/misp42splunk/local/misp.conf'
config = ConfigParser.RawConfigParser()
config.read(config_file)
# Generate args
my_args = {}
if self.mispsrv:
my_args['mispsrv'] = self.mispsrv
else:
my_args['mispsrv'] = config.get('mispsetup', 'mispsrv')
if self.mispkey:
my_args['mispkey'] = self.mispkey
else:
my_args['mispkey'] = config.get('mispsetup', 'mispkey')
if self.sslcheck:
if self.sslcheck == 'Y' or self.sslcheck == 'y' or self.sslcheck == '1':
my_args['sslcheck'] = True
else:
my_args['sslcheck'] = False
else:
my_args['sslcheck'] = config.getboolean('mispsetup', 'sslcheck')
if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1':
onlyids = True
else:
onlyids = False
if self.getuuid == 'Y' or self.getuuid == 'y' or self.getuuid == '1':
getuuid = True
else:
getuuid = False
if self.getorg == 'Y' or self.getorg == 'y' or self.getorg == '1':
getorg = True
else:
getorg = False
if self.eventid and self.last:
print('DEBUG Options "eventid" and "last" are mutually exclusive')
exit(2)
if self.eventid:
my_args['eventid'] = self.eventid
elif self.last:
my_args['last'] = self.last
else:
print('DEBUG Missing "eventid" or "last" argument')
exit(1)
_SPLUNK_PATH = '/opt/splunk'
_NEW_PYTHON_PATH = '/usr/bin/python3'
_SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH
my_process = _SPLUNK_PATH + '/etc/apps/misp42splunk/bin/pymisp_getioc.py'
# Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues
env = dict(os.environ)
del env['LD_LIBRARY_PATH']
FNULL = open(os.devnull, 'w')
p = subprocess.Popen([_NEW_PYTHON_PATH, my_process,
str(my_args)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=FNULL,
env=env)
output = p.communicate()[0]
results = {}
for v in eval(output):
# Do not display deleted attributes
if v['deleted'] == False:
# If specified, do not display attributes with the non-ids flag set to False
if onlyids == True and v['to_ids'] == False:
continue
if self.category != None and self.category != v['category']:
continue
if self.type != None and self.type != v['type']:
continue
if getuuid == True:
results['uuid'] = v['uuid']
if getorg == True:
results['orgc'] = v['orgc']
results['eventid'] = v['event_id']
results['value'] = v['value']
results['category'] = v['category']
results['type'] = v['type']
results['to_ids'] = str(v['to_ids'])
yield results
class SVCPostRestore(GeneratingCommand):
url = Option(require=True)
splunk_vc_name = Option(require=True)
app = Option(require=True)
type = Option(require=True)
obj_name = Option(require=True)
tag = Option(require=True)
restoreAsUser = Option(require=True)
scope = Option(require=True)
timeout = Option(require=True)
sslVerify = Option(require=False,
default=False,
validate=OrValidator(Boolean(), Filename()))
requestingAddress = Option(require=False, default=False)
def generate(self):
"""
The logic is:
If the requested savedsearch is owned by the current user, or the requesting user is an admin user, then
change the dispatch.ttl value of the saved search to the requested newttl value passed in
If the optional sharing level is not specified check for the savedsearch in the private / user context first
then app context
If the owner is specified look under the particular owner context, only someone with admin access can use this option
"""
urlregex = re.compile(
"^(?:https:\/\/)[\w0-9_\.-]+:8089/services/splunkversioncontrol_rest_restore$"
)
if urlregex.match(self.url):
url = self.url
else:
logger.error(
"Requested to post to remote url=%s but this did not match the regex"
% (self.url))
yield {
'result':
'Invalid url passed in, url must begin with https:// and would normally end in :8089/services/splunkversioncontrol_rest_restore, url=%s'
% (self.url)
}
return
body = {}
body['splunk_vc_name'] = self.splunk_vc_name
body['app'] = self.app
body['type'] = self.type
body['obj_name'] = self.obj_name
body['tag'] = self.tag
body['restoreAsUser'] = self.restoreAsUser
body['scope'] = self.scope
body['timeout'] = self.timeout
if self.requestingAddress:
body['requestingAddress'] = self.requestingAddress
logger.info("Attempting POST request to url=%s with body=\"%s\"" %
(url, body))
body[
'Authorization'] = 'Splunk ' + self._metadata.searchinfo.session_key
logger.debug("Using token %s" % (body['Authorization']))
attempt = requests.post(url, verify=self.sslVerify, data=body)
if attempt.status_code != 200:
logger.error(
"POST request failed with status_code=%s, reason=%s, text=%s on url=%s"
% (attempt.status_code, attempt.reason, attempt.text, url))
yield {
'result':
'Unknown failure, received a non-200 response code of %s on the url %s, reason %s, text result is %s'
% (attempt.status_code, url, attempt.reason, attempt.text)
}
return
else:
logger.debug("Received result of result=%s" % (attempt.text))
yield {'result': attempt.text}
class MispGetIocCommand(GeneratingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetioc misp_instance=<input> last=<int>(d|h|m)
| mispgetioc misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional",
"includeDecayScore": "optional",
"includeFullModel": "optional",
"decayingModel": "optional",
"excludeDecayed": "optional",
"score": "optional"
}
# status
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": forced to False,
"includeEventUuid": set to True,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": not managed,
"eventinfo": not managed,
"includeProposals": not managed
"includeDecayScore": not managed,
"includeFullModel": not managed,
"decayingModel": not managed,
"excludeDecayed": not managed,
"score": not managed
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:** MISP instance parameters
as described in local/misp42splunk_instances.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date.
**eventid**, **last** and **date** are mutually exclusive''',
require=False)
# Other params
add_description = Option(doc='''
**Syntax:** **add_description=***<1|y|Y|t|true|True
|0|n|N|f|false|False>*
**Description:**Boolean to return misp_description.''',
require=False,
validate=validators.Boolean())
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
expand_object = Option(doc='''
**Syntax:** **gexpand_object=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to have object attributes expanded (one per line).
By default, attributes of one object are displayed on same line.''',
require=False,
validate=validators.Boolean())
geteventtag = Option(doc='''
**Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean includeEventTags. By default only
attribute tag(s) are returned.''',
require=False,
validate=validators.Boolean())
getorg = Option(doc='''
**Syntax:** **getorg=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return the ID of the organisation that
created the event.''',
require=False,
validate=validators.Boolean())
getuuid = Option(doc='''
**Syntax:** **getuuid=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return attribute UUID.''',
require=False,
validate=validators.Boolean())
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search;
default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
output = Option(doc='''
**Syntax:** **output=***<default|rawy>*
**Description:**selection between the default behaviou or JSON output by attribute.''',
require=False,
validate=validators.Match("output", r"(default|raw)"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("page", r"^[0-9]+$"))
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes.''',
require=False,
validate=validators.Boolean())
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
to_ids = Option(doc='''
**Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to search only attributes with the flag
"to_ids" set to true.''',
require=False,
validate=validators.Boolean())
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
warning_list = Option(doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False,
validate=validators.Boolean())
def log_error(self, msg):
logging.error(msg)
def log_info(self, msg):
logging.info(msg)
def log_debug(self, msg):
logging.debug(msg)
def log_warn(self, msg):
logging.warning(msg)
def set_log_level(self):
logging.root
loglevel = logging_level('misp42splunk')
logging.root.setLevel(loglevel)
logging.error('[IO-101] logging level is set to %s', loglevel)
logging.error('[IO-102] PYTHON VERSION: ' + sys.version)
@staticmethod
def _record(serial_number,
time_stamp,
host,
attributes,
attribute_names,
encoder,
condensed=False):
if condensed is False:
raw = encoder.encode(attributes)
# Formulate record
fields = dict()
for f in attribute_names:
if f in attributes:
fields[f] = attributes[f]
if serial_number > 0:
fields['_serial'] = serial_number
fields['_time'] = time_stamp
if condensed is False:
fields['_raw'] = raw
fields['host'] = host
return fields
if condensed is False:
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('_raw', raw), ('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
else:
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
return record
def generate(self):
# loggging
self.set_log_level()
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
self.log_error(
'Missing "json_request", eventid", "last" or "date" argument')
raise Exception(
'Missing "json_request", "eventid", "last" or "date" argument')
elif mandatory_arg > 1:
self.log_error(
'Options "json_request", eventid", "last" and "date" are mutually exclusive'
)
raise Exception(
'Options "json_request", "eventid", "last" and "date" are mutually exclusive'
)
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
self.log_info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
self.log_info('Option "eventid" set with {}'.format(
json.dumps(body_dict['eventid'])))
elif self.last:
body_dict['last'] = self.last
self.log_info('Option "last" set with {}'.format(
body_dict['last']))
else:
body_dict['date'] = self.date.split()
self.log_info('Option "date" set with {}'.format(
json.dumps(body_dict['date'])))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
# manage to_ids and enforceWarninglist
# to avoid FP enforceWarninglist is set to True if
# to_ids is set to True (search criterion)
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.add_description is True:
my_args['add_desc'] = True
else:
my_args['add_desc'] = False
if self.expand_object is True:
my_args['expand'] = True
else:
my_args['expand'] = False
if self.getorg is True:
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.getuuid is True:
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
if self.output is not None:
my_args['output'] = self.output
else:
my_args['output'] = "default"
# add colums for each type in results
results = []
typelist = []
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
self.log_info(
"[IO301] INFO mispgetioc successful. url={}, HTTP status={}".
format(my_args['misp_url'], r.status_code))
else:
self.log_error(
"[IO302] ERROR mispgetioc failed. url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code, r.text))
raise Exception(
"[IO302] ERROR mispgetioc failed for url={} with HTTP Error={}. Check search.log for details"
.format(my_args['misp_url'], r.status_code))
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
# if raw output, returns JSON 1st-level keys as columns
if my_args['output'] == "raw":
if 'response' in response:
if 'Attribute' in response['response']:
attribute_names = list()
serial_number = 0
for a in response['response']['Attribute']:
yield MispGetIocCommand._record(
serial_number, a['timestamp'], my_args['host'], a,
attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
# default output: extract some values from JSON attributes
else:
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
v = {}
v['misp_category'] = str(a['category'])
v['misp_attribute_id'] = str(a['id'])
v['misp_event_id'] = str(a['event_id'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_to_ids'] = str(a['to_ids'])
v['misp_comment'] = str(a['comment'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
# include ID of the organisation that
# created the attribute if requested
if 'Event' in a:
v['misp_event_uuid'] = str(a['Event']['uuid'])
if my_args['getorg']:
v['misp_orgc_id'] = str(a['Event']['orgc_id'])
if my_args['add_desc'] is True:
v['misp_event_info'] = str(a['Event']['info'])
# include attribute UUID if requested
if my_args['getuuid']:
v['misp_attribute_uuid'] = str(a['uuid'])
# handle object and multivalue attributes
v['misp_object_id'] = str(a['object_id'])
if my_args['add_desc'] is True:
if int(a['object_id']) == 0:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) \
+ '" in category "' + str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) + ')'
else:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) + '" in category "' \
+ str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) \
+ ' - o' + str(a['object_id']) + ' )'
current_type = str(a['type'])
# combined: not part of an object
# AND multivalue attribute AND to be split
if int(a['object_id']) == 0 and '|' in current_type \
and my_args['pipe'] is True:
mv_type_list = current_type.split('|')
mv_value_list = str(a['value']).split('|')
left_v = v.copy()
left_v['misp_type'] = mv_type_list.pop()
left_v['misp_value'] = mv_value_list.pop()
results.append(left_v)
if left_v['misp_type'] not in typelist:
typelist.append(left_v['misp_type'])
right_v = v.copy()
right_v['misp_type'] = mv_type_list.pop()
right_v['misp_value'] = mv_value_list.pop()
results.append(right_v)
if right_v['misp_type'] not in typelist:
typelist.append(right_v['misp_type'])
else:
v['misp_type'] = current_type
v['misp_value'] = str(a['value'])
results.append(v)
if current_type not in typelist:
typelist.append(current_type)
self.log_info(json.dumps(typelist))
# consolidate attribute values under output table
if my_args['expand'] is True:
output_dict = {}
for r in results:
key = str(r['misp_event_id']) + \
'_' + str(r['misp_attribute_id'])
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace(
'|', '_p_')
v[misp_t] = []
if t == r['misp_type']:
v[misp_t].append(r['misp_value'])
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace(
'-', '_').replace('|', '_p_')
v[misp_t].append(r['misp_value']) # set value for type
misp_type = r['misp_type'] + '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = str(r['misp_value']) + '|' + str(
v['misp_value'])
v['misp_value'] = misp_value
output_dict[key] = dict(v)
else:
output_dict = {}
for r in results:
if int(r['misp_object_id']) == 0: # not an object
key = str(r['misp_event_id']) + \
'_' + str(r['misp_attribute_id'])
is_object_member = False
else: # this is a MISP object
key = str(r['misp_event_id']) \
+ '_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace(
'|', '_p_')
v[misp_t] = []
if t == r['misp_type']:
v[misp_t].append(r['misp_value'])
v['misp_to_ids'] = []
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'] = []
v['misp_category'].append(r['misp_category'])
v['misp_attribute_id'] = []
v['misp_attribute_id'].append(r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid'] = []
v['misp_attribute_uuid'].append(
r['misp_attribute_uuid'])
if my_args['add_desc'] is True:
v['misp_description'] = []
v['misp_description'].append(r['misp_description'])
v['misp_type'] = []
v['misp_type'].append(r['misp_type'])
v['misp_value'] = []
v['misp_value'].append(str(r['misp_value']))
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace(
'-', '_').replace('|', '_p_')
v[misp_t].append(r['misp_value']) # set value for type
v['misp_to_ids'].append(r['misp_to_ids'])
v['misp_category'].append(r['misp_category'])
tag_list = v['misp_tag']
for tag in r['misp_tag']:
if tag not in tag_list:
tag_list.append(tag)
v['misp_tag'] = tag_list
if my_args['add_desc'] is True:
description = v['misp_description']
if r['misp_description'] not in description:
description.append(r['misp_description'])
v['misp_description'] = description
if is_object_member is False:
if r['misp_attribute_id'] not in v[
'misp_attribute_id']:
v['misp_attribute_id'].append(
r['misp_attribute_id'])
if my_args['getuuid'] is True:
if r['misp_attribute_uuid'] not in v[
'misp_attribute_uuid']:
v['misp_attribute_uuid'].append(
r['misp_attribute_uuid'])
misp_type = []
misp_type.append(r['misp_type'] + '|' +
v['misp_type'][0])
v['misp_type'] = misp_type
misp_value = []
misp_value.append(
str(r['misp_value']) + '|' +
str(v['misp_value'][0]))
v['misp_value'] = misp_value
else:
v['misp_attribute_id'].append(
r['misp_attribute_id'])
if my_args['getuuid'] is True:
v['misp_attribute_uuid'].append(
r['misp_attribute_uuid'])
v['misp_type'].append(r['misp_type'])
v['misp_value'].append(r['misp_value'])
output_dict[key] = dict(v)
# return output table
attribute_names = list()
init_attribute_names = True
serial_number = 0
for v in output_dict.values():
if init_attribute_names is True:
for key in v.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
yield MispGetIocCommand._record(serial_number,
v['misp_timestamp'],
my_args['host'], v,
attribute_names, encoder, True)
serial_number += 1
GeneratingCommand.flush
class TtrCommand(EventingCommand):
""" Computes the time-to-repair (TTR) of a set of fields.
##Syntax
.. code-block::
ttr ttrfield=<field> resultfield=<field> passvalue=<value> <field-list>
##Example
..code-block::
ttr ttrfield=ttr resultfield=result passvalue=SUCCESS instance ci stage release
"""
ttrfield = Option(doc='''
**Syntax:** **ttrfield=***<fieldname>*
**Description:** Name of the field that will hold the computed TTR
''',
require=True,
validate=validators.Fieldname())
resultfield = Option(doc='''
**Syntax:** **resultfield=***<fieldname>*
**Description:** Name of the field that holds the result of the event
''',
require=True,
validate=validators.Fieldname())
passvalue = Option(doc='''
**Syntax:** **passvalue=***<value>*
**Description:** Value that *resultfield* needs to be to consider an event to be passing
''',
require=True)
timefield = Option(doc='''
''',
default='_time',
require=False,
validate=validators.Fieldname())
lookbeyondboundary = Option(doc='''
''',
default=False,
require=False,
validate=validators.Boolean())
def transform(self, records):
earliest_time = self.metadata.searchinfo.earliest_time
by_fieldnames = self.fieldnames
self.logger.debug(
'ttrfield: %s, resultfield: %s, passvalue: %s, timefield: %s, lookbeyondboundary: %s, fieldnames: %s, earliest: %s',
self.ttrfield, self.resultfield, self.passvalue, self.timefield,
self.lookbeyondboundary, by_fieldnames, earliest_time)
# get all events needed for TTR calculation
if self.lookbeyondboundary:
events_for_ttr = self.get_events_for_ttr(records, earliest_time,
by_fieldnames)
else:
events_for_ttr = list(records)
# calculate TTR
sort_by_args = list(by_fieldnames)
sort_by_args.append(self.timefield)
events_for_ttr.sort(key=operator.itemgetter(*sort_by_args))
metadata = {}
self.init_metadata(metadata)
last_by_fields = []
for event in events_for_ttr:
by_fields = self.get_by_fields(by_fieldnames, event)
if by_fields != last_by_fields:
self.init_metadata(metadata)
ttr = self.calc_single_ttr(float(event[self.timefield]),
event[self.resultfield], metadata)
event[self.ttrfield] = ttr
last_by_fields = by_fields
final_events = []
events_for_ttr.sort(key=operator.itemgetter(self.timefield),
reverse=True)
for event in events_for_ttr:
timestamp = float(event[self.timefield])
if self.lookbeyondboundary and timestamp < earliest_time:
# throw away events before earliest_time
break
final_events.append(event)
return final_events
def get_events_for_ttr(self, records, earliest_time, by_fieldnames):
events_for_ttr = []
metadata = {}
within_time_window = True
last_timestamp = -1
for record in records:
timestamp = float(record[self.timefield])
# make sure timestamps are decreasing
if last_timestamp != -1 and timestamp > last_timestamp:
raise Exception('events aren\'t in decreasing time order!')
last_timestamp = timestamp
result = record[self.resultfield]
by_fields = self.get_by_fields(by_fieldnames, record)
key = self.gen_ttr_prep_metadata_key(by_fields)
if timestamp >= earliest_time:
events_for_ttr.append(record)
if key not in metadata:
metadata[key] = {
'last_result': None,
'staged_events': [],
'has_success': False
}
combo = metadata[key]
combo['last_result'] = result
if result == self.passvalue:
combo['has_success'] = True
else:
if within_time_window:
for key in metadata.keys():
combo = metadata[key]
if not combo['has_success']:
del metadata[key]
within_time_window = False
if not metadata:
break
if key in metadata:
combo = metadata[key]
combo['staged_events'].append(record)
if result == self.passvalue:
events_for_ttr += combo['staged_events']
del metadata[key]
return events_for_ttr
@staticmethod
def gen_ttr_prep_metadata_key(by_fields):
key = ''
for by_field in by_fields:
key += by_field + ':'
return key[:-1]
@staticmethod
def init_metadata(metadata):
metadata['last_result'] = ''
metadata['first_failure_time'] = 0
metadata['encountered_success'] = False
def calc_single_ttr(self, timestamp, result, metadata):
ttr = None
last_result = metadata['last_result']
first_failure_time = metadata['first_failure_time']
encountered_success = metadata['encountered_success']
if not encountered_success and self.passing(result):
metadata['encountered_success'] = True
if (self.passing(last_result) or last_result
== '') and not self.passing(result) and encountered_success:
metadata['first_failure_time'] = timestamp
elif not self.passing(last_result) and self.passing(
result) and last_result != '' and first_failure_time != 0:
ttr = timestamp - first_failure_time
metadata['last_result'] = result
return ttr
def passing(self, result):
return result == self.passvalue
@staticmethod
def get_by_fields(by_fieldnames, event):
return [event[x] for x in by_fieldnames]
class MACFormatCommand(StreamingCommand):
""" Convert a given MAC address field to specified format.
##Syntax
.. code-block::
| macformat input=field-list output=field-list format=[cisco|dash|ieee|none]
## Description
Convert the fields in the `input` field list to the ones in the `output` list; Both lists are
optional. The `input` list defaults to `macaddress`. The`output` list is filled with fields in
the `input` list it the `output` list is shorter than the `input`.
The `format` option is one of [cisco|dash|ieee|none]. The default is `none`.
Raises a ValueError exception if the MAC address is invalid.
"""
format = Option(doc='''
**Syntax:** **format=**`[cisco|dash|ieee|none]`
**Description:** Format of the output MAC address. Defaults to `none`.''',
require=False,
validate=validators.Set('cisco', 'dash', 'ieee', 'none'))
inputs = Option(doc='''
**Syntax:** **inputs=***<field-list>*
**Description:** A comma-delimited list of input fields to convert. Defaults to `macaddress`.''',
require=False,
validate=validators.List())
outputs = Option(doc='''
**Syntax:** **outputs=***<field-list>*
**Description:** A comma-delimited list of fields for the results. Defaults to `inputs`.''',
require=False,
validate=validators.List())
def prepare(self):
""" Prepare the options.
:return: :const:`None`
:rtype: NoneType
"""
self.toform = globals()['_' + (self.format or self.def_format)]
inputs = self.inputs
if inputs is None:
self.inputs = inputs = self.def_inputs
outputs = self.outputs
if outputs is None:
outputs = inputs
elif len(outputs) < len(inputs):
outputs += inputs[len(outputs):]
self.outputs = outputs
self.logger.debug(
'MACFormatCommand.prepare: inputs = %s, outputs = %s', self.inputs,
outputs)
def stream(self, records):
toform = self.toform
inputs = self.inputs
outputs = self.outputs
if outputs is None:
outputs = inputs
elif len(outputs) < len(inputs):
outputs += inputs[len(outputs):]
for record in records:
self.logger.debug('MACFormatCommand: record = %s', record)
for i in range(len(inputs)):
mac = record.get(inputs[i])
if mac != None:
try:
record[outputs[i]] = toform(mac)
except Exception as err:
record[outputs[i]] = mac
self.logger.error('(input=%s) %s', inputs[i],
err.message)
yield record
def __init__(self):
StreamingCommand.__init__(self)
appdir = path.dirname(path.dirname(__file__))
defconfpath = path.join(appdir, "default", "app.conf")
defconf = cli.readConfFile(defconfpath).get('macformat') or {}
localconfpath = path.join(appdir, "local", "app.conf")
localconf = (cli.readConfFile(localconfpath).get('macformat')
or {}) if path.exists(localconfpath) else {}
self.def_format = localconf.get('format') or defconf.get(
'format') or 'none'
inputs = localconf.get('inputs') or defconf.get('inputs')
self.def_inputs = re.split('[\s,]',
inputs) if inputs else ['macaddress']
class mispgetioc(ReportingCommand):
mispsrv = Option(require=False,
validate=validators.Match(
"mispsrv",
r"^https?:\/\/[0-9a-zA-Z\-\.]+(?:\:\d+)?$"))
mispkey = Option(require=False,
validate=validators.Match("mispkey",
r"^[0-9a-zA-Z]{40}$"))
sslcheck = Option(require=False,
validate=validators.Match("sslcheck", r"^[yYnN01]$"))
eventid = Option(require=False,
validate=validators.Match("eventid", r"^[0-9]+$"))
last = Option(require=False,
validate=validators.Match("last", r"^[0-9]+[hdwm]$"))
onlyids = Option(require=False,
validate=validators.Match("onlyids", r"^[yYnN01]+$"))
getuuid = Option(require=False,
validate=validators.Match("getuuid", r"^[yYnN01]+$"))
getorg = Option(require=False,
validate=validators.Match("getorg", r"^[yYnN01]+$"))
category = Option(require=False)
type = Option(require=False)
tags = Option(require=False)
not_tags = Option(require=False)
@Configuration()
def map(self, records):
self.logger.debug('mispgetioc.map')
yield {}
return
def reduce(self, records):
self.logger.debug('mispgetioc.reduce')
if self.sslcheck == None:
self.sslcheck = 'n'
_SPLUNK_PATH = os.environ['SPLUNK_HOME']
# open misp.conf
config_file = _SPLUNK_PATH + '/etc/apps/misp42splunk/local/misp.conf'
mispconf = ConfigParser.RawConfigParser()
mispconf.read(config_file)
# Generate args
my_args = {}
#MISP instance parameters
if self.mispsrv:
my_args['mispsrv'] = self.mispsrv
else:
my_args['mispsrv'] = mispconf.get('mispsetup', 'mispsrv')
if self.mispkey:
my_args['mispkey'] = self.mispkey
else:
my_args['mispkey'] = mispconf.get('mispsetup', 'mispkey')
if self.sslcheck:
if self.sslcheck == 'Y' or self.sslcheck == 'y' or self.sslcheck == '1':
my_args['sslcheck'] = True
else:
my_args['sslcheck'] = False
else:
my_args['sslcheck'] = mispconf.getboolean('mispsetup', 'sslcheck')
#Search parameters: boolean and filter
if self.onlyids == 'Y' or self.onlyids == 'y' or self.onlyids == '1':
my_args['onlyids'] = True
else:
my_args['onlyids'] = False
if self.getuuid == 'Y' or self.getuuid == 'y' or self.getuuid == '1':
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.getorg == 'Y' or self.getorg == 'y' or self.getorg == '1':
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.category != None:
my_args['category'] = self.category
else:
my_args['category'] = None
if self.type != None:
my_args['type'] = self.type
else:
my_args['type'] = None
if self.tags != None:
my_args['tags'] = self.tags
else:
my_args['tags'] = None
if self.not_tags != None:
my_args['not_tags'] = self.not_tags
else:
my_args['not_tags'] = None
#check that ONE of mandatory fields is present
if self.eventid and self.last:
print('DEBUG Options "eventid" and "last" are mutually exclusive')
exit(2)
elif self.eventid:
my_args['eventid'] = self.eventid
elif self.last:
my_args['last'] = self.last
else:
print('DEBUG Missing "eventid" or "last" argument')
exit(1)
#path to main components either use default values or set ones
if mispconf.has_option('mispsetup', 'P3_PATH'):
_NEW_PYTHON_PATH = mispconf.get('mispsetup', 'P3_PATH')
else:
_NEW_PYTHON_PATH = '/usr/bin/python3'
if mispconf.has_option('mispsetup', 'TMP_PATH'):
_TMP_PATH = mispconf.get('mispsetup', 'TMP_PATH')
else:
_TMP_PATH = '/tmp'
_SPLUNK_PYTHON_PATH = os.environ['PYTHONPATH']
os.environ['PYTHONPATH'] = _NEW_PYTHON_PATH
my_process = _SPLUNK_PATH + '/etc/apps/misp42splunk/bin/pymisp_getioc.py'
# Remove LD_LIBRARY_PATH from the environment (otherwise, we will face some SSL issues
env = dict(os.environ)
del env['LD_LIBRARY_PATH']
FNULL = open(os.devnull, 'w')
#use pickle
swap_file = _TMP_PATH + '/mispgetioc_config'
pickle.dump(my_args, open(swap_file, "wb"), protocol=2)
env_file = _TMP_PATH + '/mispgetioc_env'
pickle.dump(env, open(env_file, "wb"), protocol=2)
p = subprocess.Popen([_NEW_PYTHON_PATH, my_process, swap_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env)
stdout, stderr = p.communicate()
# if stderr:
# print('DEBUG error in pymisp_getioc.py')
# exit(1)
output = {}
output = pickle.load(open(swap_file, "rb"))
if output:
for v in output:
yield v
class MakeHolidays(StreamingCommand):
"""
A wrapper for python-holidays providing holidays, business days, and business holidays when presented a _time (or specified) field.
##Syntax
.. code-block::
* | holidays [timefield=<field>] [country=<string>] [province=<string>] [state=<string>] [business_days=<comma_sep_int>] [custom_holiday=<date>]
##Description
A wrapper for for the python library python-holidays (https://github.com/dr-prodigy/python-holidays)
to enrich existing data with timestamps to know if the given timestamp is a holiday, business day,
or business holiday (a holiday that occurs on a normal business day). Default settings: country is
set to US and timestamp is expected in the `_time` field.
##Example
.. code-block::
* | holidays
"""
timefield = Option(
default='_time',
doc='''
**Syntax:** **timefield=***<fieldname>*
**Description:** The field containing the timestamp in unix epoch, normally this is `_time` which is the default if not set.''',
validate=validators.Fieldname())
country = Option(
default='US',
doc='''
**Syntax:** **country=***<string>*
**Description:** Country code string from https://github.com/dr-prodigy/python-holidays/blob/master/README.rst, defaults to US''',
)
province = Option(
default=None,
doc='''
**Syntax:** **province=***<string>*
**Description:** Province code string from https://github.com/dr-prodigy/python-holidays/blob/master/README.rst, defaults to None''',
)
state = Option(
default=None,
doc='''
**Syntax:** **state=***<string>*
**Description:** State code string from https://github.com/dr-prodigy/python-holidays/blob/master/README.rst, defaults to None''',
)
business_days = Option(
default=None,
doc='''
**Syntax:** **business_days=***<comma-sep-int>*
**Description:** Defaults to Monday-Friday (1-5) but if different days are business days they can be specified using numbers (Saturday and Sunday are 6 and 7 respectively) ''',
)
custom_holiday = Option(
default=None,
doc='''
**Syntax:** **custom_holiday=***<date>*
**Description:** Ability to supply a date for a non-standard holiday.''',
)
def stream(self, records):
holiday_list = holidays.CountryHoliday(
self.country,
self.province,
self.state
)
if self.custom_holiday:
holiday_list.append([self.custom_holiday])
if self.business_days:
if re.search(r'(\d+,?)+',self.business_days):
working_days = [ int(i) for i in self.business_days.split(',')]
else:
working_days = [1,2,3,4,5]
for record in records:
converted_time = time.localtime(float(record[self.timefield]))
converted_date = datetime.fromtimestamp(mktime(converted_time))
record['is_holiday'] = converted_date in holiday_list
record['holiday_name'] = holiday_list.get(converted_date)
record['is_business_day'] = int(strftime('%u',converted_time)) in working_days
record['is_business_holiday'] = int(strftime('%u',converted_time)) in working_days and converted_date in holiday_list
yield record
class GNEnrichCommand(EventingCommand):
"""
gnenrich - Transforming Command.
Transforming command that enriches Splunk search events with the context information of the IP addresses
present as values in the IP field passed in ip_field parameter.
Data pulled from: /v2/noise/context/{ip}
**Syntax**::
`index=firewall | gnenrich ip_field="ip"
**Description**::
The `gnenrich` command uses the IP represented by IP field in `ip_field` to return
context information using method :method:`quick` from GreyNoise Python SDK.
"""
ip_field = Option(doc='''
**Syntax:** **ip_field=***<ip_field>*
**Description:** Name of the field representing IP address in Splunk events''',
name='ip_field',
require=True)
api_validation_flag = False
def transform(self, records):
"""Method that processes and yield event records to the Splunk events pipeline."""
# Setup logger
logger = utility.setup_logger(
session_key=self._metadata.searchinfo.session_key,
log_context=self._metadata.searchinfo.command)
if self.search_results_info and not self.metadata.preview:
EVENTS_PER_CHUNK = 1
THREADS = 3
USE_CACHE = False
ip_field = self.ip_field
logger.info(
"Started retrieving context information for the IP addresses present in field: {}"
.format(str(ip_field)))
try:
# Strip the spaces from the parameter value if given
if ip_field:
ip_field = ip_field.strip()
# Validating the given parameters
try:
ip_field = validator.Fieldname(
option_name='ip_field').validate(ip_field)
except ValueError as e:
# Validator will throw ValueError with error message when the parameters are not proper
logger.error(str(e))
self.write_error(str(e))
exit(1)
try:
message = ''
api_key = utility.get_api_key(
self._metadata.searchinfo.session_key, logger=logger)
except APIKeyNotFoundError as e:
message = str(e)
except HTTPError as e:
message = str(e)
if message:
self.write_error(message)
logger.error(
"Error occured while retrieving API key, Error: {}".
format(message))
exit(1)
# API key validation
if not self.api_validation_flag:
api_key_validation, message = utility.validate_api_key(
api_key, logger)
logger.debug(
"API validation status: {}, message: {}".format(
api_key_validation, str(message)))
self.api_validation_flag = True
if not api_key_validation:
logger.info(message)
self.write_error(message)
exit(1)
# divide the records in the form of dict of tuples having chunk_index as key
# {<index>: (<records>, <All the ips in records>)}
chunk_dict = event_generator.batch(records,
ip_field,
EVENTS_PER_CHUNK,
logger,
optimize_requests=False)
logger.debug("Successfully divided events into chunks")
# This means there are only 1000 or below IPs to call in the entire bunch of records
# Use one thread with single thread with caching mechanism enabled for the chunk
if len(chunk_dict) == 1:
logger.debug(
"Less then 1000 distinct IPs are present, optimizing the IP requests call to GreyNoise API..."
)
THREADS = 1
USE_CACHE = True
# Opting timout 120 seconds for the requests
api_client = GreyNoise(api_key=api_key,
timeout=120,
use_cache=USE_CACHE,
integration_name=INTEGRATION_NAME)
if len(chunk_dict) > 0:
for event in event_generator.get_all_events(
self._metadata.searchinfo.session_key,
api_client,
'enrich',
ip_field,
chunk_dict,
logger,
threads=THREADS):
yield event
logger.info(
"Successfully sent all the results to the Splunk")
else:
logger.info(
"No events found, please increase the search timespan to have more search results."
)
except Exception:
logger.info(
"Exception occured while getting context information for events events, Error: {}"
.format(traceback.format_exc()))
self.write_error(
"Exception occured while enriching events with the context information of IP addresses. "
"See greynoise_main.log for more details.")
def __init__(self):
"""Initialize custom command class."""
super(GNEnrichCommand, self).__init__()
class StreamFilterCommand(StreamingCommand):
""" Returns a field with a list of non-overlapping matches to a regular expression in a set of fields.
##Syntax
.. code-block::
StreamFilterCommand fieldname=<field> pattern=<field containing regex pattern> <field-list>
##Description
Returns the non-overlapping matches to the regular expression contained in the field specified by `pattern`
The result is stored in the field specified by `fieldname`. If `fieldname` exists, its value
is replaced. If `fieldname` does not exist, it is created. Event records are otherwise passed through to the next
pipeline processor unmodified.
##Example
Return the regular expression matches in the `text` field (field named text) of each tweet in tweets.csv and store the result in `word_count`.
.. code-block::
| inputlookup tweets | eval pattern="\\w+" | streamfilter fieldname=word_count pattern=pattern text
"""
fieldname = Option(doc='''
**Syntax:** **fieldname=***<fieldname>*
**Description:** Name of the field that will hold the match count''',
require=True,
validate=validators.Fieldname())
pattern = Option(doc='''
**Syntax:** **pattern=***<fieldname>*
**Description:** Field name containign the regular expression pattern to match''',
require=True,
validate=validators.Fieldname())
#Filtering function created so we can handle multi-value pattern fields
def thefilter(self, record, pattern):
values = ""
for fieldname in self.fieldnames:
#multivalue fields come through as a list, iterate through the list and run the regex against each entry
#in the multivalued field
if not fieldname in record:
continue
if isinstance(record[fieldname], list):
for aRecord in record[fieldname]:
matches = pattern.findall(
six.text_type(aRecord.decode("utf-8")))
for match in matches:
values = values + " " + match
else:
matches = pattern.findall(
six.text_type(record[fieldname].decode("utf-8")))
for match in matches:
values = values + " " + match
return values
#stream function to work on each event which may or may not be multi-valued
def stream(self, records):
self.logger.debug('StreamFilterCommand: %s', self) # logs command line
for record in records:
values = ""
pattern = self.pattern
if pattern not in record:
self.logger.warn(
"StreamFilterCommand: pattern field is %s but cannot find this field"
% (pattern), self)
sys.exit(-1)
if isinstance(record[pattern], list):
for aPattern in record[pattern]:
pattern = re.compile(aPattern)
values = values + self.thefilter(record, pattern)
else:
pattern = re.compile(record[pattern])
values = values + self.thefilter(record, pattern)
record[self.fieldname] = values
yield record
class Outliers(OptionRemoteStreamingCommand):
threshold = Option(require=False,
default=0.01,
validate=FloatValidator(minimum=0, maximum=1))
# One-Class SVM arguments
kernel = Option(require=False, default='rbf')
degree = Option(require=False,
default=3,
validate=validators.Integer(minimum=1))
gamma = Option(require=False,
default=0.1,
validate=FloatValidator(minimum=0, maximum=1))
coef0 = Option(require=False, default=0.0, validate=FloatValidator())
# Covariance Estimator arguments
support_fraction = Option(require=False,
validate=FloatValidator(minimum=0, maximum=1))
showmah = Option(require=False,
default=False,
validate=validators.Boolean())
classifier = Option(require=False, default='one_class_svm')
code = """
import os, sys, numbers, math
import numpy as np
import scipy.sparse as sp
from scipy import stats
from sklearn import svm
from sklearn.covariance import EllipticEnvelope
from sklearn.feature_extraction.text import HashingVectorizer
if __name__ == '__channelexec__':
args = channel.receive()
fraction = 1 - args['threshold']
fields = args.get('fieldnames') or ['_raw']
by_fields = None
try:
by_index = fields.index("by")
by_fields = fields[(by_index+1):]
fields = fields[:by_index]
except:
pass
classifier = args['classifier']
svm_args = {
'nu': 0.95 * fraction + 0.05,
'kernel': args['kernel'],
'degree': args['degree'],
'gamma': args['gamma'],
'coef0': args['coef0']
}
rc_args = {
'contamination': args['threshold'],
'support_fraction': args['support_fraction']
}
classifiers = {
'one_class_svm': svm.OneClassSVM(**svm_args),
'covariance_estimator': EllipticEnvelope(**rc_args)
}
records = []
for record in channel:
if not record:
break
records.append(record)
if records:
vectorizer = HashingVectorizer(ngram_range=(1,3), n_features=int(math.sqrt(len(records))))
X = sp.lil_matrix((len(records),vectorizer.n_features))
for i, record in enumerate(records):
nums = []
strs = []
for field in fields:
if isinstance(record.get(field), numbers.Number):
nums.append(record[field])
else:
strs.append(str(record.get(field) or ""))
if nums:
X[i] = np.array(nums, dtype=np.float64)
elif strs:
X[i] = vectorizer.transform([" ".join(strs)])
X = X.toarray()
y_pred = None
mah = None
clf = classifiers.get(classifier)
if clf:
try:
clf.fit(X)
y = clf.decision_function(X).ravel()
threshold = stats.scoreatpercentile(y, 100 * fraction)
y_pred = y > threshold
if classifier == 'covariance_estimator' and args['showmah']:
mah = clf.mahalanobis(X)
except ValueError:
y_pred = np.zeros((X.shape[0]))
for i, y in enumerate(y_pred):
if y:
record = records[i]
if mah is not None:
record['mahalanobis'] = mah[i].item()
channel.send(record)
else:
channel.send({ "error": "Incorrect classifier specified %s" % classifier })
"""
def __dir__(self):
return [
'threshold', 'kernel', 'degree', 'gamma', 'coef0',
'support_fraction', 'showmah', 'classifier'
]
class MispGetEventCommand(GeneratingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| MispGetEventCommand misp_instance=<input> last=<int>(d|h|m)
| MispGetEventCommand misp_instance=<input> event=<id1>(,<id2>,...)
| MispGetEventCommand misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tag": "optional",
"tags": "optional",
"searchall": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"metadata": "optional",
"uuid": "optional",
"published": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"sgReferenceOnly": "optional",
"eventinfo": "optional",
"excludeLocalTags": "optional"
}
# status
"tag": "optional",
"searchall": "optional",
"metadata": "optional",
"published": "optional",
"sgReferenceOnly": "optional",
"eventinfo": "optional",
"excludeLocalTags": "optional"
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": not managed,
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as described
in local/misp42splunk_instances.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date. **eventid**, **last**
and **date** are mutually exclusive''',
require=False)
# Other params
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
getioc = Option(doc='''
**Syntax:** **getioc=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return the list of attributes
together with the event.''',
require=False,
validate=validators.Boolean())
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; default 1000.
0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
output = Option(doc='''
**Syntax:** **output=***<default|rawy>*
**Description:**selection between a tabular or JSON output.''',
require=False,
validate=validators.Match("output", r"(default|raw)"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("page", r"^[0-9]+$"))
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes.''',
require=False,
validate=validators.Boolean())
published = Option(doc='''
**Syntax:** **published=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**select only published events (for option from to) .''',
require=False,
validate=validators.Boolean())
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
warning_list = Option(doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False,
validate=validators.Boolean())
@staticmethod
def _record(serial_number,
time_stamp,
host,
attributes,
attribute_names,
encoder,
condensed=False):
if condensed is False:
raw = encoder.encode(attributes)
# Formulate record
fields = dict()
for f in attribute_names:
if f in attributes:
fields[f] = attributes[f]
if serial_number > 0:
fields['_serial'] = serial_number
fields['_time'] = time_stamp
if condensed is False:
fields['_raw'] = raw
fields['host'] = host
return fields
if condensed is False:
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('_raw', raw), ('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
else:
record = OrderedDict(
chain((('_serial', serial_number), ('_time', time_stamp),
('host', host)),
map(lambda name: (name, fields.get(name, '')),
attribute_names)))
return record
def generate(self):
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
if self.published is True:
body_dict['published'] = True
elif self.published is False:
body_dict['published'] = False
# Search parameters: boolean and filter
# manage enforceWarninglist
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getioc is True:
getioc = True
else:
getioc = False
if self.pipesplit is True:
pipesplit = True
else:
pipesplit = False
if self.output is not None:
output = self.output
else:
output = "default"
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.error('MispGetEventCommand request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
logging.info("[EV301] INFO mispgetevent successful. "
"url={}, HTTP status={}".format(
my_args['misp_url'], r.status_code))
else:
logging.error("[EV302] ERROR mispgetevent failed. "
"url={}, data={}, HTTP Error={}, content={}".format(
my_args['misp_url'], body, r.status_code,
r.text))
raise Exception(
"[EV302] ERROR mispgetevent failed. "
"url={}, data={}, HTTP Error={}, content={}".format(
my_args['misp_url'], body, r.status_code, r.text))
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if output == "raw":
if 'response' in response:
attribute_names = list()
serial_number = 0
for r_item in response['response']:
if 'Event' in r_item:
for e in r_item.values():
yield MispGetEventCommand._record(
serial_number, e['timestamp'], my_args['host'],
e, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
# build output table and list of types
events = []
typelist = []
column_list = format_output_table(response, events, typelist,
getioc, pipesplit)
logging.info('typelist containss %s values', str(len(typelist)))
logging.debug('typelist is %s', json.dumps(typelist))
logging.info('results contains %s records', str(len(events)))
if getioc is False:
attribute_names = list()
init_attribute_names = True
serial_number = 0
for e in events:
if init_attribute_names is True:
for key in e.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
yield MispGetEventCommand._record(serial_number,
e['misp_timestamp'],
my_args['host'], e,
attribute_names, encoder,
True)
serial_number += 1
GeneratingCommand.flush
else:
output_dict = {}
for e in events:
if 'Attribute' in e:
for a in e['Attribute']:
if int(a['misp_object_id']) == 0: # not an object
key = str(e['misp_event_id']) + '_' \
+ str(a['misp_attribute_id'])
is_object_member = False
else: # this is a MISP object
key = str(e['misp_event_id']) + \
'_object_' + str(a['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = init_misp_output(e, a, column_list)
for t in typelist:
misp_t = 'misp_' \
+ t.replace('-', '_')\
.replace('|', '_p_')
v[misp_t] = []
if t == a['misp_type']:
v[misp_t].append(a['misp_value'])
if is_object_member is True:
v['misp_type'] = v['misp_object_name']
v['misp_value'] = v['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + a['misp_type']\
.replace('-', '_').replace('|', '_p_')
v[misp_t].append(a['misp_value'])
if a['misp_to_ids'] not in v['misp_to_ids']:
v['misp_to_ids'].append(a['misp_to_ids'])
if a['misp_category'] not in v[
'misp_category']:
v['misp_category'].append(
a['misp_category'])
v['misp_attribute_uuid']\
.append(a['misp_attribute_uuid'])
v['misp_attribute_id']\
.append(a['misp_attribute_id'])
if a['misp_attribute_tag'] is not None:
a_tag = v['misp_attribute_tag']
for t in a['misp_attribute_tag']:
if t not in a_tag:
a_tag.append(t)
v['misp_attribute_tag'] = a_tag
if a['misp_comment'] not in v['misp_comment']:
v['misp_comment'].append(a['misp_comment'])
if is_object_member is False:
misp_type = a['misp_type'] \
+ '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = a['misp_value'] + \
'|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
if output_dict is not None:
attribute_names = list()
init_attribute_names = True
serial_number = 0
for v in output_dict.values():
if init_attribute_names is True:
for key in v.keys():
if key not in attribute_names:
attribute_names.append(key)
attribute_names.sort()
init_attribute_names = False
yield MispGetEventCommand._record(
serial_number, v['misp_timestamp'],
my_args['host'], v, attribute_names, encoder, True)
serial_number += 1
GeneratingCommand.flush
class MispCollectCommand(GeneratingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetioc misp_instance=<input> last=<int>(d|h|m)
| mispgetioc misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional",
"includeDecayScore": "optional",
"includeFullModel": "optional",
"decayingModel": "optional",
"excludeDecayed": "optional",
"score": "optional"
}
# status
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": forced to False,
"includeEventUuid": set to True,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": not managed,
"eventinfo": not managed,
"includeProposals": not managed
"includeDecayScore": not managed,
"includeFullModel": not managed,
"decayingModel": not managed,
"excludeDecayed": not managed,
"score": not managed
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(
doc='''
**Syntax:** **misp_instance=instance_name*
**Description:** MISP instance parameters
as described in local/misp42splunk_instances.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(
doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(
doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False, validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(
doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False, validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(
doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date.
**eventid**, **last** and **date** are mutually exclusive''',
require=False)
# Other params
category = Option(
doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
endpoint = Option(
doc='''
**Syntax:** **endpoint=***<events|attributes>*
**Description:**selection of MISP API restSearch endpoint.
**default**: /attributes/restSearch''',
require=False, validate=validators.Match("endpoint", r"(events|attributes)"))
geteventtag = Option(
doc='''
**Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean includeEventTags. By default only
attribute tag(s) are returned.''',
require=False, validate=validators.Boolean())
keep_related = Option(
doc='''
**Syntax:** **keep_related=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to keep related events.
default is to drop RelatedEvents to reduce volume.''',
require=False, validate=validators.Boolean())
limit = Option(
doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search;
default 1000. 0 = no pagination.''',
require=False, validate=validators.Match("limit", r"^[0-9]+$"))
not_tags = Option(
doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
page = Option(
doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False, validate=validators.Match("page", r"^[0-9]+$"))
tags = Option(
doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
to_ids = Option(
doc='''
**Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to search only attributes with the flag
"to_ids" set to true.''',
require=False, validate=validators.Boolean())
type = Option(
doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
warning_list = Option(
doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False, validate=validators.Boolean())
def log_error(self, msg):
logging.error(msg)
def log_info(self, msg):
logging.info(msg)
def log_debug(self, msg):
logging.debug(msg)
def log_warn(self, msg):
logging.warning(msg)
def set_log_level(self):
logging.root
loglevel = logging_level('misp42splunk')
logging.root.setLevel(loglevel)
logging.error('[CO-101] logging level is set to %s', loglevel)
logging.error('[CO-102] PYTHON VERSION: ' + sys.version)
@staticmethod
def _record(serial_number, time_stamp, host, attributes, attribute_names, encoder):
raw = encoder.encode(attributes)
# Formulate record
fields = dict()
for f in attribute_names:
if f in attributes:
fields[f] = attributes[f]
if serial_number > 0:
fields['_serial'] = serial_number
fields['_time'] = time_stamp
fields['_raw'] = raw
fields['host'] = host
return fields
record = OrderedDict(chain(
(('_serial', serial_number), ('_time', time_stamp),
('_raw', raw), ('host', host)),
map(lambda name: (name, fields.get(name, '')), attribute_names)))
return record
def generate(self):
# loggging
self.set_log_level()
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception("Sorry, no configuration for misp_instance={}".format(misp_instance))
my_args['host'] = my_args['misp_url'].replace('https://', '')
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
raise Exception('Missing "json_request", "eventid", "last" or "date" argument')
elif mandatory_arg > 1:
raise Exception('Options "json_request", "eventid", "last" and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
self.log_info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
self.log_info('Option "eventid" set with {}'
.format(json.dumps(body_dict['eventid'])))
elif self.last:
body_dict['last'] = self.last
self.log_info('Option "last" set with {}'
.format(str(body_dict['last'])))
else:
body_dict['date'] = self.date.split()
self.log_info('Option "date" set with {}'
.format(json.dumps(body_dict['date'])))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
# manage to_ids and enforceWarninglist
# to avoid FP enforceWarninglist is set to True if
# to_ids is set to True (search criterion)
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.endpoint == 'events':
my_args['misp_url'] = my_args['misp_url'] + '/events/restSearch'
else:
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.keep_related is True:
keep_related = True
else:
keep_related = False
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['enforceWarninglist'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.warning_list is True:
body_dict['enforceWarninglist'] = True
elif self.warning_list is False:
body_dict['enforceWarninglist'] = False
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
# search
r = requests.post(my_args['misp_url'], headers=headers, data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
self.log_info(
"[CO301] INFO mispcollect successful. url={}, HTTP status={}".format(my_args['misp_url'], r.status_code)
)
else:
self.log_error(
"[CO302] ERROR mispcollect failed. url={}, data={}, HTTP Error={}, content={}".format(my_args['misp_url'], body, r.status_code, r.text)
)
raise Exception(
"[CO302] ERROR mispcollect failed for url={} with HTTP Error={}. Check search.log for details".format(my_args['misp_url'], r.status_code)
)
# response is 200 by this point or we would have thrown an exception
response = r.json()
encoder = json.JSONEncoder(ensure_ascii=False, separators=(',', ':'))
if self.endpoint == "events":
if 'response' in response:
for r_item in response['response']:
if 'Event' in r_item:
attribute_names = []
serial_number = 0
for e in list(r_item.values()):
if keep_related is False:
e.pop('RelatedEvent', None)
if serial_number == 0:
for k in list(e.keys()):
attribute_names.append(k)
yield MispCollectCommand._record(
serial_number, e['timestamp'], my_args['host'],
e, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
else:
if 'response' in response:
if 'Attribute' in response['response']:
attribute_names = []
serial_number = 0
for a in response['response']['Attribute']:
if serial_number == 0:
for k in list(a.keys()):
attribute_names.append(k)
yield MispCollectCommand._record(
serial_number, a['timestamp'], my_args['host'],
a, attribute_names, encoder)
serial_number += 1
GeneratingCommand.flush
class TestSearchCommand(SearchCommand):
boolean = Option(
doc='''
**Syntax:** **boolean=***<value>*
**Description:** A boolean value''',
validate=validators.Boolean())
required_boolean = Option(
doc='''
**Syntax:** **boolean=***<value>*
**Description:** A boolean value''',
require=True, validate=validators.Boolean())
aliased_required_boolean = Option(
doc='''
**Syntax:** **boolean=***<value>*
**Description:** A boolean value''',
name='foo', require=True, validate=validators.Boolean())
code = Option(
doc='''
**Syntax:** **code=***<value>*
**Description:** A Python expression, if mode == "eval", or statement, if mode == "exec"''',
validate=validators.Code())
required_code = Option(
doc='''
**Syntax:** **code=***<value>*
**Description:** A Python expression, if mode == "eval", or statement, if mode == "exec"''',
require=True, validate=validators.Code())
duration = Option(
doc='''
**Syntax:** **duration=***<value>*
**Description:** A length of time''',
validate=validators.Duration())
required_duration = Option(
doc='''
**Syntax:** **duration=***<value>*
**Description:** A length of time''',
require=True, validate=validators.Duration())
fieldname = Option(
doc='''
**Syntax:** **fieldname=***<value>*
**Description:** Name of a field''',
validate=validators.Fieldname())
required_fieldname = Option(
doc='''
**Syntax:** **fieldname=***<value>*
**Description:** Name of a field''',
require=True, validate=validators.Fieldname())
file = Option(
doc='''
**Syntax:** **file=***<value>*
**Description:** Name of a file''',
validate=validators.File())
required_file = Option(
doc='''
**Syntax:** **file=***<value>*
**Description:** Name of a file''',
require=True, validate=validators.File())
integer = Option(
doc='''
**Syntax:** **integer=***<value>*
**Description:** An integer value''',
validate=validators.Integer())
required_integer = Option(
doc='''
**Syntax:** **integer=***<value>*
**Description:** An integer value''',
require=True, validate=validators.Integer())
float = Option(
doc='''
**Syntax:** **float=***<value>*
**Description:** An float value''',
validate=validators.Float())
required_float = Option(
doc='''
**Syntax:** **float=***<value>*
**Description:** An float value''',
require=True, validate=validators.Float())
map = Option(
doc='''
**Syntax:** **map=***<value>*
**Description:** A mapping from one value to another''',
validate=validators.Map(foo=1, bar=2, test=3))
required_map = Option(
doc='''
**Syntax:** **map=***<value>*
**Description:** A mapping from one value to another''',
require=True, validate=validators.Map(foo=1, bar=2, test=3))
match = Option(
doc='''
**Syntax:** **match=***<value>*
**Description:** A value that matches a regular expression pattern''',
validate=validators.Match('social security number', r'\d{3}-\d{2}-\d{4}'))
required_match = Option(
doc='''
**Syntax:** **required_match=***<value>*
**Description:** A value that matches a regular expression pattern''',
require=True, validate=validators.Match('social security number', r'\d{3}-\d{2}-\d{4}'))
optionname = Option(
doc='''
**Syntax:** **optionname=***<value>*
**Description:** The name of an option (used internally)''',
validate=validators.OptionName())
required_optionname = Option(
doc='''
**Syntax:** **optionname=***<value>*
**Description:** The name of an option (used internally)''',
require=True, validate=validators.OptionName())
regularexpression = Option(
doc='''
**Syntax:** **regularexpression=***<value>*
**Description:** Regular expression pattern to match''',
validate=validators.RegularExpression())
required_regularexpression = Option(
doc='''
**Syntax:** **regularexpression=***<value>*
**Description:** Regular expression pattern to match''',
require=True, validate=validators.RegularExpression())
set = Option(
doc='''
**Syntax:** **set=***<value>*
**Description:** A member of a set''',
validate=validators.Set('foo', 'bar', 'test'))
required_set = Option(
doc='''
**Syntax:** **set=***<value>*
**Description:** A member of a set''',
require=True, validate=validators.Set('foo', 'bar', 'test'))
class ConfigurationSettings(SearchCommand.ConfigurationSettings):
@classmethod
def fix_up(cls, command_class):
pass
class MispSearchCommand(StreamingCommand):
"""
search in MISP for attributes matching the value of field.
##Syntax
code-block::
mispsearch field=<field> to_ids=y|n
##Description
body = {
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional"
}
##Example
Search in MISP for value of fieldname r_ip (remote IP in proxy logs).
code-block::
* | mispsearch field=r_ip
"""
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as \
described in local/misp42splunk_instances.conf''',
require=True)
field = Option(doc='''
**Syntax:** **field=***<fieldname>*
**Description:**Name of the field containing \
the value to search for.''',
require=True,
validate=validators.Fieldname())
to_ids = Option(doc='''
**Syntax:** **to_ids=***<y|n>*
**Description:** Boolean to search only attributes with to_ids set''',
require=False,
validate=validators.Boolean())
includeEventUuid = Option(doc='''
**Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
includeEventTags = Option(doc='''
**Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include Event Tags to results.''',
require=False,
validate=validators.Boolean())
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:**Publication duration in day(s), hour(s) or minute(s)
to limit search scope only to published events in last X timerange.''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; \
default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("page", r"^[0-9]+$"))
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
def log_error(self, msg):
logging.error(msg)
def log_info(self, msg):
logging.info(msg)
def log_debug(self, msg):
logging.debug(msg)
def log_warn(self, msg):
logging.warning(msg)
def set_log_level(self):
logging.root
loglevel = logging_level('misp42splunk')
logging.root.setLevel(loglevel)
logging.error('[SE-101] logging level is set to %s', loglevel)
logging.error('[SE-102] PYTHON VERSION: ' + sys.version)
def stream(self, records):
# loggging
self.set_log_level()
# Phase 1: Preparation
misp_instance = self.misp_instance
storage = self.service.storage_passwords
my_args = prepare_config(self, 'misp42splunk', misp_instance, storage)
if my_args is None:
raise Exception(
"Sorry, no configuration for misp_instance={}".format(
misp_instance))
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
pagination = True
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 1000
if self.page is not None:
page = int(self.page)
else:
page = 1
if self.json_request is not None:
body_dict = json.loads(self.json_request)
self.log_info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
if 'page' in body_dict:
page = body_dict['page']
pagination = False
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
if self.to_ids is True:
body_dict['to_ids'] = "True"
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
if self.last is not None:
body_dict['last'] = self.last
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict['value'] = str(value)
misp_category = []
misp_event_id = []
misp_event_uuid = []
misp_orgc_id = []
misp_to_ids = []
misp_comment = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
# check if status is anything other than 200;
# throw an exception if it is
if r.status_code in (200, 201, 204):
self.log_info(
"[SE301] INFO mispsearch successful. url={}, HTTP status={}"
.format(my_args['misp_url'], r.status_code))
else:
self.log_error(
"[SE302] ERROR mispsearch failed. url={}, data={}, HTTP Error={}, content={}"
.format(my_args['misp_url'], body, r.status_code,
r.text))
raise Exception(
"[SE302] ERROR mispsearch failed for url={} with HTTP Error={}. Check search.log for details"
.format(my_args['misp_url'], r.status_code))
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['comment']) not in misp_comment:
misp_comment.append(str(a['comment']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
if 'Event' in a:
if a['Event']['uuid'] \
not in misp_event_uuid:
misp_event_uuid.append(
str(a['Event']['uuid']))
if a['Event']['orgc_id'] \
not in misp_orgc_id:
misp_orgc_id.append(
str(a['Event']['orgc_id']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_comment'] = misp_comment
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
record['misp_event_uuid'] = misp_event_uuid
record['misp_orgc_id'] = misp_orgc_id
record['misp_tag'] = misp_tag
yield record
class StreamFilterWildcardCommand(StreamingCommand):
""" Returns a field with a list of non-overlapping matches to a wildcard pattern in a set of fields.
##Syntax
.. code-block::
StreamFilterWildcardCommand fieldname=<field> pattern=<field containing wildcard pattern> <field-list>
##Description
Returns the non-overlapping matches to the wildcard pattern contained in the field specified by `pattern`
The result is stored in the field specified by `fieldname`. If `fieldname` exists, its value
is replaced. If `fieldname` does not exist, it is created. Event records are otherwise passed through to the next
pipeline processor unmodified.
##Example
Return the wildcard pattern matches in the `text` of each tweet in tweets.csv and store the result in `word_count`.
.. code-block::
| inputlookup tweets | eval pattern="\\w+" | streamfilter fieldname=word_count pattern=pattern text
"""
fieldname = Option(
doc='''
**Syntax:** **fieldname=***<fieldname>*
**Description:** Name of the field that will hold the match count''',
require=True, validate=validators.Fieldname())
pattern = Option(
doc='''
**Syntax:** **pattern=***<fieldname>*
**Description:** Field name containing the wildcard pattern pattern to match''',
require=True, validate=validators.Fieldname())
#Filter the data based on the passed in Wildcard pattern, this function exists so we can handle mutli-value pattern fields
def thefilter(self, record, pattern):
values = ""
for fieldname in self.fieldnames:
#multivalue fields come through as a ListType, iterate through the list and run the Wildcard against each entry
#in the multivalued field
if isinstance(record[fieldname], types.ListType):
for aRecord in record[fieldname]:
matches = pattern.findall(six.text_type(aRecord.decode("utf-8")))
for match in matches:
values = values + " " + match
else:
matches = pattern.findall(six.text_type(record[fieldname].decode("utf-8")))
for match in matches:
values = values + " " + match
return values
#Change a wildcard pattern to a Wildcard pattern
def changeToWildcard(self, pattern):
pattern = pattern.replace("\"", "")
pattern = pattern.replace("'", "")
pattern = pattern.replace("*", ".*")
if pattern.find(".*") == 0:
pattern = "[^_].*" + pattern[2:]
pattern = "(?i)^" + pattern + "$"
return pattern
#Streaming command to work with each record
def stream(self, records):
self.logger.debug('StreamFilterWildcardCommand: %s', self) # logs command line
for record in records:
values = ""
pattern = self.pattern
if not record.has_key(pattern):
self.logger.warn("StreamFilterWildcardCommand: pattern field is %s but cannot find this field" % (pattern), self)
sys.exit(-1)
if isinstance(record[pattern], types.ListType):
for aPattern in record[pattern]:
pattern = re.compile(self.changeToWildcard(aPattern))
values = values + self.thefilter(record, pattern)
else:
pattern = re.compile(self.changeToWildcard(record[pattern]))
values = values + self.thefilter(record, pattern)
record[self.fieldname] = values
yield record
class AwsSnsAlertCommand(StreamingCommand, SNSPublisher):
account = Option(require=True)
region = Option(require=True)
topic_name = Option(require=True)
publish_all = Option(require=False)
def stream(self, records):
logger.info('Search Alert - Started')
splunkd_uri = self.search_results_info.splunkd_uri
session_key = self.search_results_info.auth_token
publish_all = util.is_true(self.publish_all or 'false')
err = 0
count = 0
for i, rec in enumerate(records):
try:
count += 1
yield self._handle_record(splunkd_uri, session_key, rec, i)
except Exception as exc:
err += 1
yield self._handle_error(exc, traceback.format_exc(), rec, i)
if not publish_all:
break
if err:
self.write_error('%s in %s events failed. '
'Check response events for detail' % (err, count))
def _handle_record(self, splunkd_uri, session_key, record, serial):
resp = self.publish(splunkd_uri,
session_key,
self.account,
self.region,
self.topic_name,
record=record)
result = {'result': 'Success', 'response': json.dumps(resp)}
res = AwsSnsAlertCommand.make_event(**result)
logger.debug('Search Alert', **result)
return {'_serial': serial, '_time': record.get('_time'), '_raw': res}
def _handle_error(self, exc, tb, record, serial):
logger.error('Search Alert', result='Failed', error=tb)
res = AwsSnsAlertCommand.make_event('Failed', error=exc)
return {'_serial': serial, '_time': record.get('_time'), '_raw': res}
@staticmethod
def make_event(result, **kwargs):
event = 'Search Alert - result="{result}"'.format(result=result)
arr = ['%s="%s"' % (key, val) for key, val in kwargs.iteritems()]
arr.insert(0, event)
return ', '.join(arr)
def make_subject(self, *args, **kwargs):
return 'Splunk - Alert from Search'
def make_message(self, *args, **kwargs):
record = kwargs['record']
return SNSMessageContent(
message=record.get('message', ''),
timestamp=record.get('timestamp', record.get('_time')),
entity=record.get('entity', ''),
correlation_id=record.get('record', self.search_results_info.sid),
source=record.get('source', ''),
event=record.get('event', record.get('_raw')),
search_name='',
results_link='',
app=self.search_results_info.ppc_app,
owner=self.search_results_info.ppc_user,
)
class abuseipCommand(StreamingCommand):
""" %(synopsis)
##Syntax
%(syntax)
##Description
%(description)
"""
ipfield = Option(doc='''
**Syntax:** **ipfield=***<fieldname>*
**Description:** Name of the IP address field to look up''',
require=True,
validate=validators.Fieldname())
def stream(self, events):
# Load config with user specified API key, if this file does not exist copy it from ../default
with open('../local/config.json') as config_file:
data = json.load(config_file)
api_key = data['abuseip'][0]['api_key']
# API required headers
headers = {
'Key': api_key,
'Accept': 'application/json',
}
for event in events:
event_dest_ip = event[self.ipfield]
# API required parameters
params = (
('ipAddress', event_dest_ip),
('maxAgeInDays', '90'),
('verbose', ''),
)
# Make API Request
error = 0
response = req.get('https://api.abuseipdb.com/api/v2/check',
headers=headers,
params=params)
if (response.status_code == 200):
data = response.json()
if 'data' in data:
addr_country_name = data['data']['countryName']
addr_domain = data['data']['domain']
addr_isp = data['data']['isp']
addr_last_reported = data['data']['lastReportedAt']
addr_abuse_confidence = data['data'][
'abuseConfidenceScore']
else:
error = 1
event[
'AbuseApiError'] = "Invalid Response:Missing data key"
else:
error = 1
event['AbuseApiError'] = "Invalid Request:status_code=" + str(
response.status_code)
# Set event values to be returned
if error == 0:
event["CountryName"] = addr_country_name
event["Domain"] = addr_domain
event["ISP"] = addr_isp
event["LastReportedAt"] = addr_last_reported
event["AbuseConfidence"] = addr_abuse_confidence
# Finalize event
yield event
class EmailValidationCommand(StreamingCommand):
field = Option(require=True, default=True, validate=validators.Fieldname())
fast = Option(require=False, default=False, validate=validators.Boolean())
timeout = Option(require=False, default=7, validate=validators.Integer())
suggest_domain = Option(require=False,
default=False,
validate=validators.Boolean())
strictness = Option(require=False,
default=0,
validate=validators.Integer())
abuse_strictness = Option(require=False,
default=0,
validate=validators.Integer())
def stream(self, records):
logger = setup_logging()
correct_records = []
incorrect_records = []
for record in records:
if self.field in record:
correct_records.append(record)
else:
incorrect_records.append(record)
if len(incorrect_records) > 0:
self.logger.error('email field missing from ' +
str(len(incorrect_records)) +
" events. They will be ignored.")
if len(correct_records) > 0:
storage_passwords = self.service.storage_passwords
for credential in storage_passwords:
if credential.content.get('realm') != 'ipqualityscore_realm':
continue
usercreds = {
'username': credential.content.get('username'),
'password': credential.content.get('clear_password')
}
if usercreds is not None:
ipqualityscoreclient = IPQualityScoreClient(
usercreds.get('password'), logger)
emails = []
rs = []
for record in correct_records:
emails.append(record.get(self.field))
rs.append(record)
results_dict = ipqualityscoreclient.email_validation_multithreaded(
emails,
fast=self.fast,
timeout=self.timeout,
suggest_domain=self.suggest_domain,
strictness=self.strictness,
abuse_strictness=self.abuse_strictness)
for record in rs:
detection_result = results_dict.get(record[self.field])
if detection_result is not None:
for key, val in detection_result.items():
new_key = ipqualityscoreclient.get_prefix(
) + "_" + key
record[new_key] = val
record[ipqualityscoreclient.get_prefix() +
"_status"] = 'api call success'
else:
record[ipqualityscoreclient.get_prefix() +
"_status"] = 'api call failed'
yield record
else:
raise Exception("No credentials have been found")
else:
raise Exception("There are no events with email field.")
class mispapireport(ReportingCommand):
""" MISP API wrapper for endpoint /attributes/restSearch.
return format is JSON for the momemnt
##Syntax
use paramater names to set values in the POST request body below.
.. code-block::
| mispapireport page=<int> limit=<int> value=string type=CSVstring category=CSVstring org=string
tags=CSVstring not_tags=CSVstrings date_from=date_string date_to=date_string last=<int>(d|h|m)
eventid=CSVint uuid=CSVuuid_string enforceWarninglist=True|False
to_ids=True|False deleted=True|False includeEventUuid=True|False includeEventTags==True|False
threat_level_id=<int> eventinfo=string
forced parameters:
"returnFormat": "json"
withAttachments: False
not handled parameters:
"publish_timestamp": "optional",
"timestamp": "optional",
"event_timestamp": "optional",
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional"
}
"""
# Superseede MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as decibed in lookup/misp_instances.csv.''',
require=False)
misp_url = Option(doc='''
**Syntax:** **misp_url=***<MISP URL>*
**Description:**URL of MISP instance.''',
require=False,
validate=validators.Match(
"misp_url",
r"^https?:\/\/[0-9a-zA-Z\-\.]+(?:\:\d+)?$"))
misp_key = Option(doc='''
**Syntax:** **misp_key=***<AUTH_KEY>*
**Description:**MISP API AUTH KEY.''',
require=False,
validate=validators.Match("misp_key",
r"^[0-9a-zA-Z]{40}$"))
misp_verifycert = Option(doc='''
**Syntax:** **misp_verifycert=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Verify or not MISP certificate.''',
require=False,
validate=validators.Boolean())
# mode: p - give parameters one by one / j provide a complete JSON string
# default is mode=p
mode = Option(doc='''
**Syntax:** **mode=***p|j<AUTH_KEY>*
**Description:**mode to build the JSON request.''',
require=False,
validate=validators.Match("mode", r"^(p|j)$"))
# if mode=j a complete JSON request has to be provided
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
# specific formats
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:**publication duration in day(s), hour(s) or minute(s).''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date_from = Option(doc='''
**Syntax:** **date_from=***date_string"*
**Description:**starting date.''',
require=False)
date_to = Option(doc='''
**Syntax:** **date_to=***date_string"*
**Description:**(optional)ending date in searches with date_from. if not set default is now''',
require=False)
threat_level_id = Option(doc='''
**Syntax:** **threat_level_id=***1-4*
**Description:**Threat level.''',
require=False,
validate=validators.Match("threat_level_id",
r"^[1-4]$"))
org = Option(doc='''
**Syntax:** **org=***CSV string*
**Description:**Comma(,)-separated string of org name(s), id(s), uuid(s).''',
require=False)
# CSV numeric list
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9,]+$"))
# strings
value = Option(doc='''
**Syntax:** **value=***string*
**Description:**value.''',
require=False)
eventinfo = Option(doc='''
**Syntax:** **eventinfo=***string*
**Description:**eventinfo string''',
require=False)
# numeric values
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; default 10000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page of result to get.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
# CSV strings
uuid = Option(doc='''
**Syntax:** **uuid=***id1(,id2,...)*
**Description:**list of event UUID(s).''',
require=False)
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''',
require=False)
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for. Wildcard is %.''',
require=False)
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for. Wildcard is %.''',
require=False)
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude from results. Wildcard is %.''',
require=False)
# Booleans
to_ids = Option(doc='''
**Syntax:** **to_ids=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to search only attributes with the flag "to_ids" set to true.''',
require=False,
validate=validators.Boolean())
enforceWarninglist = Option(doc='''
**Syntax:** **enforceWarninglist=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to apply warning lists to results.''',
require=False,
validate=validators.Boolean())
deleted = Option(doc='''
**Syntax:** **deleted=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include deleted attributes to results.''',
require=False,
validate=validators.Boolean())
includeEventUuid = Option(doc='''
**Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
includeEventTags = Option(doc='''
**Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes into 2 attributes.''',
require=False,
validate=validators.Boolean())
@Configuration()
def map(self, records):
# self.logger.debug('mispgetioc.map')
return records
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
jsonmode = False
if self.mode is not None:
if 'j' in self.mode and self.json_request is not None:
jsonmode = True
if jsonmode is True:
pagination = True
other_page = True
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set with %s',
self.json_request)
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
else:
limit = 10000
if 'page' in body_dict:
page = body_dict['page']
else:
page = 1
l = 0
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
# add provided parameters to JSON request body
# specific formats
if self.last is not None:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', body_dict['last'])
if self.date_from is not None:
body_dict['from'] = self.date_from
logging.info('Option "date_from" set with %s',
body_dict['from'])
if self.date_to is not None:
body_dict['to'] = self.date_to
logging.info('Option "date_to" set with %s',
body_dict['to'])
else:
logging.info('Option "date_to" will be set to now().')
if self.threat_level_id is not None:
body_dict['threat_level_id'] = self.threat_level_id
logging.info('Option "threat_level_id" set with %s',
body_dict['threat_level_id'])
if self.org is not None:
body_dict['org'] = self.org
logging.info('Option "org" set with %s', body_dict['org'])
if self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
body_dict['eventid'])
if self.value is not None:
body_dict['value'] = self.value
logging.info('Option "value" set with %s', body_dict['value'])
if self.eventinfo is not None:
body_dict['eventinfo'] = self.eventinfo
logging.info('Option "eventinfo" set with %s',
body_dict['eventinfo'])
# CSV strings
if self.category is not None:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
if self.type is not None:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
if self.uuid is not None:
uuid_criteria = {}
uuid_list = self.uuid.split(",")
uuid_criteria['OR'] = uuid_list
body_dict['uuid'] = uuid_criteria
# Booleans
if self.to_ids is not None:
body_dict['to_ids'] = self.to_ids
logging.info('Option "to_ids" set with %s',
body_dict['to_ids'])
if self.enforceWarninglist is not None:
body_dict['enforceWarninglist'] = self.enforceWarninglist
logging.info('Option "enforceWarninglist" set with %s',
body_dict['enforceWarninglist'])
if self.deleted is not None:
body_dict['deleted'] = self.deleted
logging.info('Option "deleted" set with %s',
body_dict['deleted'])
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
logging.info('Option "includeEventUuid" set with %s',
body_dict['includeEventUuid'])
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
logging.info('Option "includeEventTags" set with %s',
body_dict['includeEventTags'])
# Search pagination
pagination = True
other_page = True
if self.page:
page = self.page
else:
page = 1
l = 0
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 10000
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
results = []
# add colums for each type in results
while other_page:
if pagination == True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.error('INFO MISP REST API REQUEST: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
l = len(response['response']['Attribute'])
for a in response['response']['Attribute']:
v = {}
v['misp_Object'] = "-"
if self.includeEventTags is True:
v['misp_tag'] = "-"
for ak, av in a.items():
if ak == 'Event':
json_event = a['Event']
for ek, ev in json_event.items():
key = 'misp_event_' + ek
v[key] = str(ev)
elif ak == 'Tag':
tag_list = []
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
else:
vkey = 'misp_' + ak
v[vkey] = av
results.append(v)
if pagination == True:
if l < limit:
other_page = False
else:
page = page + 1
else:
other_page = False
# add colums for each type in results
typelist = []
for r in results:
if r['misp_type'] not in typelist:
typelist.append(r['misp_type'])
output_dict = {}
increment = 1
for r in results:
key = str(r['misp_event_id']) + '_' + str(increment)
increment = increment + 1
v = r
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
output_dict[key] = v
for k, v in output_dict.items():
yield v
class BlaggertCommand(StreamingCommand):
opt_token = Option(doc='''
**Syntax:** **token=***<fieldname>*
**Description:** HEC token to use.
**Default:** None''',
name='token',
require=True,
validate=validators.Fieldname())
opt_server = Option(doc='''
**Syntax:** **server=***<fieldname>*
**Description:** Server to send the payload to.
**Default:** localhost''',
name='server',
require=False,
default='localhost',
validate=validators.Fieldname())
opt_port = Option(doc='''
**Syntax:** **port=***<fieldname>*
**Description:** HEC Port, not fortified red wine.
**Default:** 8088''',
name='port',
require=False,
default=8088,
validate=validators.Integer())
def __init__(self):
super(BlaggertCommand, self).__init__()
def prepare(self):
return
def stream(self, records):
# Put your event transformation code here
url = "https://{}:{}/services/collector/event".format(
self.opt_server, self.opt_port)
headers = {"Authorization": "Splunk {}".format(self.opt_token)}
for record in records:
self.logger.info('Record {0}'.format(record))
t2 = time.time()
payload = {"event": {"event_id": str(uuid.uuid4())}}
for k, v in record.iteritems():
payload["event"][k] = v
payload_str = json.dumps(payload)
self.logger.info('send to HEC url={} - payload='.format(
url, payload_str))
try:
res = requests.post(url,
data=payload_str,
headers=headers,
verify=False)
res.raise_for_status()
self.logger.info("Sweet as {} {}".format(
res.status_code, res.text))
record["blaggert_says"] = "Done it"
except Exception as e:
self.logger.error('Send HEC Caught exception: {}'.format(e))
record["blaggert_says"] = "Buggered it {}".format(e)
yield record
class GNQuickCommand(EventingCommand):
"""
This command can be used as generating command as well as transforming command,
When used as generating command, it returns noise status of the given IP addresses,
When used as transforming command, it adds the noise status information to the events that are returned from Splunk search.
Data pulled from /v2/noise/multi/quick?ips=<ip_address1>,<ip_address2> using GreyNoise Python SDK
**Syntax**::
`| gnquick ip="10.0.1.254"`
`| gnquick ip="1.2.3.4,8.8.8.8"`
`index=_internal | gnquick ip_field="ip"`
**Description**::
When used as generating command, gnquick command uses the IP address or IP addresses provided in ip field to return
Noise status, when used as transforming command, gnquick command uses the field representing IP address presented by ip_field
to add the noise information to each events. The Noise status is pulled using method :method:quick from GreyNoise Python SDK.
"""
ip = Option(
doc='''**Syntax:** **ip=***<ip_address>*
**Description:** IP address(es) for which noise status needs to be retrieved from GreyNoise''',
name='ip', require=False
)
ip_field = Option(
doc='''
**Syntax:** **ip_field=***<ip_field>*
**Description:** Name of the field representing IP address in Splunk events''',
name='ip_field', require=False
)
def transform(self, records):
ip_addresses = self.ip
ip_field = self.ip_field
api_key = ""
EVENTS_PER_CHUNK = 1000
THREADS = 3
USE_CACHE = False
logger = utility.setup_logger(session_key=self._metadata.searchinfo.session_key, log_context=self._metadata.searchinfo.command)
if ip_addresses and ip_field:
logger.error("Please use parameter ip to work gnquick as generating command or use parameter ip_field to work gnquick as transforming command.")
self.write_error("Please use parameter ip to work gnquick as generating command or use parameter ip_field to work gnquick as transforming command")
exit(1)
try:
message = ''
api_key = utility.get_api_key(self._metadata.searchinfo.session_key, logger=logger)
except APIKeyNotFoundError as e:
message = str(e)
except HTTPError as e:
message = str(e)
if message:
self.write_error(message)
logger.error("Error occured while retrieving API key, Error: {}".format(message))
exit(1)
if ip_addresses and not ip_field:
# This peice of code will work as generating command and will not use the Splunk events.
# Splitting the ip_addresses by commas and stripping spaces from both the sides for each IP address
ip_addresses = [ip.strip() for ip in ip_addresses.split(',')]
logger.info("Started retrieving results")
try:
logger.debug("Initiating to fetch noise status for IP address(es): {}".format(str(ip_addresses)))
api_client = GreyNoise(api_key=api_key, timeout=120, integration_name="Splunk")
# Opting timout 120 seconds for the requests
noise_status = api_client.quick(ip_addresses)
logger.info("Retrieved results successfully")
# Process the API response and send the noise status information of IP with extractions to the Splunk
# Using this flag to handle the field extraction issue in custom commands
# Only the fields extracted from the first event of generated by custom command will be extracted from all events
first_record_flag = True
# Flag to indicate whether erroneous IPs are present
erroneous_ip_present = False
for ip in ip_addresses:
for sample in noise_status:
if ip == sample['ip']:
yield event_generator.make_valid_event('quick', sample, first_record_flag)
if first_record_flag:
first_record_flag = False
logger.debug("Fetched noise status for ip={} from GreyNoise API".format(str(ip)))
break
else:
erroneous_ip_present = True
logger.debug("Generating noise status for ip={} manually".format(str(ip)))
event = {
'ip': ip,
'error': 'IP address doesn\'t match the valid IP format'
}
yield event_generator.make_invalid_event('quick', event, first_record_flag)
if first_record_flag:
first_record_flag = False
if erroneous_ip_present:
logger.warn("Value of one or more IP address(es) is invalid")
self.write_warning("Value of one or more IP address(es) passed to {command_name} is invalid".format(command_name=str(self._metadata.searchinfo.command)))
except RateLimitError:
logger.error("Rate limit error occured while fetching the context information for ips={}".format(str(ip_addresses)))
self.write_error("The Rate Limit has been exceeded. Please contact the Administrator")
except RequestFailure as e:
response_code, response_message = e.args
if response_code == 401:
msg = "Unauthorized. Please check your API key."
else:
# Need to handle this, as splunklib is unable to handle the exception with (400, {'error': 'error_reason'}) format
msg = "The API call to the GreyNoise platform have been failed with status_code: {} and error: {}".format(response_code, response_message['error'] if isinstance(response_message, dict) else response_message)
logger.error("{}".format(str(msg)))
self.write_error(msg)
except ConnectionError:
logger.error("Error while connecting to the Server. Please check your connection and try again.")
self.write_error("Error while connecting to the Server. Please check your connection and try again.")
except RequestException:
logger.error("There was an ambiguous exception that occurred while handling your Request. Please try again.")
self.write_error("There was an ambiguous exception that occurred while handling your Request. Please try again.")
except Exception as e:
logger.error("Exception: {} ".format(str(traceback.format_exc())))
self.write_error("Exception occured while fetching the noise status of the IP address(es). See greynoise_main.log for more details.")
elif ip_field:
# Enter the mechanism only when the Search is complete and all the events are available
if self.search_results_info and not self.metadata.preview:
try:
# Strip the spaces from the parameter value if given
ip_field = ip_field.strip()
# Validating the given parameter
try:
ip_field = validator.Fieldname(option_name='ip_field').validate(ip_field)
except ValueError as e:
# Validator will throw ValueError with error message when the parameters are not proper
logger.error(str(e))
self.write_error(str(e))
exit(1)
# API key validation
api_key_validation, message = utility.validate_api_key(api_key, logger)
logger.debug("API validation status: {}, message: {}".format(api_key_validation, str(message)))
if not api_key_validation:
logger.info(message)
self.write_error(message)
exit(1)
# This piece of code will work as transforming command and will use the Splunk ingested events and field which is specified in ip_field.
chunk_dict = event_generator.batch(records, ip_field, EVENTS_PER_CHUNK, logger)
# This means there are only 1000 or below IPs to call in the entire bunch of records
# Use one thread with single thread with caching mechanism enabled for the chunk
if len(chunk_dict) == 1:
logger.info("Less then 1000 distinct IPs are present, optimizing the IP requests call to GreyNoise API...")
THREADS = 1
USE_CACHE = True
api_client = GreyNoise(api_key=api_key, timeout=120, use_cache=USE_CACHE, integration_name="Splunk")
# When no records found, batch will return {0:([],[])}
if len(list(chunk_dict.values())[0][0]) >= 1:
for event in event_generator.get_all_events(api_client, 'multi', ip_field, chunk_dict, logger, threads=THREADS):
yield event
else:
logger.info("No events found, please increase the search timespan to have more search results.")
except Exception as e:
logger.info("Exception occured while adding the noise status to the events, Error: {}".format(traceback.format_exc()))
self.write_error("Exception occured while adding the noise status of the IP addresses to events. See greynoise_main.log for more details.")
else:
logger.error("Please specify exactly one parameter from ip and ip_field with some value.")
self.write_error("Please specify exactly one parameter from ip and ip_field with some value.")
def __init__(self):
super(GNQuickCommand, self).__init__()
class GNFilterCommand(EventingCommand):
"""
gnfilter - Transforming Command.
Transforming command that returns events having noisy/not noisy IP addresses
as specified with noise_events parameter, defaults to true.
Data pulled from: /v2/noise/multi/quick
**Syntax**::
`index=firewall | gnfilter ip_field="ip" noise_events="false"
**Description**::
The `gnfilter` command returns the events having noisy/not noisy IP addresses represented by `ip_field` parameter
using method :method:`quick` from GreyNoise Python SDK.
"""
ip_field = Option(
doc='''
**Syntax:** **ip_field=***<ip_field>*
**Description:** Name of the field representing IP address in Splunk events''',
name='ip_field', require=True
)
noise_events = Option(
doc='''
**Syntax:** **noise_events=***<true/false>*
**Description:** Flag specifying whether to return events having noisy IP or
events having non-noisy IP addresses''',
name='noise_events', require=False, default="True"
)
api_validation_flag = False
def transform(self, records):
"""Method that processes and yield event records to the Splunk events pipeline."""
method = 'filter'
# Setup logger
logger = utility.setup_logger(
session_key=self._metadata.searchinfo.session_key, log_context=self._metadata.searchinfo.command)
# Enter the mechanism only when the Search is complete and all the events are available
if self.search_results_info and not self.metadata.preview:
EVENTS_PER_CHUNK = 1000
THREADS = 3
USE_CACHE = False
ip_field = self.ip_field
noise_events = self.noise_events
logger.info("Started filtering the IP address(es) present in field: {}, with noise_status: {}".format(
str(ip_field), str(noise_events)))
try:
if ip_field:
ip_field = ip_field.strip()
if noise_events:
noise_events = noise_events.strip()
# Validating the given parameters
try:
ip_field = validator.Fieldname(option_name='ip_field').validate(ip_field)
noise_events = validator.Boolean(option_name='noise_events').validate(noise_events)
except ValueError as e:
# Validator will throw ValueError with error message when the parameters are not proper
logger.error(str(e))
self.write_error(str(e))
exit(1)
try:
message = ''
api_key = utility.get_api_key(self._metadata.searchinfo.session_key, logger=logger)
except APIKeyNotFoundError as e:
message = str(e)
except HTTPError as e:
message = str(e)
if message:
self.write_error(message)
logger.error("Error occured while retrieving API key, Error: {}".format(message))
exit(1)
# API key validation
if not self.api_validation_flag:
api_key_validation, message = utility.validate_api_key(api_key, logger)
logger.debug("API validation status: {}, message: {}".format(api_key_validation, str(message)))
self.api_validation_flag = True
if not api_key_validation:
logger.info(message)
self.write_error(message)
exit(1)
# divide the records in the form of dict of tuples having chunk_index as key
# {<index>: (<records>, <All the ips in records>)}
chunk_dict = event_generator.batch(records, ip_field, EVENTS_PER_CHUNK, logger)
logger.debug("Successfully divided events into chunks")
# This means there are only 1000 or below IPs to call in the entire bunch of records
# Use one thread with single thread with caching mechanism enabled for the chunk
if len(chunk_dict) == 1:
logger.info(
"Less then 1000 distinct IPs are present, optimizing the IP requests call to GreyNoise API...")
THREADS = 1
USE_CACHE = True
# Opting timout 120 seconds for the requests
api_client = GreyNoise(api_key=api_key, timeout=120,
use_cache=USE_CACHE, integration_name=INTEGRATION_NAME)
# When no records found, batch will return {0:([],[])}
if len(list(chunk_dict.values())[0][0]) >= 1:
for chunk_index, result in event_generator.get_all_events(
self._metadata.searchinfo.session_key, api_client, method, ip_field, chunk_dict, logger,
threads=THREADS):
# Pass the collected data to the event filter method
for event in event_filter(
chunk_index, result, chunk_dict[chunk_index], ip_field, noise_events, method):
yield event
# Deleting the chunk with the events that are already indexed
del chunk_dict[chunk_index]
logger.info("Successfully sent all the results to the Splunk")
else:
logger.info("No events found, please increase the search timespan to have more search results.")
except Exception:
logger.info("Exception occured while filtering events, Error: {}".format(traceback.format_exc()))
self.write_error("Exception occured while filtering the events based on noise status. "
"See greynoise_main.log for more details.")
def __init__(self):
"""Initialize custom command class."""
super(GNFilterCommand, self).__init__()
class ListAlertsCommand(GeneratingCommand):
data = Option(doc='''
**Syntax:** **data=***<field>*
**Description:** Field name that will receive the alert data in json format''',
require=False,
validate=validators.Fieldname())
data_prefix = Option(doc='''
**Syntax:** **data_prefix=***<string>*
**Description:** Prefix that will be inserted before the data fields. Each data field will appear as a separate field.''',
require=False)
json_field = Option(doc='''
**syntax:** **raw=***<field>*
**description:** field name that will receive the entire record as a json object.''',
require=False,
name='json',
validate=validators.Fieldname())
status = Option(doc='''
**syntax:** **status=***<comma_separated_list_of_status>*
**description:** Only selects alerts with the provided statuses''',
require=False)
type = Option(doc='''
**syntax:** **type=***<comma_separated_list_of_types>*
**description:** Only selects alerts with the provided types''',
require=False)
severity = Option(doc='''
**syntax:** **severity=***<comma_separated_list_of_severity>*
**description:** Only selects alerts with the provided severity''',
require=False)
analyst = Option(doc='''
**syntax:** **analyst=***<comma_separated_list_of_analyst>*
**description:** Only selects alerts with the provided analysts''',
require=False)
#count = Option(require=True, validate=validators.Integer(0))
alerts = None
def generate(self):
self.logger.info('ListAlertsCommand: %s', self)
if not self.alerts:
self.alerts = AlertCollection(
self._metadata.searchinfo.session_key)
if self.status:
status = self.status.split(',')
else:
status = []
if self.type:
type = self.type.split(',')
else:
type = []
if self.severity:
severity = self.severity.split(',')
else:
severity = []
if self.analyst:
analyst = self.analyst.split(',')
else:
analyst = []
if self._metadata.searchinfo.earliest_time != 0:
earliest_time = self._metadata.searchinfo.earliest_time
else:
earliest_time = None
if self._metadata.searchinfo.latest_time != 0:
latest_time = self._metadata.searchinfo.latest_time
else:
latest_time = None
for record in self.alerts.list(status=status,
type=type,
severity=severity,
analyst=analyst,
earliest_time=earliest_time,
latest_time=latest_time,
logger=self.logger):
event = {
'_time': record['time'],
'sourcetype': 'alerts',
'type': record['type'],
'severity': record.get('severity'),
'entity': record['entity'],
'kv_key': record['_key'],
'analyst': record.get('analyst'),
'status': record['status'],
'sid': record['sid']
}
data = record['data']
if self.data:
event[self.data] = json.dumps(record['data'])
if self.data_prefix is not None:
for key, value in data.iteritems():
event[self.data_prefix + key] = value
if self.json_field:
event[self.json_field] = json.dumps(record)
yield event
class MispSearchCommand(StreamingCommand):
"""
search in MISP for attributes matching the value of field.
##Syntax
code-block::
mispsearch field=<field> to_ids=y|n
##Description
body = {
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional"
}
##Example
Search in MISP for value of fieldname r_ip (remote IP in proxy logs).
code-block::
* | mispsearch field=r_ip
"""
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as \
described in local/inputs.conf''',
require=True)
field = Option(doc='''
**Syntax:** **field=***<fieldname>*
**Description:**Name of the field containing \
the value to search for.''',
require=True,
validate=validators.Fieldname())
to_ids = Option(doc='''
**Syntax:** **to_ids=***<y|n>*
**Description:** Boolean to search only attributes with to_ids set''',
require=False,
validate=validators.Boolean())
includeEventUuid = Option(doc='''
**Syntax:** **includeEventUuid=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
includeEventTags = Option(doc='''
**Syntax:** **includeEventTags=***y|Y|1|true|True|n|N|0|false|False*
**Description:**Boolean to include event UUID(s) to results.''',
require=False,
validate=validators.Boolean())
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:**publication duration in day(s), hour(s) or minute(s). \
**eventid**, **last** and **date_from** are mutually exclusive''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search; \
default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
def stream(self, records):
# Generate args
my_args = prepare_config(self, 'misp42splunk')
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
pagination = True
if self.limit is not None:
if int(self.limit) == 0:
pagination = False
else:
limit = int(self.limit)
else:
limit = 1000
if self.page is not None:
page = int(self.page)
else:
page = 1
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
if 'limit' in body_dict:
limit = int(body_dict['limit'])
if limit == 0:
pagination = False
if 'page' in body_dict:
page = body_dict['page']
pagination = False
else:
# build search JSON object
body_dict = {"returnFormat": "json", "withAttachments": False}
if self.to_ids is True:
body_dict['to_ids'] = "True"
if self.includeEventUuid is not None:
body_dict['includeEventUuid'] = self.includeEventUuid
if self.includeEventTags is not None:
body_dict['includeEventTags'] = self.includeEventTags
if self.last is not None:
body_dict['last'] = self.last
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
body_dict['value'] = str(value)
misp_category = []
misp_event_id = []
misp_event_uuid = []
misp_orgc_id = []
misp_to_ids = []
misp_comment = []
misp_tag = []
misp_type = []
misp_value = []
misp_uuid = []
# search
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispsearch request body: %s', body)
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
# print >> sys.stderr, "DEBUG MISP REST API response: %s" % response.json()
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if str(a['type']) not in misp_type:
misp_type.append(str(a['type']))
if str(a['value']) not in misp_value:
misp_value.append(str(a['value']))
if str(a['to_ids']) not in misp_to_ids:
misp_to_ids.append(str(a['to_ids']))
if str(a['comment']) not in misp_comment:
misp_comment.append(str(a['comment']))
if str(a['category']) not in misp_category:
misp_category.append(str(a['category']))
if str(a['uuid']) not in misp_uuid:
misp_uuid.append(str(a['uuid']))
if str(a['event_id']) not in misp_event_id:
misp_event_id.append(str(a['event_id']))
if 'Tag' in a:
for tag in a['Tag']:
if str(tag['name']) not in misp_tag:
misp_tag.append(str(tag['name']))
if 'Event' in a:
if a['Event']['uuid'] \
not in misp_event_uuid:
misp_event_uuid.append(
str(a['Event']['uuid']))
if a['Event']['orgc_id'] \
not in misp_orgc_id:
misp_orgc_id.append(
str(a['Event']['orgc_id']))
record['misp_type'] = misp_type
record['misp_value'] = misp_value
record['misp_to_ids'] = misp_to_ids
record['misp_comment'] = misp_comment
record['misp_category'] = misp_category
record['misp_attribute_uuid'] = misp_uuid
record['misp_event_id'] = misp_event_id
record['misp_event_uuid'] = misp_event_uuid
record['misp_orgc_id'] = misp_orgc_id
record['misp_tag'] = misp_tag
yield record
class mispgetioc(ReportingCommand):
""" get the attributes from a MISP instance.
##Syntax
.. code-block::
| mispgetioc misp_instance=<input> last=<int>(d|h|m)
| mispgetioc misp_instance=<input> event=<id1>(,<id2>,...)
| mispgetioc misp_instance=<input> date=<<YYYY-MM-DD>
(date_to=<YYYY-MM-DD>)
##Description
{
"returnFormat": "mandatory",
"page": "optional",
"limit": "optional",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"date": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"includeEventTags": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional",
"eventinfo": "optional",
"includeProposals": "optional",
"includeDecayScore": "optional",
"includeFullModel": "optional",
"decayingModel": "optional",
"excludeDecayed": "optional",
"score": "optional"
}
# status
"returnFormat": forced to json,
"page": param,
"limit": param,
"value": not managed,
"type": param, CSV string,
"category": param, CSV string,
"org": not managed,
"tags": param, see also not_tags
"date": param,
"last": param,
"eventid": param,
"withAttachments": forced to false,
"uuid": not managed,
"publish_timestamp": managed via param last
"timestamp": not managed,
"enforceWarninglist": param,
"to_ids": param,
"deleted": forced to False,
"includeEventUuid": set to True,
"includeEventTags": param,
"event_timestamp": not managed,
"threat_level_id": not managed,
"eventinfo": not managed,
"includeProposals": not managed
"includeDecayScore": not managed,
"includeFullModel": not managed,
"decayingModel": not managed,
"excludeDecayed": not managed,
"score": not managed
}
"""
# MANDATORY MISP instance for this search
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:** MISP instance parameters
as described in local/inputs.conf.''',
require=True)
# MANDATORY: json_request XOR eventid XOR last XOR date
json_request = Option(doc='''
**Syntax:** **json_request=***valid JSON request*
**Description:**Valid JSON request''',
require=False)
eventid = Option(doc='''
**Syntax:** **eventid=***id1(,id2,...)*
**Description:**list of event ID(s) or event UUID(s).''',
require=False,
validate=validators.Match("eventid", r"^[0-9a-f,\-]+$"))
last = Option(doc='''
**Syntax:** **last=***<int>d|h|m*
**Description:** publication duration in day(s), hour(s) or minute(s).
**nota bene:** last is an alias of published_timestamp''',
require=False,
validate=validators.Match("last", r"^[0-9]+[hdm]$"))
date = Option(doc='''
**Syntax:** **date=***The user set event date field
- any of valid time related filters"*
**Description:**starting date.
**eventid**, **last** and **date** are mutually exclusive''',
require=False)
# Other params
page = Option(doc='''
**Syntax:** **page=***<int>*
**Description:**define the page for each MISP search; default 1.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
limit = Option(doc='''
**Syntax:** **limit=***<int>*
**Description:**define the limit for each MISP search;
default 1000. 0 = no pagination.''',
require=False,
validate=validators.Match("limit", r"^[0-9]+$"))
type = Option(doc='''
**Syntax:** **type=***CSV string*
**Description:**Comma(,)-separated string of types to search for.
Wildcard is %.''',
require=False)
category = Option(doc='''
**Syntax:** **category=***CSV string*
**Description:**Comma(,)-separated string of categories to search for.
Wildcard is %.''',
require=False)
tags = Option(doc='''
**Syntax:** **tags=***CSV string*
**Description:**Comma(,)-separated string of tags to search for.
Wildcard is %.''',
require=False)
not_tags = Option(doc='''
**Syntax:** **not_tags=***CSV string*
**Description:**Comma(,)-separated string of tags to exclude.
Wildcard is %.''',
require=False)
warning_list = Option(doc='''
**Syntax:** **warning_list=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to filter out well known values.''',
require=False,
validate=validators.Boolean())
to_ids = Option(doc='''
**Syntax:** **to_ids=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to search only attributes with the flag
"to_ids" set to true.''',
require=False,
validate=validators.Boolean())
geteventtag = Option(doc='''
**Syntax:** **geteventtag=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean includeEventTags. By default only
attribute tag(s) are returned.''',
require=False,
validate=validators.Boolean())
getuuid = Option(doc='''
**Syntax:** **getuuid=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return attribute UUID.''',
require=False,
validate=validators.Boolean())
getorg = Option(doc='''
**Syntax:** **getorg=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to return the ID of the organisation that
created the event.''',
require=False,
validate=validators.Boolean())
pipesplit = Option(doc='''
**Syntax:** **pipesplit=***<1|y|Y|t|true|True|0|n|N|f|false|False>*
**Description:**Boolean to split multivalue attributes.''',
require=False,
validate=validators.Boolean())
add_description = Option(doc='''
**Syntax:** **add_description=***<1|y|Y|t|true|True
|0|n|N|f|false|False>*
**Description:**Boolean to return misp_description.''',
require=False,
validate=validators.Boolean())
@Configuration()
def map(self, records):
# self.logger.debug('mispgetioc.map')
return records
def reduce(self, records):
# Phase 1: Preparation
my_args = prepare_config(self)
my_args['misp_url'] = my_args['misp_url'] + '/attributes/restSearch'
# check that ONE of mandatory fields is present
mandatory_arg = 0
if self.json_request is not None:
mandatory_arg = mandatory_arg + 1
if self.eventid:
mandatory_arg = mandatory_arg + 1
if self.last:
mandatory_arg = mandatory_arg + 1
if self.date:
mandatory_arg = mandatory_arg + 1
if mandatory_arg == 0:
logging.error('Missing "json_request", eventid", \
"last" or "date" argument')
raise Exception('Missing "json_request", "eventid", \
"last" or "date" argument')
elif mandatory_arg > 1:
logging.error('Options "json_request", eventid", "last" \
and "date" are mutually exclusive')
raise Exception('Options "json_request", "eventid", "last" \
and "date" are mutually exclusive')
body_dict = dict()
# Only ONE combination was provided
if self.json_request is not None:
body_dict = json.loads(self.json_request)
logging.info('Option "json_request" set')
elif self.eventid:
if "," in self.eventid:
event_criteria = {}
event_list = self.eventid.split(",")
event_criteria['OR'] = event_list
body_dict['eventid'] = event_criteria
else:
body_dict['eventid'] = self.eventid
logging.info('Option "eventid" set with %s',
json.dumps(body_dict['eventid']))
elif self.last:
body_dict['last'] = self.last
logging.info('Option "last" set with %s', str(body_dict['last']))
else:
body_dict['date'] = self.date.split()
logging.info('Option "date" set with %s',
json.dumps(body_dict['date']))
# Force some values on JSON request
body_dict['returnFormat'] = 'json'
body_dict['withAttachments'] = False
body_dict['deleted'] = False
body_dict['includeEventUuid'] = True
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
# Search pagination
pagination = True
if self.limit is not None:
limit = int(self.limit)
elif 'limit' in body_dict:
limit = int(body_dict['limit'])
else:
limit = 1000
if limit == 0:
pagination = False
if self.page is not None:
page = int(self.page)
elif 'page' in body_dict:
page = body_dict['page']
else:
page = 1
# Search parameters: boolean and filter
if self.to_ids is True:
body_dict['to_ids'] = True
body_dict['warning_list'] = True # protection
elif self.to_ids is False:
body_dict['to_ids'] = False
if self.warning_list is True:
body_dict['warning_list'] = True
elif self.warning_list is False:
body_dict['warning_list'] = False
if self.geteventtag is True:
body_dict['includeEventTags'] = True
if self.category is not None:
if "," in self.category:
cat_criteria = {}
cat_list = self.category.split(",")
cat_criteria['OR'] = cat_list
body_dict['category'] = cat_criteria
else:
body_dict['category'] = self.category
if self.type is not None:
if "," in self.type:
type_criteria = {}
type_list = self.type.split(",")
type_criteria['OR'] = type_list
body_dict['type'] = type_criteria
else:
body_dict['type'] = self.type
if self.tags is not None or self.not_tags is not None:
tags_criteria = {}
if self.tags is not None:
tags_list = self.tags.split(",")
tags_criteria['OR'] = tags_list
if self.not_tags is not None:
tags_list = self.not_tags.split(",")
tags_criteria['NOT'] = tags_list
body_dict['tags'] = tags_criteria
# output filter parameters
if self.getuuid is True:
my_args['getuuid'] = True
else:
my_args['getuuid'] = False
if self.getorg is True:
my_args['getorg'] = True
else:
my_args['getorg'] = False
if self.pipesplit is True:
my_args['pipe'] = True
else:
my_args['pipe'] = False
if self.add_description is True:
my_args['add_desc'] = True
else:
my_args['add_desc'] = False
results = []
# add colums for each type in results
typelist = []
if pagination is True:
body_dict['page'] = page
body_dict['limit'] = limit
body = json.dumps(body_dict)
logging.debug('mispgetioc request body: %s', body)
# search
r = requests.post(my_args['misp_url'],
headers=headers,
data=body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200;
# throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
v = {}
v['misp_category'] = str(a['category'])
v['misp_attribute_id'] = str(a['id'])
v['misp_event_id'] = str(a['event_id'])
v['misp_timestamp'] = str(a['timestamp'])
v['misp_to_ids'] = str(a['to_ids'])
v['misp_comment'] = str(a['comment'])
tag_list = []
if 'Tag' in a:
for tag in a['Tag']:
try:
tag_list.append(str(tag['name']))
except Exception:
pass
v['misp_tag'] = tag_list
# include ID of the organisation that
# created the attribute if requested
if 'Event' in a:
v['misp_event_uuid'] = str(a['Event']['uuid'])
if my_args['getorg']:
v['misp_orgc_id'] = str(a['Event']['orgc_id'])
if my_args['add_desc'] is True:
v['misp_event_info'] = str(a['Event']['info'])
# include attribute UUID if requested
if my_args['getuuid']:
v['misp_attribute_uuid'] = str(a['uuid'])
# handle object and multivalue attributes
v['misp_object_id'] = str(a['object_id'])
if my_args['add_desc'] is True:
if int(a['object_id']) == 0:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) \
+ '" in category "' + str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) + ')'
else:
v['misp_description'] = 'MISP e' \
+ str(a['event_id']) + ' attribute ' \
+ str(a['uuid']) + ' of type "' \
+ str(a['type']) + '" in category "' \
+ str(a['category']) \
+ '" (to_ids:' + str(a['to_ids']) \
+ ' - o' + str(a['object_id']) + ' )'
current_type = str(a['type'])
# combined: not part of an object
# AND multivalue attribute AND to be split
if int(a['object_id']) == 0 and '|' in current_type \
and my_args['pipe'] is True:
mv_type_list = current_type.split('|')
mv_value_list = str(a['value']).split('|')
left_v = v.copy()
left_v['misp_type'] = mv_type_list.pop()
left_v['misp_value'] = mv_value_list.pop()
results.append(left_v)
if left_v['misp_type'] not in typelist:
typelist.append(left_v['misp_type'])
right_v = v.copy()
right_v['misp_type'] = mv_type_list.pop()
right_v['misp_value'] = mv_value_list.pop()
results.append(right_v)
if right_v['misp_type'] not in typelist:
typelist.append(right_v['misp_type'])
else:
v['misp_type'] = current_type
v['misp_value'] = str(a['value'])
results.append(v)
if current_type not in typelist:
typelist.append(current_type)
logging.info(json.dumps(typelist))
output_dict = {}
# relevant_cat = ['Artifacts dropped', 'Financial fraud',
# 'Network activity','Payload delivery','Payload installation']
for r in results:
if int(r['misp_object_id']) == 0: # not an object
key = str(r['misp_event_id']) + '_' + r['misp_attribute_id']
is_object_member = False
else: # this is a MISP object
key = str(r['misp_event_id']) \
+ '_object_' + str(r['misp_object_id'])
is_object_member = True
if key not in output_dict:
v = dict(r)
for t in typelist:
misp_t = 'misp_' + t.replace('-', '_').replace('|', '_p_')
if t == r['misp_type']:
v[misp_t] = r['misp_value']
else:
v[misp_t] = ''
to_ids = []
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = []
category.append(r['misp_category'])
v['misp_category'] = category
if my_args['add_desc'] is True:
description = []
description.append(r['misp_description'])
v['misp_description'] = description
if my_args['getuuid'] is True:
attribute_uuid = []
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is True:
v['misp_type'] = 'misp_object'
v['misp_value'] = r['misp_object_id']
output_dict[key] = dict(v)
else:
v = dict(output_dict[key])
misp_t = 'misp_' + r['misp_type'].replace('-', '_')
v[misp_t] = r['misp_value'] # set value for relevant type
to_ids = v['misp_to_ids']
if r['misp_to_ids'] not in to_ids:
to_ids.append(r['misp_to_ids'])
v['misp_to_ids'] = to_ids
category = v['misp_category']
if r['misp_category'] not in category: # append category
category.append(r['misp_category'])
v['misp_category'] = category
if my_args['add_desc'] is True:
description = v['misp_description']
if r['misp_description'] not in description:
description.append(r['misp_description'])
v['misp_description'] = description
if my_args['getuuid'] is True:
attribute_uuid = v['misp_attribute_uuid']
if r['misp_attribute_uuid'] not in attribute_uuid:
attribute_uuid.append(r['misp_attribute_uuid'])
v['misp_attribute_uuid'] = attribute_uuid
if is_object_member is False:
misp_type = r['misp_type'] + '|' + v['misp_type']
v['misp_type'] = misp_type
misp_value = r['misp_value'] + '|' + v['misp_value']
v['misp_value'] = misp_value
output_dict[key] = dict(v)
for k, v in list(output_dict.items()):
yield v
class ip2intCommand(StreamingCommand):
""" Converts IPv4 to integer
##Syntax
.. code-block::
ip2int fields="<ipv4_field>[,<ipv4_field>]*" [destfield=<new_field>]
##Parameters:
fields = Comma seperated list of fields containing IPv4 addresses
destfield = *optional* If provided then the result of the command
will be stored in this new *destfield*.
Only 1 destination field can be provided so if the fields
parameter contains a list then the destfield will be a
overwritten with the last result!
If destfield is not provided then a new field will be
created as <field_int>
##Description
Splunk Streaming command (SCPv2) that will change an IPv4 address
into an integer value. Very often IP addresses are stored in
integer format in databases.
##Example
Convert a single field containing an IPv4 address into an integer. The result
will be stored in a new field called ipv4_int
.. code-block::
| makeresults
| eval ipv4="192.168.1.10"
| ip2int fields=ipv4
Convert a single field containing an IPv4 address into an integer. The result
will be stored in a new field called ip_as_int
.. code-block::
| makeresults
| eval ipv4="192.168.1.10"
| ip2int fields=ipv4 destfield=ip_as_int
"""
destfield = Option(doc='''
**Syntax:** **destfield=***<fieldname>*
**Description:** Name of the field that will be created''',
require=False,
validate=validators.Fieldname())
@Option()
def fields(self):
""" **Syntax:** logging_configuration=<path>
**Description:** Loads an alternative logging configuration file for a command invocation. The logging
configuration file must be in Python ConfigParser-format. The *<path>* name and all path names specified in
configuration are relative to the app root directory.
"""
return self._fields.split(",")
@fields.setter
def fields(self, value):
if value is not None:
self._fields = value
def __init__(self, *args, **kwargs):
super(StreamingCommand, self).__init__(*args, **kwargs)
self._fields = None
def stream(self, records):
self.logger.debug('ip2intCommand: %s', self)
for record in records:
for fld in self.fields:
if fld in record:
destfield = self.destfield if self.destfield else "{}_int".format(
fld)
try:
record[destfield] = int(self.ip2int(record[fld]))
except:
self.logger.error(
"unable to convert IP '{}' to an integer".format(
record[fld]))
record[destfield] = "conversion error"
else:
self.logger.warning("field '{}' is not found".format(fld))
yield record
def ip2int(self, addr):
return struct.unpack("!I", socket.inet_aton(str(addr)))[0]
class mispsight(StreamingCommand):
""" search in MISP for attributes matching the value of field.
##Syntax
code-block::
mispsearch field=<field> onlyids=y|n
##Description
search_body = {"returnFormat": "json",
"value": "optional",
"type": "optional",
"category": "optional",
"org": "optional",
"tags": "optional",
"from": "optional",
"to": "optional",
"last": "optional",
"eventid": "optional",
"withAttachments": "optional",
"uuid": "optional",
"publish_timestamp": "optional",
"timestamp": "optional",
"enforceWarninglist": "optional",
"to_ids": "optional",
"deleted": "optional",
"includeEventUuid": "optional",
"event_timestamp": "optional",
"threat_level_id": "optional"
}
##Example
Search in MISP for value of fieldname r_ip (remote IP in proxy logs).
code-block::
* | mispsearch fieldname=r_ip
"""
field = Option(doc='''
**Syntax:** **field=***<fieldname>*
**Description:**Name of the field containing the value to search for.''',
require=True,
validate=validators.Fieldname())
misp_instance = Option(doc='''
**Syntax:** **misp_instance=instance_name*
**Description:**MISP instance parameters as described in local/inputs.conf.''',
require=True)
def stream(self, records):
# self.logger.debug('mispgetioc.reduce')
# Generate args
my_args = prepare_config(self)
# set proper headers
headers = {'Content-type': 'application/json'}
headers['Authorization'] = my_args['misp_key']
headers['Accept'] = 'application/json'
fieldname = str(self.field)
for record in records:
if fieldname in record:
value = record.get(fieldname, None)
if value is not None:
search_url = my_args['misp_url'] + '/attributes/restSearch'
search_dict = {"returnFormat": "json"}
search_dict['value'] = str(value)
search_dict['withAttachments'] = "false",
search_body = json.dumps(search_dict)
sight_url = my_args[
'misp_url'] + '/sightings/restSearch/attribute'
sight_dict = {"returnFormat": "json"}
misp_value = ''
misp_fp = False
misp_fp_timestamp = 0
misp_fp_event_id = ''
misp_sight_seen = False
misp_sight = {
'count': 0,
'first': 0,
'first_event_id': 0,
'last': 0,
'last_event_id': 0
}
# search
logging.debug('mispsight request body: %s', body)
r = requests.post(search_url,
headers=headers,
data=search_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
r.raise_for_status()
# response is 200 by this point or we would have thrown an exception
response = r.json()
logging.info(
"MISP REST API %s has got a response with status code 200",
search_url)
logging.debug("MISP REST API %s has got a response: %s" %
(search_url, r.json()))
if 'response' in response:
if 'Attribute' in response['response']:
for a in response['response']['Attribute']:
if misp_value == '':
misp_value = str(a['value'])
if misp_fp == False:
sight_dict['id'] = str(a['id'])
sight_body = json.dumps(sight_dict)
s = requests.post(
sight_url,
headers=headers,
data=sight_body,
verify=my_args['misp_verifycert'],
cert=my_args['client_cert_full_path'],
proxies=my_args['proxies'])
# check if status is anything other than 200; throw an exception if it is
s.raise_for_status()
# response is 200 by this point or we would have thrown an exception
sight = s.json()
logging.info(
"MISP REST API %s has got a response with status code 200",
sight_url)
logging.debug(
"MISP REST API %s has got a response: %s"
% (sight_url, s.json()))
if 'response' in sight:
for se in sight['response']:
if 'Sighting' in se:
if int(se['Sighting']['type']
) == 0: #true sighting
misp_sight_seen = True
misp_sight[
'count'] = misp_sight[
'count'] + 1
if misp_sight['first'] == 0 or \
misp_sight['first'] > int(se['Sighting']['date_sighting']):
misp_sight[
'first'] = int(
se['Sighting']
['date_sighting']
)
misp_sight[
'first_event_id'] = se[
'Sighting'][
'event_id']
if misp_sight['last'] < int(
se['Sighting']
['date_sighting']):
misp_sight['last'] = int(
se['Sighting']
['date_sighting'])
misp_sight[
'last_event_id'] = se[
'Sighting'][
'event_id']
elif int(
se['Sighting']['type']
) == 1: #false positive
misp_fp = True
misp_fp_timestamp = int(
se['Sighting']
['date_sighting'])
misp_fp_event_id = se[
'Sighting']['event_id']
if misp_fp == True:
record['misp_value'] = misp_value
record['misp_fp'] = "True"
record['misp_fp_timestamp'] = str(
misp_fp_timestamp)
record['misp_fp_event_id'] = str(
misp_fp_event_id)
if misp_sight_seen == True:
record['misp_value'] = misp_value
record['misp_sight_count'] = str(
misp_sight['count'])
record['misp_sight_first'] = str(
misp_sight['first'])
record['misp_sight_first_event_id'] = str(
misp_sight['first_event_id'])
record['misp_sight_last'] = str(
misp_sight['last'])
record['misp_sight_last_event_id'] = str(
misp_sight['last_event_id'])
yield record
class StackOperation(GeneratingCommand):
command = Option(require=True)
stack_id = Option(require=True)
def generate(self):
root_logger = logging.getLogger()
root_logger.setLevel("DEBUG")
class EventBufferHandler(logging.Handler):
_events = None
_last_time = None
def __init__(self):
logging.Handler.__init__(self)
self._events = []
def emit(self, record):
msg = record.getMessage().replace('"', '\\"')
time = round(record.created, 3)
if self._last_time and time <= self._last_time:
time = self._last_time + 0.001
self._last_time = time
self._events.append({
"_raw": "%.3f, level=\"%s\", msg=\"%s\"" % (
time,
record.levelname,
msg
),
})
@property
def events(self):
return self._events
buffer_handler = EventBufferHandler()
root_logger.addHandler(buffer_handler)
logging.debug("running '%s' command .." % self.command)
# HTTPError: HTTP 503 Service Unavailable - - KV Store is initializing.
# Please try again later.
try:
_ = self.service.kvstore["stacks"].data
except splunklib.binding.HTTPError as e:
if e.status == 503:
logging.warning("%s" % e)
return
raise
try:
if self.command == "up":
up(self.service, self.stack_id)
return
elif self.command == "down" or self.command == "kill":
down(self.service, self.stack_id,
force=self.command == "kill")
else:
logging.error("unknown command: %s" % self.command)
logging.debug("will stop '%s' command" % self.command)
except errors.RetryOperation as e:
msg = "%s" % e
if msg:
logging.info("%s" % msg)
logging.debug("will check in 1m")
return
except:
import traceback
logging.error("%s\n(will try again in 1m)" %
traceback.format_exc())
return
finally:
logging.shutdown()
for e in buffer_handler.events:
yield e
unschedule_operation(self.service, self.stack_id)
