def stream_events(self, inputs, ew):
"""The method called to stream events into Splunk.
This method overrides method in splunklib modular input.
It pre-processes the input args and call collect_events to stream events.
:param inputs: An ``InputDefinition`` object.
:param ew: An object with methods to write events and log messages to Splunk.
"""
# the input metadata is like
# {
# 'server_uri': 'https://127.0.0.1:8089',
# 'server_host': 'localhost',
# 'checkpoint_dir': '...',
# 'session_key': 'ceAvf3z^hZHYxe7wjTyTNo6_0ZRpf5cvWPdtSg'
# }
self.context_meta = inputs.metadata
# init setup util
uri = inputs.metadata["server_uri"]
session_key = inputs.metadata['session_key']
self.setup_util = Setup_Util(uri, session_key, self.logger)
input_definition = smi.input_definition.InputDefinition()
input_definition.metadata = copy.deepcopy(inputs.metadata)
input_definition.inputs = copy.deepcopy(inputs.inputs)
try:
self.parse_input_args(input_definition)
except Exception as e:
import traceback
self.log_error(traceback.format_exc())
print(traceback.format_exc(), file=sys.stderr)
# print >> sys.stderr, traceback.format_exc()
self.input_stanzas = {}
if not self.input_stanzas:
# if no stanza found. Just return
return
try:
self.set_log_level(self.log_level)
except:
self.log_debug('set log level fails.')
try:
self.collect_events(ew)
except Exception as e:
import traceback
self.log_error('Get error when collecting events.\n' +
traceback.format_exc())
print(traceback.format_exc(), file=sys.stderr)
# print >> sys.stderr, traceback.format_exc()
raise RuntimeError(str(e))
def __init__(self, ta_name, alert_name):
self._alert_name = alert_name
# self._logger_name = "modalert_" + alert_name
self._logger_name = alert_name + "_modalert"
self._logger = get_logger(self._logger_name)
super(ModularAlertBase, self).__init__(
sys.stdin.read(), self._logger, alert_name)
self.setup_util_module = None
self.setup_util = None
self.result_handle = None
self.ta_name = ta_name
self.splunk_uri = self.settings.get('server_uri')
self.setup_util = Setup_Util(self.splunk_uri, self.session_key, self._logger)
self.rest_helper = TARestHelper(self._logger)
def listenerPost(self, **kwargs):
cherrypy.response.headers['Content-Type'] = 'text/plain'
# Get arguments
publictoken = kwargs["public_token"]
splunkd_session = cherrypy.request.cookie["splunkd_8000"].value
# Sometimes I look at my code and think "What did I just do?"
# This is one of those moments
# Though really what I think I did is hijack the modular input class definition, create its setup_util attribute using the Setup_Util constructor, and then pull the necessary decrypted attributes
helper = ModInputplaid_modular_input()
helper.setup_util = Setup_Util("https://127.0.0.1:8089", splunkd_session, helper.logger)
enable_development_mode = helper.get_global_setting("enable_development_mode_")
mode = "sandbox"
if enable_development_mode.isdigit():
if int(enable_development_mode) == 1:
mode = "development"
# API Credentials
clientid = helper.get_global_setting("client_id")
client_secret = helper.get_global_setting("client_secret_%s_" % mode)
# Set the base URL
base_url = "https://%s.plaid.com" % mode
# Our json request body, exchange the public token for an access_token
data = {
'client_id': clientid,
'secret': client_secret,
'public_token': publictoken
}
# Exchange our public token for an access token
req = urllib2.Request('%s/item/public_token/exchange' % base_url, json.dumps(data), headers={'Content-type': 'application/json', 'Accept': 'application/json'})
response = urllib2.urlopen(req)
return response.read()
def reduce(self, records):
_max_content_bytes = 100000
_max_content_records = 1000
self.logger.debug('synckvstore: %s', self) # logs command line
self.logger.info("synckvstore command started.")
splunkserver = self.splunkserver
self.logger.debug("splunkserver={}".format(splunkserver))
destappname = self.destappname
self.logger.debug("destappname={}".format(destappname))
destcollection = self.destcollection
self.logger.debug("destcollection={}".format(destcollection))
if self.desttableaction:
desttableaction = self.desttableaction
self.logger.debug("desttableaction={}".format(desttableaction))
else:
desttableaction = "update"
self.logger.debug("desttableaction=update (default)")
username = self.username
self.logger.debug("username={}".format(username))
setup_util = Setup_Util(self.metadata.searchinfo.splunkd_uri,
self.metadata.searchinfo.session_key,
self.logger)
user_account = setup_util.get_credential_by_username(username)
if not user_account:
self.logger.error("unable to find user account")
return
splunkService = splunkClient.connect(
host=splunkserver,
port=8089,
username=user_account.get('username'),
password=user_account.get('password'),
owner='nobody',
app=destappname)
#Check if KVStore collection exists
if destcollection not in splunkService.kvstore:
self.logger.error(
"message=\"Collection Not Found\" dest_server={0} dest_collection={1}"
.format(splunkserver, destcollection))
for collection in splunkService.kvstore:
self.logger.error("{}".format(collection.name))
return 1
# If replace method is selected use SDK KVStore to delete the data in the collection
if desttableaction == "replace":
splunkService.kvstore[destcollection].data.delete()
self.logger.info(
"action=deleted collection_name={0} message=\"Remote Collection Data Deleted\""
.format(destcollection))
# any attempts to use the threaded class splunk_sendto_kvstore resulted in the network connection hanging
# if executed in an independent python script it worked fine so not using the threaded class...
postList = []
for entry in records:
if ((len(json.dumps(postList)) + len(json.dumps(entry))) <
_max_content_bytes) and (len(postList) + 1 <
_max_content_records):
postList.append(entry)
else:
destKVStore.postDataToSplunk(postList)
self.splunk_sendto_kvstore_func(splunkserver, destappname,
destcollection, postList,
user_account.get('username'),
user_account.get('password'))
postList = []
postList.append(entry)
yield entry
self.logger.error("post list: %s" % (postList))
self.splunk_sendto_kvstore_func(splunkserver, destappname,
destcollection, postList,
user_account.get('username'),
user_account.get('password'))
self.logger.info("synckvstore command completed.")
# return ast.literal_eval(value)
return int(value)
elif hasattr(value, '__iter__'):
return map(convert_json, value)
return value
if __name__ == "__main__":
splunk_results, unused1, settings = splunk.Intersplunk.getOrganizedResults(
)
keywords, options = splunk.Intersplunk.getKeywordsAndOptions()
session_key = settings.get("sessionKey")
uri = "https://localhost:8089"
setup_util = Setup_Util(uri, session_key, logger)
aws_access_key = setup_util.get_customized_setting("aws_access_key")
aws_secret_key = setup_util.get_customized_setting("aws_secret_key")
iam_role_arn = setup_util.get_customized_setting("iam_role_arn")
if aws_secret_key != None and aws_access_key != None:
boto_session = boto3.session.Session(
region_name="eu-west-2",
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key,
aws_session_token=None)
else:
boto_session = None
boto_session = aws_session(session=boto_session,
role_arn=None,