def execute():
try:
keywords, argvals = isp.getKeywordsAndOptions()
results, dummyresults, settings = isp.getOrganizedResults()
sessionKey = settings.get('sessionKey')
if sessionKey == None:
return vixutils.generateErrorResults(
'sessionKey not passed to the search command, something\'s very wrong!'
)
#check that the command is being executed by the scheduler
sid = settings.get('sid')
if not sid.startswith('scheduler_') and not argvals.get(
'forcerun', '') == '1':
return vixutils.generateErrorResults(
'rollercontroller is supposed to be ran by the scheduler, add forcerun=1 to force execution'
)
# check if error messaging is disabled
global ERRMSGS_ENABLED
ERRMSGS_ENABLED = 'disablemsgs' not in keywords
providers = erp_launcher.listProviders(sessionKey)
rollVixes = erp_launcher.listVixes(
sessionKey, 'disabled=0 AND vix.output.buckets.from.indexes=*')
rollProviders = filterRollProviders(rollVixes, providers)
searchString = genSearchString(rollVixes, rollProviders)
kwargs = {}
for k in ['owner', 'namespace', 'sessionKey', 'hostPath']:
if k in settings:
kwargs[k] = settings[k]
if not os.path.exists(vixutils.getAppBinJars()):
# first time we're copying jars, force bundle replication
kwargs['force_bundle_replication'] = 1
prepareSearchExecution()
numRetries = argvals.get("retries", 1)
for i in range(0, int(numRetries)):
logger.info("Dispatching the search: %s" % searchString)
search = splunk.search.dispatch(searchString, **kwargs)
try:
streamSearch(search, sessionKey)
finally:
cancelSearch(search)
except Exception as e:
import traceback
splunkio.write([{
"stack": traceback.format_exc(),
"exception": str(e)
}])
finally:
sys.stdout.flush()
def launchSplunkMRForIndexes(sessionKey, action, logFileName, providers, vixes,
indexFilterFunc):
"""
Will execute an action on each of a set of providers. Here "action" is meant in the sense of the ERP protocol
contract, and must have a handler registerd with the SplunkMR java class. This function expects a set of providers
and virtual indexes of interest to be provided as arguments. A process will be launched for each
such provider that is associated with at least one such virtual index. The information in the index's provider's
configuration will be respected, including the actual command that gets run, environment variables, etc. A set of
(presumably non-virtual) indexes will be provided to the action as well, based on the provided filtering funciton.
FILTER FUNCTION
One argument to this method should be a filter function. This is a function that takes 2 arguments.
--indexes: A dict of indexes obtained from the REST endpoint.
--vixes: A dict virtual indexes, which will be a subset of the parameter of the same name to the outer function.
This method should filter the indexes dict to include only those associated with the vixes. If this method is null,
the list of indexes will not be filtered before being given to the provider's process. As an example, for archiving,
this method should take a dict of virtual indexes, and return only those (non-virtual) indexes which get archived
into the former.
:param sessionKey: An authentication token for a valid Splunk session.
:param action: A string that is recognized by the SplunkMR class.
:param logFileName: Name of the file to which the output of the external process will be piped. Will be placed in
the <Splunk home>/var/log/splunk dir.
:param providers: The providers this command should consider, as a map from provider name to a splunk entity,
represented as nested json dicts.
:param vixes: The virtual indexes this command should consider, as a map from index name to a splunk entity,
represented as nested json dicts.
"""
t = None
try:
if sessionKey == None:
return vixutils.generateErrorResults(
"username/password authorization not given to 'input'.")
# Expand env variables
providers = _replaceAllSplunkHomeBinJars(providers)
vixes = _replaceAllSplunkHomeBinJars(vixes)
# Get indexes from the REST endpoint
indexes = _entityToDict(_listIndexes(sessionKey, 'disabled=0'))
if indexFilterFunc:
indexes = indexFilterFunc(indexes, vixes.values())
serverId, serverName = _getServerId(sessionKey)
# Everything seems ok, start message thread
t = Thread(target=_messageSH)
t.setDaemon(True)
t.start()
t_parent_checker = Thread(target=_checkParentProcess,
kwargs={'serverName': serverName})
t_parent_checker.setDaemon(True)
t_parent_checker.start()
_executeJavaProcesses(action, logFileName, indexFilterFunc, providers,
vixes, indexes, serverId, serverName, sessionKey)
except Exception, e:
_outputError(e, traceback.format_exc())
for vixMap in vixes:
if rollKey in vixMap and vixMap[rollKey]:
indexesToKeep.update(set(splitCommaList(vixMap[rollKey])))
return {k: v for k, v in indexes.items() if k in indexesToKeep}
def getProvidersAndVixesFromStdIn():
# Ghetto parsing since splunk's parsing doesn't care about quote escaping
# Only works when there's one and only one argument.
jsonStr = sys.argv[1:][0].split("=", 1)[1]
if jsonStr == None:
raise Exception("Missing required json blob in arguments " +
str(sys.argv))
jzon = json.loads(jsonStr)
return (jzon['providers'], jzon['vixes'])
if __name__ == '__main__':
providers, vixes = getProvidersAndVixesFromStdIn()
results, dummyresults, settings = isp.getOrganizedResults()
sessionKey = settings.get("sessionKey")
if sessionKey == None:
vixutils.generateErrorResults(
"username/password authorization not given to 'input'.")
else:
erp_launcher.launchSplunkMRForIndexes(sessionKey, 'roll',
'splunk_archiver.log', providers,
vixes, keepRollingIndexes)