def execute():
results = []
try:
results, dummyresults, settings = si.getOrganizedResults()
# default values
args = {"namespace": "search"}
# get commandline args
keywords, options = si.getKeywordsAndOptions()
# override default args with settings from search kernel
args.update(settings)
# override default args with commandline args
args.update(options)
sessionKey = args.get("sessionKey", None)
owner = args.get("owner", "admin")
namespace = args.get("namespace", None)
if namespace.lower() == "none":
namespace = None
messages = {}
if sessionKey == None:
# this shouldn't happen, but it's useful for testing.
try:
sessionKey = sa.getSessionKey("admin", "changeme")
si.addWarnMessage(
messages, "No session given to 'tune' command. Using default admin account and password."
)
except splunk.AuthenticationFailed, e:
si.addErrorMessage(messages, "No session given to 'tune' command.")
return
if len(keywords) != 1:
usage()
# e.g., '/data/inputs/monitor'
entity = keywords[0]
logger.info("Entity: %s Args: %s" % (entity, args))
results = [] # we don't care about incoming results
try:
entitys = en.getEntities(entity, sessionKey=sessionKey, owner=owner, namespace=namespace, count=-1)
for name, entity in entitys.items():
try:
myapp = entity["eai:acl"]["app"]
if namespace != None and myapp != namespace:
continue
except:
continue # if no eai:acl/app, filter out
result = entityToResult(name, entity)
results.append(result)
except splunk.ResourceNotFound, e2:
pass
def main():
try:
messages = {}
keywords,options = si.getKeywordsAndOptions()
DEFAULT_MAX_TYPES = 10
maxtypes = options.get('max', str(DEFAULT_MAX_TYPES))
error = None
if not maxtypes.isdigit():
error = 'max must be an integer between 1-%s.' % MAXRESULTS
else:
maxtypes = int(maxtypes)
if not (0 < maxtypes <= MAXRESULTS):
error = 'max must be an integer between 1-%s.' % MAXRESULTS
if error:
si.generateErrorResults(error)
return
ignore_covered = 'notcovered' in keywords
useraw = 'useraw' in keywords
results,dummyresults,settings = si.getOrganizedResults()
#for r in results:
# for attr in r:
# print attr, r[attr], len(r[attr])
if len(results) > MAXRESULTS:
results = results[:MAXRESULTS]
si.addWarnMessage(messages, "For performance reasons, the maximum number of results used to discover event types was capped at %s. Consider a more restrictive search." % MAXRESULTS)
argc = len(sys.argv)
argv = sys.argv
sessionKey = settings.get("sessionKey", None)
owner = settings.get("owner", None)
namespace = settings.get("namespace", None)
searchhead = ''
try:
searches = sutils.getCommands(settings.get("search", ''), None)
firstcmd = searches[0][0][0]
firstarg = searches[0][0][1].strip()
if firstcmd == 'search' and firstarg != '*':
searchhead = firstarg
except Exception, e:
pass
results = discover(results, searchhead, maxtypes, ignore_covered, useraw)
if len(results) == 0:
si.addWarnMessage(messages, "Unable to isolate useful groups of events.")
def main():
if len(sys.argv) < 3:
usage()
tname = sys.argv[1]
#log("args")
#for v in sys.argv:
# log(v)
options = ["max_terms", "use_disjunct", "eventsonly"]
srchargs = []
log("ARGS: %s" % sys.argv[2:])
for arg in sys.argv[2:]:
for option in options:
if arg.startswith(option):
break
else:
srchargs.append(arg)
if len(srchargs) == 0:
usage()
tsearch = ' '.join(srchargs)
log("SEARCH: %s" % tsearch)
results, dummyresults, settings = si.getOrganizedResults()
results = [] # we don't care about incoming results
########TEST#####################
if 'sessionKey' not in settings:
settings['owner'] = 'admin'
settings['password'] = '******'
settings['namespace'] = 'search'
settings['sessionKey'] = splunk.auth.getSessionKey('admin', 'changeme')
########TEST####################
kwargs = {}
for f in ['owner', 'namespace', 'sessionKey', 'hostPath']:
if f in settings:
kwargs[f] = settings[f]
messages = {}
try:
maxTerms = int(settings.get("max_terms", MAX_SEARCH_COMPLEXITY))
if maxTerms > MAX_SEARCH_COMPLEXITY or maxTerms < 1:
si.addWarnMessage(
messages,
"max_terms must be between 1 and %s. Using default." %
MAX_SEARCH_COMPLEXITY)
maxTerms = MAX_SEARCH_COMPLEXITY
except Exception, e:
maxTerms = MAX_SEARCH_COMPLEXITY
def main():
if len(sys.argv) < 3:
usage()
tname = sys.argv[1]
#log("args")
#for v in sys.argv:
# log(v)
options = ["max_terms", "use_disjunct", "eventsonly"]
srchargs = []
log("ARGS: %s" % sys.argv[2:])
for arg in sys.argv[2:]:
for option in options:
if arg.startswith(option):
break
else:
srchargs.append(arg)
if len(srchargs) == 0:
usage()
tsearch = ' '.join(srchargs)
log("SEARCH: %s" % tsearch)
results,dummyresults,settings = si.getOrganizedResults()
results = [] # we don't care about incoming results
########TEST#####################
if 'sessionKey' not in settings:
settings['owner'] = 'admin'
settings['password'] = '******'
settings['namespace'] = 'search'
settings['sessionKey'] = splunk.auth.getSessionKey('admin', 'changeme')
########TEST####################
kwargs = {}
for f in ['owner','namespace','sessionKey','hostPath']:
if f in settings:
kwargs[f] = settings[f]
messages = {}
try:
maxTerms = int(settings.get("max_terms", MAX_SEARCH_COMPLEXITY))
if maxTerms > MAX_SEARCH_COMPLEXITY or maxTerms < 1:
si.addWarnMessage(messages, "max_terms must be between 1 and %s. Using default." % MAX_SEARCH_COMPLEXITY)
maxTerms = MAX_SEARCH_COMPLEXITY
except Exception, e:
maxTerms = MAX_SEARCH_COMPLEXITY
def main():
if len(sys.argv) < 3:
usage()
tname = sys.argv[1]
#log("args")
#for v in sys.argv:
# log(v)
options = ["max_terms", "use_disjunct", "eventsonly"]
srchargs = []
log("ARGS: %s" % sys.argv[2:])
for arg in sys.argv[2:]:
for option in options:
if arg.startswith(option):
break
else:
srchargs.append(arg)
if len(srchargs) == 0:
usage()
tsearch = ' '.join(srchargs)
log("SEARCH: %s" % tsearch)
results, dummyresults, settings = si.getOrganizedResults()
results = [] # we don't care about incoming results
########TEST#####################
if 'sessionKey' not in settings:
settings['owner'] = 'admin'
settings['password'] = '******'
settings['namespace'] = 'search'
settings['sessionKey'] = splunk.auth.getSessionKey('admin', 'changeme')
########TEST####################
kwargs = {}
for f in ['owner', 'namespace', 'sessionKey', 'hostPath']:
if f in settings:
kwargs[f] = settings[f]
messages = {}
try:
maxTerms = int(settings.get("max_terms", MAX_SEARCH_COMPLEXITY))
if maxTerms > MAX_SEARCH_COMPLEXITY or maxTerms < 1:
si.addWarnMessage(
messages,
"max_terms must be between 1 and %s. Using default." %
MAX_SEARCH_COMPLEXITY)
maxTerms = MAX_SEARCH_COMPLEXITY
except Exception as e:
maxTerms = MAX_SEARCH_COMPLEXITY
dummy, options = si.getKeywordsAndOptions()
makeORs = isTrue(options.get("use_disjunct", "t"))
eventsOnly = isTrue(options.get("eventsonly", "f"))
log("MAXTERMS: %s MAKEORS: %s eventsOnly: %s" %
(maxTerms, makeORs, eventsOnly))
log("tsearch: %s" % tsearch)
results = []
try:
results = findTransaction(tname, tsearch, makeORs, eventsOnly,
maxTerms, messages, **kwargs)
except Exception as e:
error(e)
events = []
log("RESULTS: %s" % len(results))
for result in results: # api fail
event = {}
for field in result:
if field == '_time':
event['_time'] = util.dt2epoch(
util.parseISO(str(result['_time'])))
else:
event[field] = result[field]
events.append(event)
si.outputResults(events, messages)
def findTransaction(tname, tconstraint, useORs, eventsOnly, maxTerms, messages,
**kwargs):
base_search, fields, maxspan = getTransactionInfo(tname, **kwargs)
if maxspan == None:
si.addWarnMessage(
messages,
"Add a maxspan contraint to the %s transactiontype definition to improve performance. Searching over all time for transitive values."
% tname)
log("MAXSPAN: %s" % maxspan)
# require one field in transaction definition
fieldsearch = " OR ".join(["%s=*" % field for field in fields])
initialConstraint = tconstraint
if useORs:
## forces an OR of terms. slow and unnessary
## initialConstraint = disjunctify(tconstraint)
# get the most restrictive term in the search and use that as the initial constrait to find events
restrictiveTerm = getMostRestrictiveTerm(tconstraint, **kwargs)
log("MOST RESTRICTIVE: %s" % restrictiveTerm)
initialConstraint = restrictiveTerm
# e.g., "sourcetype=sendmail" + "from=amrit" + "(qid=* OR mid=* OR pid=*)"
index_search = "search (%s) (%s) (%s)" % (base_search, initialConstraint,
fieldsearch)
log("INDEX SEARCH: %s" % index_search)
field_list_str = " ".join(fields)
max_combos = maxTerms / len(fields)
log("MAX_COMBINATION: %s" % max_combos)
needsTIME = ""
if maxspan != None:
needsTIME = "_time"
# make search to get field value pairs.
# # e.g. | stats values(qid) as qid values(mid) as mid values(pid) as pid
# stats_search = "| stats " + " ".join("values(%s) as %s" % (field, field) for field in fields)
# # use top
# stats_search = '| fillnull value="%s" %s | top %s %s showperc=false | addcoltotals' % (NULL_VAL, field_list_str, MAX_FIELD_COMBOS, field_list_str)
#
# TODO: if transactiondefinition contains maxspan, consider making
# first stats_search return time ranges to limit values of fields
stats_search = '| table %s %s | fillnull value="%s" %s | dedup %s | head %d' % (
field_list_str, needsTIME, NULL_VAL, field_list_str, field_list_str,
max_combos)
seenFields = set()
while True:
search = index_search + stats_search
log("running search: %s" % search)
results = splunk.search.searchAll(search, **kwargs)
## generate an OR of ANDS of field combinations -- (qid=1 pid=2) OR (qid=3 pid=4)..."
ors = []
# for each top permuation of field values
for result in results:
ands = []
# for each field
for field in result:
if field == '_time': # if we have time field we must have maxspan
# if we have maxspan info about event, use it to limit window of events to +/- maxspan of window
# we don't need float precision, because subseconds don't matter in maxpan spec
eventtime = int(
util.dt2epoch(util.parseISO(str(result['_time']))))
ands.append('_time>=%s' % (eventtime - maxspan))
ands.append('_time<=%s' % (eventtime + maxspan))
else:
val = result[field]
# ignore empty values
if val != NULL_VAL:
seenFields.add(
field) # add to list of fields with a value
ands.append('%s="%s"' % (field, escVal(result[field])))
ands_str = "(" + " ".join(ands) + ")"
ors.append(ands_str)
field_constraints = " OR ".join(ors)
# e.g., "sourcetype=sendmail (qid=1 pid=2) OR (qid=3 pid=4)..."
index_search = "search (%s) (%s)" % (base_search, field_constraints)
log("INDEXSEARCH: %s" % index_search)
if len(results) >= max_combos:
si.addWarnMessage(
messages,
"Reached max complexity in trying to find transaction events with %s unique values per field. Preferring more recent values. A more detailed initial transaction constraint will allow more complete transactions"
% max_combos)
if seenFields == set(fields):
log("SEEN VALUES FOR ALL FIELDS: %s" % fields)
break
if len(results) == 0:
msg = "No results in searching for required fields"
si.addWarnMessage(messages, msg)
return []
# we've retrieved all the events we're going to with the last index_search!
if eventsOnly:
# no transaction search, just return the events
transaction_search = ""
else:
# this is it, find the transactions!
transaction_search = '| transaction name="%s" | search %s' % (
tname, tconstraint)
search = index_search + transaction_search
log("running final search! %s" % search)
results = splunk.search.searchAll(search, **kwargs)
return results
def findTransaction(tname, tconstraint, useORs, eventsOnly, maxTerms, messages, **kwargs):
base_search, fields, maxspan = getTransactionInfo(tname, **kwargs)
if maxspan == None:
si.addWarnMessage(messages, "Add a maxspan contraint to the %s transactiontype definition to improve performance. Searching over all time for transitive values." % tname)
log("MAXSPAN: %s" % maxspan)
# require one field in transaction definition
fieldsearch = " OR ".join(["%s=*" % field for field in fields])
initialConstraint = tconstraint
if useORs:
## forces an OR of terms. slow and unnessary
## initialConstraint = disjunctify(tconstraint)
# get the most restrictive term in the search and use that as the initial constrait to find events
restrictiveTerm = getMostRestrictiveTerm(tconstraint, **kwargs)
log("MOST RESTRICTIVE: %s" % restrictiveTerm)
initialConstraint = restrictiveTerm
# e.g., "sourcetype=sendmail" + "from=amrit" + "(qid=* OR mid=* OR pid=*)"
index_search = "search (%s) (%s) (%s)" % (base_search, initialConstraint, fieldsearch)
log("INDEX SEARCH: %s" % index_search)
field_list_str = " ".join(fields)
max_combos = maxTerms / len(fields)
log("MAX_COMBINATION: %s" % max_combos)
needsTIME = ""
if maxspan != None:
needsTIME = "_time"
# make search to get field value pairs.
# # e.g. | stats values(qid) as qid values(mid) as mid values(pid) as pid
# stats_search = "| stats " + " ".join("values(%s) as %s" % (field, field) for field in fields)
# # use top
# stats_search = '| fillnull value="%s" %s | top %s %s showperc=false | addcoltotals' % (NULL_VAL, field_list_str, MAX_FIELD_COMBOS, field_list_str)
#
# TODO: if transactiondefinition contains maxspan, consider making
# first stats_search return time ranges to limit values of fields
stats_search = '| table %s %s | fillnull value="%s" %s | dedup %s | head %s' % (field_list_str, needsTIME, NULL_VAL, field_list_str, field_list_str, max_combos)
seenFields = set()
while True:
search = index_search + stats_search
log("running search: %s" % search)
results = splunk.search.searchAll(search, **kwargs)
## generate an OR of ANDS of field combinations -- (qid=1 pid=2) OR (qid=3 pid=4)..."
ors = []
# for each top permuation of field values
for result in results:
ands = []
# for each field
for field in result:
if field == '_time': # if we have time field we must have maxspan
# if we have maxspan info about event, use it to limit window of events to +/- maxspan of window
# we don't need float precision, because subseconds don't matter in maxpan spec
eventtime = int(util.dt2epoch(util.parseISO(str(result['_time']))))
ands.append('_time>=%s' % (eventtime - maxspan))
ands.append('_time<=%s' % (eventtime + maxspan))
else:
val = result[field]
# ignore empty values
if val != NULL_VAL:
seenFields.add(field) # add to list of fields with a value
ands.append('%s="%s"' % (field, escVal(result[field])))
ands_str = "(" + " ".join(ands) + ")"
ors.append(ands_str)
field_constraints = " OR ".join(ors)
# e.g., "sourcetype=sendmail (qid=1 pid=2) OR (qid=3 pid=4)..."
index_search = "search (%s) (%s)" % (base_search, field_constraints)
log("INDEXSEARCH: %s" % index_search)
if len(results) >= max_combos:
si.addWarnMessage(messages, "Reached max complexity in trying to find transaction events with %s unique values per field. Preferring more recent values. A more detailed initial transaction constraint will allow more complete transactions" % max_combos)
if seenFields == set(fields):
log("SEEN VALUES FOR ALL FIELDS: %s" % fields)
break
if len(results) == 0:
msg = "No results in searching for required fields"
si.addWarnMessage(messages, msg)
return []
# we've retrieved all the events we're going to with the last index_search!
if eventsOnly:
# no transaction search, just return the events
transaction_search = ""
else:
# this is it, find the transactions!
transaction_search = '| transaction name="%s" | search %s' % (tname, tconstraint)
search = index_search + transaction_search
log("running final search! %s" % search)
results = splunk.search.searchAll(search, **kwargs)
return results
def execute():
results = []
try:
results, dummyresults, settings = si.getOrganizedResults()
# default values
args = {'namespace': 'search'}
# get commandline args
keywords, options = si.getKeywordsAndOptions()
# override default args with settings from search kernel
args.update(settings)
# override default args with commandline args
args.update(options)
sessionKey = args.get("sessionKey", None)
owner = args.get("owner", 'admin')
namespace = args.get("namespace", None)
if namespace.lower() == "none":
namespace = None
messages = {}
if sessionKey == None:
# this shouldn't happen, but it's useful for testing.
try:
sessionKey = sa.getSessionKey('admin', 'changeme')
si.addWarnMessage(
messages,
"No session given to 'tune' command. Using default admin account and password."
)
except splunk.AuthenticationFailed, e:
si.addErrorMessage(messages,
"No session given to 'tune' command.")
return
if len(keywords) != 1:
usage()
# e.g., '/data/inputs/monitor'
entity = keywords[0]
logger.info("Entity: %s Args: %s" % (entity, args))
results = [] # we don't care about incoming results
try:
entitys = en.getEntities(entity,
sessionKey=sessionKey,
owner=owner,
namespace=namespace,
count=-1)
for name, entity in entitys.items():
try:
myapp = entity["eai:acl"]["app"]
if namespace != None and myapp != namespace:
continue
except:
continue # if no eai:acl/app, filter out
result = entityToResult(name, entity)
results.append(result)
except splunk.ResourceNotFound, e2:
pass
