def reallyGetNextData(user, stanzas, sessionKey, namespace):
commandResults = []
nextmap = {}
commoncounts = {}
argsmap = {}
literals = describer.getLiterals(stanzas, user, namespace)
bootstrapSearches, userSearches = getPastSearches(user, sessionKey, namespace)
searches = bootstrapSearches + userSearches
searches = searches[-MAX_HISTORY:]
aliasMap = utils.getAliasMap(stanzas)
badCommands = set()
# for each search in file
for search in searches:
commandseqs = utils.getCommands(search, aliasMap)
# for each sequency of commands for that search
for j, commands in enumerate(commandseqs):
commands.append((END,""))
# for each command
for i, commandarg in enumerate(commands):
command, arg = commandarg
if command not in literals:
if command != END:
badCommands.add(command)
arg = arg.strip()
if command == END:
break
addCount(argsmap, command, arg)
addCount(nextmap, command, commands[i+1][0])
addCommonCount(commoncounts, command)
if len(badCommands) > 0:
logger.warn("No searchbnf for these commands: %s!" % list(badCommands))
commandAndCounts = commoncounts.items()
commandAndCounts.sort( lambda x, y: y[1] - x[1] )
for command,count in commandAndCounts:
thisdata = {}
commandResults.append(thisdata)
thisdata['command'] = command
thisdata['count'] = count
thisargs = thisdata['args'] = []
thisnexts = thisdata['nextcommands'] = []
addSortedValueAndCounts(thisargs, argsmap[command])
addSortedValueAndCounts(thisnexts, nextmap[command])
return commandResults, userSearches
def reallyGetNextData(user, stanzas, sessionKey, namespace):
commandResults = []
nextmap = {}
commoncounts = {}
argsmap = {}
literals = describer.getLiterals(stanzas)
bootstrapSearches, userSearches = getPastSearches(user, sessionKey, namespace)
searches = bootstrapSearches + userSearches
searches = searches[-MAX_HISTORY:]
aliasMap = utils.getAliasMap(stanzas)
badCommands = set()
# for each search in file
for search in searches:
commandseqs = utils.getCommands(search, aliasMap)
# for each sequency of commands for that search
for j, commands in enumerate(commandseqs):
commands.append((END,""))
# for each command
for i, commandarg in enumerate(commands):
command, arg = commandarg
if command not in literals:
if command != END:
badCommands.add(command)
arg = arg.strip()
if command == END:
break
addCount(argsmap, command, arg)
addCount(nextmap, command, commands[i+1][0])
addCommonCount(commoncounts, command)
if len(badCommands) > 0:
logger.warn("No searchbnf for these commands: %s!" % list(badCommands))
commandAndCounts = commoncounts.items()
commandAndCounts.sort( lambda x, y: y[1] - x[1] )
for command,count in commandAndCounts:
thisdata = {}
commandResults.append(thisdata)
thisdata['command'] = command
thisdata['count'] = count
thisargs = thisdata['args'] = []
thisnexts = thisdata['nextcommands'] = []
addSortedValueAndCounts(thisargs, argsmap[command])
addSortedValueAndCounts(thisnexts, nextmap[command])
return commandResults, userSearches
def doHelp(sessionKey, namespace, user, search, insertpos=None, earliest_time=None, latest_time=None, count=10, max_time=None, servers=None,
useTypeahead=False, showCommandHelp=True, showCommandHistory=True, showFieldInfo=True):
"""
"did you mean ___?"
"did you know ___?"
"the 'sort' operator takes blah arguments and does blah"
"you might also be interested in ___?"
"the fields ___ can help narrow does these results"
"these past searches are similar to your search"
"these saved searches are similar to your search"
"you are searching for ip and host and then deduplicating by host"
"your search would be faster if you ..."
"""
originalsearch = search
if insertpos == None: # no insertion point, use end
insertpos = len(search)
else:
try:
insertpos = int(insertpos)
except:
insertpos = len(search)
search = search[:insertpos].strip()
if search == "":
search = "| search"
elif not search.startswith("|"):
search = "| " + search
usersquery = originalsearch
if usersquery.startswith("search "):
usersquery = usersquery[len("search "):]
queryprefix = utils.allButLast(usersquery)
# defaults
output = { 'notices': [], 'fields': [], 'args': [], 'nexts': [], 'autonexts':[], 'autocomplete':[], 'autocomplete_match':'', 'command':{}, 'typeahead': [],
'search': usersquery, 'searchprefix': queryprefix, 'allcommands': [], 'savedsearches': [], 'arg_typeahead':[], 'has_field_args':False}
try:
## overallstart = start = time.time()
bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)
###################
## now = time.time()
## timing_bnf = now - start
## start = now
###################
output['allcommands'] = utils.getAllCommands(bnf, user, namespace)
###################
## now = time.time()
## timing_allcommands = now - start
## start = now
###################
aliasMap = utils.getAliasMap(bnf)
###################
## now = time.time()
## timing_aliasmap = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(useTypeahead)):
suggestSearchTypeahead(output, search, usersquery, count, max_time, earliest_time, latest_time, servers, namespace, user)
###################
## now = time.time()
## timing_typeahead = now - start
## start = now
###################
firstTermShouldBeCommand(output, search, aliasMap)
###################
## now = time.time()
## timing_firstterm = now - start
## start = now
###################
didYouMean.help(output, bnf, sessionKey, namespace, user, search, usersquery)
###################
## now = time.time()
## timing_didyoumean = now - start
## start = now
###################
didYouKnow.help(output, aliasMap, user, search)
###################
## now = time.time()
## timing_didyouknow = now - start
## start = now
###################
relatedPastSearches(output, user, search)
###################
## now = time.time()
## timing_relatedpastsearches = now - start
## start = now
###################
relatedSearches(output, sessionKey, namespace, user, search)
###################
## now = time.time()
## timing_relatedsearches = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showCommandHelp)):
commandHelp(output, user, search, aliasMap, bnf)
###################
## now = time.time()
## timing_commandhelp = now - start
## start = now
###################
nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, splunk.util.normalizeBoolean(showCommandHistory))
###################
## now = time.time()
## timing_nextcommand = now - start
## start = now
###################
relatedTerms(output, user, search)
###################
## now = time.time()
## timing_relatedterms = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showFieldInfo)):
fieldInfo.usefulFields(output, sessionKey, namespace, user, usersquery)
###################
## now = time.time()
## timing_usefulfields = now - start
## start = now
###################
describeSearch(output, user, search)
###################
## now = time.time()
## timing_describesearch = now - start
## start = now
###################
suggestOptimizations(output, user, search)
###################
## now = time.time()
## timing_optimize = now - start
## start = now
###################
argTypeahead(output, sessionKey, namespace, user, bnf, search)
###################
## now = time.time()
## timing_argtypeahead = now - start
## start = now
###################
## overall_time = now - overallstart
## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields)
## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg))
except Exception, e:
msg = "! Error in search assistant: %s" % e
msg += traceback.format_exc()
output['notices'].insert(0,msg)
logger.error(msg)
def doHelp(sessionKey,
namespace,
user,
search,
insertpos=None,
earliest_time=None,
latest_time=None,
count=10,
max_time=None,
servers=None,
useTypeahead=False,
showCommandHelp=True,
showCommandHistory=True,
showFieldInfo=True):
"""
"did you mean ___?"
"did you know ___?"
"the 'sort' operator takes blah arguments and does blah"
"you might also be interested in ___?"
"the fields ___ can help narrow does these results"
"these past searches are similar to your search"
"these saved searches are similar to your search"
"you are searching for ip and host and then deduplicating by host"
"your search would be faster if you ..."
"""
originalsearch = search
if insertpos == None: # no insertion point, use end
insertpos = len(search)
else:
try:
insertpos = int(insertpos)
except:
insertpos = len(search)
search = search[:insertpos].strip()
if search == "":
search = "| search"
elif not search.startswith("|"):
search = "| " + search
usersquery = originalsearch
if usersquery.startswith("search "):
usersquery = usersquery[len("search "):]
queryprefix = utils.allButLast(usersquery)
# defaults
output = {
'notices': [],
'fields': [],
'args': [],
'nexts': [],
'autonexts': [],
'autocomplete': [],
'autocomplete_match': '',
'command': {},
'typeahead': [],
'search': usersquery,
'searchprefix': queryprefix,
'allcommands': [],
'savedsearches': [],
'arg_typeahead': [],
'has_field_args': False
}
try:
## overallstart = start = time.time()
bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)
###################
## now = time.time()
## timing_bnf = now - start
## start = now
###################
output['allcommands'] = utils.getAllCommands(bnf, user, namespace)
###################
## now = time.time()
## timing_allcommands = now - start
## start = now
###################
aliasMap = utils.getAliasMap(bnf)
###################
## now = time.time()
## timing_aliasmap = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(useTypeahead)):
suggestSearchTypeahead(output, search, usersquery, count, max_time,
earliest_time, latest_time, servers,
namespace, user)
###################
## now = time.time()
## timing_typeahead = now - start
## start = now
###################
firstTermShouldBeCommand(output, search, aliasMap)
###################
## now = time.time()
## timing_firstterm = now - start
## start = now
###################
didYouMean.help(output, bnf, sessionKey, namespace, user, search,
usersquery)
###################
## now = time.time()
## timing_didyoumean = now - start
## start = now
###################
didYouKnow.help(output, aliasMap, user, search)
###################
## now = time.time()
## timing_didyouknow = now - start
## start = now
###################
relatedPastSearches(output, user, search)
###################
## now = time.time()
## timing_relatedpastsearches = now - start
## start = now
###################
relatedSearches(output, sessionKey, namespace, user, search)
###################
## now = time.time()
## timing_relatedsearches = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showCommandHelp)):
commandHelp(output, user, search, aliasMap, bnf)
###################
## now = time.time()
## timing_commandhelp = now - start
## start = now
###################
nextCommand(output, sessionKey, namespace, user, search, usersquery,
queryprefix, aliasMap, bnf,
splunk.util.normalizeBoolean(showCommandHistory))
###################
## now = time.time()
## timing_nextcommand = now - start
## start = now
###################
relatedTerms(output, user, search)
###################
## now = time.time()
## timing_relatedterms = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showFieldInfo)):
fieldInfo.usefulFields(output, sessionKey, namespace, user,
usersquery)
###################
## now = time.time()
## timing_usefulfields = now - start
## start = now
###################
describeSearch(output, user, search)
###################
## now = time.time()
## timing_describesearch = now - start
## start = now
###################
suggestOptimizations(output, user, search)
###################
## now = time.time()
## timing_optimize = now - start
## start = now
###################
argTypeahead(output, sessionKey, namespace, user, bnf, search)
###################
## now = time.time()
## timing_argtypeahead = now - start
## start = now
###################
## overall_time = now - overallstart
## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields)
## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg))
except Exception, e:
msg = "! Error in search assistant: %s" % e
msg += traceback.format_exc()
output['notices'].insert(0, msg)
logger.error(msg)
