def didYouMeanCommands(bnf, search):
output = []
# get list of public commands
knownCommands = utils.getAllCommands(bnf)
# get list of commands user entered
userCommandsAndArgs = utils.getCommands(search, None)[-1:] # just last
searchCommands = [c.strip() for search in userCommandsAndArgs for c, a in search ]
# get mapping of tags to commands
tagmap = getTagsToCommands(bnf, knownCommands)
# for each command user entered
for searchCommand in searchCommands:
# if not known, suggest something
if not searchCommand in knownCommands:
suggestion = getSuggestions(knownCommands, searchCommand, tagmap)
if suggestion != "":
output.extend(suggestion) #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion}))
return output, knownCommands
def getLiterals(stanzas):
literals = {}
commands = utils.getAllCommands(stanzas)
for command in commands:
myliterals = set()
cname = command+"-command"
if cname not in stanzas:
continue
unexpanded = {}
stanza = stanzas[cname]
syntax = recurseSyntax(command, stanzas, stanza, unexpanded, False)
syntax = cleanSyntax(syntax)
uppers = re.findall("[A-Z][A-Z_-]+(?=[^a-zA-Z])", syntax)
myliterals.update(uppers)
quoted = re.findall('"(.*?)"', syntax)
myliterals.update(quoted)
terms = re.findall("(?<![<a-zA-Z0-9_-])([a-zA-Z0-9_-]+)", syntax)
myliterals.update(terms)
literals[command] = myliterals
return literals
def didYouMeanCommands(bnf, search):
output = []
# get list of public commands
knownCommands = utils.getAllCommands(bnf)
# get list of commands user entered
userCommandsAndArgs = utils.getCommands(search, None)[-1:] # just last
searchCommands = [
c.strip() for search in userCommandsAndArgs for c, a in search
]
# get mapping of tags to commands
tagmap = getTagsToCommands(bnf, knownCommands)
# for each command user entered
for searchCommand in searchCommands:
# if not known, suggest something
if not searchCommand in knownCommands:
suggestion = getSuggestions(knownCommands, searchCommand, tagmap)
if suggestion != "":
output.extend(
suggestion
) #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion}))
return output, knownCommands
def doHelp(sessionKey, namespace, user, search, insertpos=None, earliest_time=None, latest_time=None, count=10, max_time=None, servers=None,
useTypeahead=False, showCommandHelp=True, showCommandHistory=True, showFieldInfo=True):
"""
"did you mean ___?"
"did you know ___?"
"the 'sort' operator takes blah arguments and does blah"
"you might also be interested in ___?"
"the fields ___ can help narrow does these results"
"these past searches are similar to your search"
"these saved searches are similar to your search"
"you are searching for ip and host and then deduplicating by host"
"your search would be faster if you ..."
"""
originalsearch = search
if insertpos == None: # no insertion point, use end
insertpos = len(search)
else:
try:
insertpos = int(insertpos)
except:
insertpos = len(search)
search = search[:insertpos].strip()
if search == "":
search = "| search"
elif not search.startswith("|"):
search = "| " + search
usersquery = originalsearch
if usersquery.startswith("search "):
usersquery = usersquery[len("search "):]
queryprefix = utils.allButLast(usersquery)
# defaults
output = { 'notices': [], 'fields': [], 'args': [], 'nexts': [], 'autonexts':[], 'autocomplete':[], 'autocomplete_match':'', 'command':{}, 'typeahead': [],
'search': usersquery, 'searchprefix': queryprefix, 'allcommands': [], 'savedsearches': [], 'arg_typeahead':[], 'has_field_args':False}
try:
## overallstart = start = time.time()
bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)
###################
## now = time.time()
## timing_bnf = now - start
## start = now
###################
output['allcommands'] = utils.getAllCommands(bnf, user, namespace)
###################
## now = time.time()
## timing_allcommands = now - start
## start = now
###################
aliasMap = utils.getAliasMap(bnf)
###################
## now = time.time()
## timing_aliasmap = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(useTypeahead)):
suggestSearchTypeahead(output, search, usersquery, count, max_time, earliest_time, latest_time, servers, namespace, user)
###################
## now = time.time()
## timing_typeahead = now - start
## start = now
###################
firstTermShouldBeCommand(output, search, aliasMap)
###################
## now = time.time()
## timing_firstterm = now - start
## start = now
###################
didYouMean.help(output, bnf, sessionKey, namespace, user, search, usersquery)
###################
## now = time.time()
## timing_didyoumean = now - start
## start = now
###################
didYouKnow.help(output, aliasMap, user, search)
###################
## now = time.time()
## timing_didyouknow = now - start
## start = now
###################
relatedPastSearches(output, user, search)
###################
## now = time.time()
## timing_relatedpastsearches = now - start
## start = now
###################
relatedSearches(output, sessionKey, namespace, user, search)
###################
## now = time.time()
## timing_relatedsearches = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showCommandHelp)):
commandHelp(output, user, search, aliasMap, bnf)
###################
## now = time.time()
## timing_commandhelp = now - start
## start = now
###################
nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, splunk.util.normalizeBoolean(showCommandHistory))
###################
## now = time.time()
## timing_nextcommand = now - start
## start = now
###################
relatedTerms(output, user, search)
###################
## now = time.time()
## timing_relatedterms = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showFieldInfo)):
fieldInfo.usefulFields(output, sessionKey, namespace, user, usersquery)
###################
## now = time.time()
## timing_usefulfields = now - start
## start = now
###################
describeSearch(output, user, search)
###################
## now = time.time()
## timing_describesearch = now - start
## start = now
###################
suggestOptimizations(output, user, search)
###################
## now = time.time()
## timing_optimize = now - start
## start = now
###################
argTypeahead(output, sessionKey, namespace, user, bnf, search)
###################
## now = time.time()
## timing_argtypeahead = now - start
## start = now
###################
## overall_time = now - overallstart
## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields)
## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg))
except Exception, e:
msg = "! Error in search assistant: %s" % e
msg += traceback.format_exc()
output['notices'].insert(0,msg)
logger.error(msg)
def nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, showargs):
## overallstart = start = time.time()
## timing_last_command = 0
## timing_all_commands = 0
## timing_get_next_data = 0
## timing_get_args = 0
## timing_add_commands = 0
## timing_past_matches = 0
## timing_sort_past_matches = 0
atPipe = False
# if search ends in "|", don't give args for last command, but
# give next information from previous commandd
if search[-1] == "|":
search = search[:-1]
showargs = False
atPipe = True
nextcommands = []
typeaheadcommands = []
commandAndArgs = utils.getLastCommand(search, aliasMap)
###################
## now = time.time()
## timing_last_command = now - start
## start = now
###################
if commandAndArgs == None:
# list all generating commands.
# make a list() copy so we don't trash it by adding search
typeaheadcommands = list(utils.getAllCommands(bnf, user, namespace, True))
###################
## now = time.time()
## timing_all_commands = now - start
## start = now
###################
else:
command, args = commandAndArgs
data, pastsearches = next.getNextData(user, bnf, sessionKey, namespace)
###################
## now = time.time()
## timing_get_next_data = now - start
## start = now
###################
for datum in data:
if datum['command'] == command:
typeaheadcommands = [x for x,y in datum['nextcommands'] if x != "<RUN>"]
if showargs:
matchingargs = []
fs = fuzzSearch(usersquery)
for arg, perc in datum['args']:
replacement = "%s | %s %s" % (queryprefix, command, arg)
fr = fuzzSearch(replacement)
if fr.startswith(fs) and fr != fs:
matchingargs.append((arg,perc))
output['args'] = matchingargs
#output['args'] = datum['args']
break
###################
## now = time.time()
## timing_get_args = now - start
## start = now
###################
# now add in all commands that were not already added
if command in aliasMap:
# adding all the other commands not already added
for thiscommand in utils.getAllCommands(bnf, user, namespace):
if thiscommand not in nextcommands and thiscommand not in typeaheadcommands:
nextcommands.append(thiscommand)
###################
## now = time.time()
## timing_add_commands = now - start
## start = now
###################
# look for pastsearches that the current search is a subset of (like firefox url autocomplete looking for any term)
usersearch = normalizeSearch(search)
# if user didn't enter anything don't match on "search" or "search *", just get most recent
if usersearch == "" or usersearch == "*":
pastMatches = [userifySearch(p) for p in pastsearches]
else:
pastMatches = [userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(True, usersquery, pastsearch)]
pastMatches.extend([userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(False, usersquery, pastsearch)])
###################
## now = time.time()
## timing_past_matches = now - start
## start = now
###################
# dedup
pastMatches = sorted(list(set(pastMatches)), key=pastMatches.index)
output['autocomplete'] = pastMatches[:10] # just the 10 most recent
output['autocomplete_match'] = usersearch
###################
## now = time.time()
## timing_sort_past_matches = now - start
## start = now
###################
# new. only show next command if we aren't showing the args for the current command.
# use will see next commands when they type "|"
if atPipe:
# keep only those that alias to themselves -- i.e., don't show aliases
nextcommands = [x for x in nextcommands if aliasMap.get(x, '') == x]
typeaheadcommands = [x for x in typeaheadcommands if aliasMap.get(x, '') == x]
s = usersquery.strip()
if '|' in s:
s = s[:s.rindex('|')].strip()
# make triplets of (command, description, replacement)
output['autonexts'] = [(x, utils.getAttr(bnf,x,"shortdesc",""), s + " | " + x) for x in typeaheadcommands]
output['nexts'] = [(x, utils.getAttr(bnf,x,"shortdesc",""), s + " | " + x) for x in nextcommands]
def doHelp(sessionKey,
namespace,
user,
search,
insertpos=None,
earliest_time=None,
latest_time=None,
count=10,
max_time=None,
servers=None,
useTypeahead=False,
showCommandHelp=True,
showCommandHistory=True,
showFieldInfo=True):
"""
"did you mean ___?"
"did you know ___?"
"the 'sort' operator takes blah arguments and does blah"
"you might also be interested in ___?"
"the fields ___ can help narrow does these results"
"these past searches are similar to your search"
"these saved searches are similar to your search"
"you are searching for ip and host and then deduplicating by host"
"your search would be faster if you ..."
"""
originalsearch = search
if insertpos == None: # no insertion point, use end
insertpos = len(search)
else:
try:
insertpos = int(insertpos)
except:
insertpos = len(search)
search = search[:insertpos].strip()
if search == "":
search = "| search"
elif not search.startswith("|"):
search = "| " + search
usersquery = originalsearch
if usersquery.startswith("search "):
usersquery = usersquery[len("search "):]
queryprefix = utils.allButLast(usersquery)
# defaults
output = {
'notices': [],
'fields': [],
'args': [],
'nexts': [],
'autonexts': [],
'autocomplete': [],
'autocomplete_match': '',
'command': {},
'typeahead': [],
'search': usersquery,
'searchprefix': queryprefix,
'allcommands': [],
'savedsearches': [],
'arg_typeahead': [],
'has_field_args': False
}
try:
## overallstart = start = time.time()
bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)
###################
## now = time.time()
## timing_bnf = now - start
## start = now
###################
output['allcommands'] = utils.getAllCommands(bnf, user, namespace)
###################
## now = time.time()
## timing_allcommands = now - start
## start = now
###################
aliasMap = utils.getAliasMap(bnf)
###################
## now = time.time()
## timing_aliasmap = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(useTypeahead)):
suggestSearchTypeahead(output, search, usersquery, count, max_time,
earliest_time, latest_time, servers,
namespace, user)
###################
## now = time.time()
## timing_typeahead = now - start
## start = now
###################
firstTermShouldBeCommand(output, search, aliasMap)
###################
## now = time.time()
## timing_firstterm = now - start
## start = now
###################
didYouMean.help(output, bnf, sessionKey, namespace, user, search,
usersquery)
###################
## now = time.time()
## timing_didyoumean = now - start
## start = now
###################
didYouKnow.help(output, aliasMap, user, search)
###################
## now = time.time()
## timing_didyouknow = now - start
## start = now
###################
relatedPastSearches(output, user, search)
###################
## now = time.time()
## timing_relatedpastsearches = now - start
## start = now
###################
relatedSearches(output, sessionKey, namespace, user, search)
###################
## now = time.time()
## timing_relatedsearches = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showCommandHelp)):
commandHelp(output, user, search, aliasMap, bnf)
###################
## now = time.time()
## timing_commandhelp = now - start
## start = now
###################
nextCommand(output, sessionKey, namespace, user, search, usersquery,
queryprefix, aliasMap, bnf,
splunk.util.normalizeBoolean(showCommandHistory))
###################
## now = time.time()
## timing_nextcommand = now - start
## start = now
###################
relatedTerms(output, user, search)
###################
## now = time.time()
## timing_relatedterms = now - start
## start = now
###################
if (splunk.util.normalizeBoolean(showFieldInfo)):
fieldInfo.usefulFields(output, sessionKey, namespace, user,
usersquery)
###################
## now = time.time()
## timing_usefulfields = now - start
## start = now
###################
describeSearch(output, user, search)
###################
## now = time.time()
## timing_describesearch = now - start
## start = now
###################
suggestOptimizations(output, user, search)
###################
## now = time.time()
## timing_optimize = now - start
## start = now
###################
argTypeahead(output, sessionKey, namespace, user, bnf, search)
###################
## now = time.time()
## timing_argtypeahead = now - start
## start = now
###################
## overall_time = now - overallstart
## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields)
## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg))
except Exception, e:
msg = "! Error in search assistant: %s" % e
msg += traceback.format_exc()
output['notices'].insert(0, msg)
logger.error(msg)
def nextCommand(output, sessionKey, namespace, user, search, usersquery,
queryprefix, aliasMap, bnf, showargs):
## overallstart = start = time.time()
## timing_last_command = 0
## timing_all_commands = 0
## timing_get_next_data = 0
## timing_get_args = 0
## timing_add_commands = 0
## timing_past_matches = 0
## timing_sort_past_matches = 0
atPipe = False
# if search ends in "|", don't give args for last command, but
# give next information from previous commandd
if search[-1] == "|":
search = search[:-1]
showargs = False
atPipe = True
nextcommands = []
typeaheadcommands = []
commandAndArgs = utils.getLastCommand(search, aliasMap)
###################
## now = time.time()
## timing_last_command = now - start
## start = now
###################
if commandAndArgs == None:
# list all generating commands.
# make a list() copy so we don't trash it by adding search
typeaheadcommands = list(
utils.getAllCommands(bnf, user, namespace, True))
###################
## now = time.time()
## timing_all_commands = now - start
## start = now
###################
else:
command, args = commandAndArgs
data, pastsearches = next.getNextData(user, bnf, sessionKey, namespace)
###################
## now = time.time()
## timing_get_next_data = now - start
## start = now
###################
for datum in data:
if datum['command'] == command:
typeaheadcommands = [
x for x, y in datum['nextcommands'] if x != "<RUN>"
]
if showargs:
matchingargs = []
fs = fuzzSearch(usersquery)
for arg, perc in datum['args']:
replacement = "%s | %s %s" % (queryprefix, command,
arg)
fr = fuzzSearch(replacement)
if fr.startswith(fs) and fr != fs:
matchingargs.append((arg, perc))
output['args'] = matchingargs
#output['args'] = datum['args']
break
###################
## now = time.time()
## timing_get_args = now - start
## start = now
###################
# now add in all commands that were not already added
if command in aliasMap:
# adding all the other commands not already added
for thiscommand in utils.getAllCommands(bnf, user, namespace):
if thiscommand not in nextcommands and thiscommand not in typeaheadcommands:
nextcommands.append(thiscommand)
###################
## now = time.time()
## timing_add_commands = now - start
## start = now
###################
# look for pastsearches that the current search is a subset of (like firefox url autocomplete looking for any term)
usersearch = normalizeSearch(search)
# if user didn't enter anything don't match on "search" or "search *", just get most recent
if usersearch == "" or usersearch == "*":
pastMatches = [userifySearch(p) for p in pastsearches]
else:
pastMatches = [
userifySearch(pastsearch) for pastsearch in pastsearches
if normalizedSearchMatch(True, usersquery, pastsearch)
]
pastMatches.extend([
userifySearch(pastsearch) for pastsearch in pastsearches
if normalizedSearchMatch(False, usersquery, pastsearch)
])
###################
## now = time.time()
## timing_past_matches = now - start
## start = now
###################
# dedup
pastMatches = sorted(list(set(pastMatches)), key=pastMatches.index)
output['autocomplete'] = pastMatches[:10] # just the 10 most recent
output['autocomplete_match'] = usersearch
###################
## now = time.time()
## timing_sort_past_matches = now - start
## start = now
###################
# new. only show next command if we aren't showing the args for the current command.
# use will see next commands when they type "|"
if atPipe:
# keep only those that alias to themselves -- i.e., don't show aliases
nextcommands = [x for x in nextcommands if aliasMap.get(x, '') == x]
typeaheadcommands = [
x for x in typeaheadcommands if aliasMap.get(x, '') == x
]
s = usersquery.strip()
if '|' in s:
s = s[:s.rindex('|')].strip()
# make triplets of (command, description, replacement)
output['autonexts'] = [(x, utils.getAttr(bnf, x, "shortdesc",
""), s + " | " + x)
for x in typeaheadcommands]
output['nexts'] = [(x, utils.getAttr(bnf, x, "shortdesc",
""), s + " | " + x)
for x in nextcommands]