def didYouMeanCommands(bnf, search): output = [] # get list of public commands knownCommands = utils.getAllCommands(bnf) # get list of commands user entered userCommandsAndArgs = utils.getCommands(search, None)[-1:] # just last searchCommands = [c.strip() for search in userCommandsAndArgs for c, a in search ] # get mapping of tags to commands tagmap = getTagsToCommands(bnf, knownCommands) # for each command user entered for searchCommand in searchCommands: # if not known, suggest something if not searchCommand in knownCommands: suggestion = getSuggestions(knownCommands, searchCommand, tagmap) if suggestion != "": output.extend(suggestion) #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion})) return output, knownCommands
def getLiterals(stanzas): literals = {} commands = utils.getAllCommands(stanzas) for command in commands: myliterals = set() cname = command+"-command" if cname not in stanzas: continue unexpanded = {} stanza = stanzas[cname] syntax = recurseSyntax(command, stanzas, stanza, unexpanded, False) syntax = cleanSyntax(syntax) uppers = re.findall("[A-Z][A-Z_-]+(?=[^a-zA-Z])", syntax) myliterals.update(uppers) quoted = re.findall('"(.*?)"', syntax) myliterals.update(quoted) terms = re.findall("(?<![<a-zA-Z0-9_-])([a-zA-Z0-9_-]+)", syntax) myliterals.update(terms) literals[command] = myliterals return literals
def didYouMeanCommands(bnf, search): output = [] # get list of public commands knownCommands = utils.getAllCommands(bnf) # get list of commands user entered userCommandsAndArgs = utils.getCommands(search, None)[-1:] # just last searchCommands = [ c.strip() for search in userCommandsAndArgs for c, a in search ] # get mapping of tags to commands tagmap = getTagsToCommands(bnf, knownCommands) # for each command user entered for searchCommand in searchCommands: # if not known, suggest something if not searchCommand in knownCommands: suggestion = getSuggestions(knownCommands, searchCommand, tagmap) if suggestion != "": output.extend( suggestion ) #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion})) return output, knownCommands
def doHelp(sessionKey, namespace, user, search, insertpos=None, earliest_time=None, latest_time=None, count=10, max_time=None, servers=None, useTypeahead=False, showCommandHelp=True, showCommandHistory=True, showFieldInfo=True): """ "did you mean ___?" "did you know ___?" "the 'sort' operator takes blah arguments and does blah" "you might also be interested in ___?" "the fields ___ can help narrow does these results" "these past searches are similar to your search" "these saved searches are similar to your search" "you are searching for ip and host and then deduplicating by host" "your search would be faster if you ..." """ originalsearch = search if insertpos == None: # no insertion point, use end insertpos = len(search) else: try: insertpos = int(insertpos) except: insertpos = len(search) search = search[:insertpos].strip() if search == "": search = "| search" elif not search.startswith("|"): search = "| " + search usersquery = originalsearch if usersquery.startswith("search "): usersquery = usersquery[len("search "):] queryprefix = utils.allButLast(usersquery) # defaults output = { 'notices': [], 'fields': [], 'args': [], 'nexts': [], 'autonexts':[], 'autocomplete':[], 'autocomplete_match':'', 'command':{}, 'typeahead': [], 'search': usersquery, 'searchprefix': queryprefix, 'allcommands': [], 'savedsearches': [], 'arg_typeahead':[], 'has_field_args':False} try: ## overallstart = start = time.time() bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace) ################### ## now = time.time() ## timing_bnf = now - start ## start = now ################### output['allcommands'] = utils.getAllCommands(bnf, user, namespace) ################### ## now = time.time() ## timing_allcommands = now - start ## start = now ################### aliasMap = utils.getAliasMap(bnf) ################### ## now = time.time() ## timing_aliasmap = now - start ## start = now ################### if (splunk.util.normalizeBoolean(useTypeahead)): suggestSearchTypeahead(output, search, usersquery, count, max_time, earliest_time, latest_time, servers, namespace, user) ################### ## now = time.time() ## timing_typeahead = now - start ## start = now ################### firstTermShouldBeCommand(output, search, aliasMap) ################### ## now = time.time() ## timing_firstterm = now - start ## start = now ################### didYouMean.help(output, bnf, sessionKey, namespace, user, search, usersquery) ################### ## now = time.time() ## timing_didyoumean = now - start ## start = now ################### didYouKnow.help(output, aliasMap, user, search) ################### ## now = time.time() ## timing_didyouknow = now - start ## start = now ################### relatedPastSearches(output, user, search) ################### ## now = time.time() ## timing_relatedpastsearches = now - start ## start = now ################### relatedSearches(output, sessionKey, namespace, user, search) ################### ## now = time.time() ## timing_relatedsearches = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showCommandHelp)): commandHelp(output, user, search, aliasMap, bnf) ################### ## now = time.time() ## timing_commandhelp = now - start ## start = now ################### nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, splunk.util.normalizeBoolean(showCommandHistory)) ################### ## now = time.time() ## timing_nextcommand = now - start ## start = now ################### relatedTerms(output, user, search) ################### ## now = time.time() ## timing_relatedterms = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showFieldInfo)): fieldInfo.usefulFields(output, sessionKey, namespace, user, usersquery) ################### ## now = time.time() ## timing_usefulfields = now - start ## start = now ################### describeSearch(output, user, search) ################### ## now = time.time() ## timing_describesearch = now - start ## start = now ################### suggestOptimizations(output, user, search) ################### ## now = time.time() ## timing_optimize = now - start ## start = now ################### argTypeahead(output, sessionKey, namespace, user, bnf, search) ################### ## now = time.time() ## timing_argtypeahead = now - start ## start = now ################### ## overall_time = now - overallstart ## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields) ## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg)) except Exception, e: msg = "! Error in search assistant: %s" % e msg += traceback.format_exc() output['notices'].insert(0,msg) logger.error(msg)
def nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, showargs): ## overallstart = start = time.time() ## timing_last_command = 0 ## timing_all_commands = 0 ## timing_get_next_data = 0 ## timing_get_args = 0 ## timing_add_commands = 0 ## timing_past_matches = 0 ## timing_sort_past_matches = 0 atPipe = False # if search ends in "|", don't give args for last command, but # give next information from previous commandd if search[-1] == "|": search = search[:-1] showargs = False atPipe = True nextcommands = [] typeaheadcommands = [] commandAndArgs = utils.getLastCommand(search, aliasMap) ################### ## now = time.time() ## timing_last_command = now - start ## start = now ################### if commandAndArgs == None: # list all generating commands. # make a list() copy so we don't trash it by adding search typeaheadcommands = list(utils.getAllCommands(bnf, user, namespace, True)) ################### ## now = time.time() ## timing_all_commands = now - start ## start = now ################### else: command, args = commandAndArgs data, pastsearches = next.getNextData(user, bnf, sessionKey, namespace) ################### ## now = time.time() ## timing_get_next_data = now - start ## start = now ################### for datum in data: if datum['command'] == command: typeaheadcommands = [x for x,y in datum['nextcommands'] if x != "<RUN>"] if showargs: matchingargs = [] fs = fuzzSearch(usersquery) for arg, perc in datum['args']: replacement = "%s | %s %s" % (queryprefix, command, arg) fr = fuzzSearch(replacement) if fr.startswith(fs) and fr != fs: matchingargs.append((arg,perc)) output['args'] = matchingargs #output['args'] = datum['args'] break ################### ## now = time.time() ## timing_get_args = now - start ## start = now ################### # now add in all commands that were not already added if command in aliasMap: # adding all the other commands not already added for thiscommand in utils.getAllCommands(bnf, user, namespace): if thiscommand not in nextcommands and thiscommand not in typeaheadcommands: nextcommands.append(thiscommand) ################### ## now = time.time() ## timing_add_commands = now - start ## start = now ################### # look for pastsearches that the current search is a subset of (like firefox url autocomplete looking for any term) usersearch = normalizeSearch(search) # if user didn't enter anything don't match on "search" or "search *", just get most recent if usersearch == "" or usersearch == "*": pastMatches = [userifySearch(p) for p in pastsearches] else: pastMatches = [userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(True, usersquery, pastsearch)] pastMatches.extend([userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(False, usersquery, pastsearch)]) ################### ## now = time.time() ## timing_past_matches = now - start ## start = now ################### # dedup pastMatches = sorted(list(set(pastMatches)), key=pastMatches.index) output['autocomplete'] = pastMatches[:10] # just the 10 most recent output['autocomplete_match'] = usersearch ################### ## now = time.time() ## timing_sort_past_matches = now - start ## start = now ################### # new. only show next command if we aren't showing the args for the current command. # use will see next commands when they type "|" if atPipe: # keep only those that alias to themselves -- i.e., don't show aliases nextcommands = [x for x in nextcommands if aliasMap.get(x, '') == x] typeaheadcommands = [x for x in typeaheadcommands if aliasMap.get(x, '') == x] s = usersquery.strip() if '|' in s: s = s[:s.rindex('|')].strip() # make triplets of (command, description, replacement) output['autonexts'] = [(x, utils.getAttr(bnf,x,"shortdesc",""), s + " | " + x) for x in typeaheadcommands] output['nexts'] = [(x, utils.getAttr(bnf,x,"shortdesc",""), s + " | " + x) for x in nextcommands]
def doHelp(sessionKey, namespace, user, search, insertpos=None, earliest_time=None, latest_time=None, count=10, max_time=None, servers=None, useTypeahead=False, showCommandHelp=True, showCommandHistory=True, showFieldInfo=True): """ "did you mean ___?" "did you know ___?" "the 'sort' operator takes blah arguments and does blah" "you might also be interested in ___?" "the fields ___ can help narrow does these results" "these past searches are similar to your search" "these saved searches are similar to your search" "you are searching for ip and host and then deduplicating by host" "your search would be faster if you ..." """ originalsearch = search if insertpos == None: # no insertion point, use end insertpos = len(search) else: try: insertpos = int(insertpos) except: insertpos = len(search) search = search[:insertpos].strip() if search == "": search = "| search" elif not search.startswith("|"): search = "| " + search usersquery = originalsearch if usersquery.startswith("search "): usersquery = usersquery[len("search "):] queryprefix = utils.allButLast(usersquery) # defaults output = { 'notices': [], 'fields': [], 'args': [], 'nexts': [], 'autonexts': [], 'autocomplete': [], 'autocomplete_match': '', 'command': {}, 'typeahead': [], 'search': usersquery, 'searchprefix': queryprefix, 'allcommands': [], 'savedsearches': [], 'arg_typeahead': [], 'has_field_args': False } try: ## overallstart = start = time.time() bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace) ################### ## now = time.time() ## timing_bnf = now - start ## start = now ################### output['allcommands'] = utils.getAllCommands(bnf, user, namespace) ################### ## now = time.time() ## timing_allcommands = now - start ## start = now ################### aliasMap = utils.getAliasMap(bnf) ################### ## now = time.time() ## timing_aliasmap = now - start ## start = now ################### if (splunk.util.normalizeBoolean(useTypeahead)): suggestSearchTypeahead(output, search, usersquery, count, max_time, earliest_time, latest_time, servers, namespace, user) ################### ## now = time.time() ## timing_typeahead = now - start ## start = now ################### firstTermShouldBeCommand(output, search, aliasMap) ################### ## now = time.time() ## timing_firstterm = now - start ## start = now ################### didYouMean.help(output, bnf, sessionKey, namespace, user, search, usersquery) ################### ## now = time.time() ## timing_didyoumean = now - start ## start = now ################### didYouKnow.help(output, aliasMap, user, search) ################### ## now = time.time() ## timing_didyouknow = now - start ## start = now ################### relatedPastSearches(output, user, search) ################### ## now = time.time() ## timing_relatedpastsearches = now - start ## start = now ################### relatedSearches(output, sessionKey, namespace, user, search) ################### ## now = time.time() ## timing_relatedsearches = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showCommandHelp)): commandHelp(output, user, search, aliasMap, bnf) ################### ## now = time.time() ## timing_commandhelp = now - start ## start = now ################### nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, splunk.util.normalizeBoolean(showCommandHistory)) ################### ## now = time.time() ## timing_nextcommand = now - start ## start = now ################### relatedTerms(output, user, search) ################### ## now = time.time() ## timing_relatedterms = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showFieldInfo)): fieldInfo.usefulFields(output, sessionKey, namespace, user, usersquery) ################### ## now = time.time() ## timing_usefulfields = now - start ## start = now ################### describeSearch(output, user, search) ################### ## now = time.time() ## timing_describesearch = now - start ## start = now ################### suggestOptimizations(output, user, search) ################### ## now = time.time() ## timing_optimize = now - start ## start = now ################### argTypeahead(output, sessionKey, namespace, user, bnf, search) ################### ## now = time.time() ## timing_argtypeahead = now - start ## start = now ################### ## overall_time = now - overallstart ## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields) ## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg)) except Exception, e: msg = "! Error in search assistant: %s" % e msg += traceback.format_exc() output['notices'].insert(0, msg) logger.error(msg)
def nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, showargs): ## overallstart = start = time.time() ## timing_last_command = 0 ## timing_all_commands = 0 ## timing_get_next_data = 0 ## timing_get_args = 0 ## timing_add_commands = 0 ## timing_past_matches = 0 ## timing_sort_past_matches = 0 atPipe = False # if search ends in "|", don't give args for last command, but # give next information from previous commandd if search[-1] == "|": search = search[:-1] showargs = False atPipe = True nextcommands = [] typeaheadcommands = [] commandAndArgs = utils.getLastCommand(search, aliasMap) ################### ## now = time.time() ## timing_last_command = now - start ## start = now ################### if commandAndArgs == None: # list all generating commands. # make a list() copy so we don't trash it by adding search typeaheadcommands = list( utils.getAllCommands(bnf, user, namespace, True)) ################### ## now = time.time() ## timing_all_commands = now - start ## start = now ################### else: command, args = commandAndArgs data, pastsearches = next.getNextData(user, bnf, sessionKey, namespace) ################### ## now = time.time() ## timing_get_next_data = now - start ## start = now ################### for datum in data: if datum['command'] == command: typeaheadcommands = [ x for x, y in datum['nextcommands'] if x != "<RUN>" ] if showargs: matchingargs = [] fs = fuzzSearch(usersquery) for arg, perc in datum['args']: replacement = "%s | %s %s" % (queryprefix, command, arg) fr = fuzzSearch(replacement) if fr.startswith(fs) and fr != fs: matchingargs.append((arg, perc)) output['args'] = matchingargs #output['args'] = datum['args'] break ################### ## now = time.time() ## timing_get_args = now - start ## start = now ################### # now add in all commands that were not already added if command in aliasMap: # adding all the other commands not already added for thiscommand in utils.getAllCommands(bnf, user, namespace): if thiscommand not in nextcommands and thiscommand not in typeaheadcommands: nextcommands.append(thiscommand) ################### ## now = time.time() ## timing_add_commands = now - start ## start = now ################### # look for pastsearches that the current search is a subset of (like firefox url autocomplete looking for any term) usersearch = normalizeSearch(search) # if user didn't enter anything don't match on "search" or "search *", just get most recent if usersearch == "" or usersearch == "*": pastMatches = [userifySearch(p) for p in pastsearches] else: pastMatches = [ userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(True, usersquery, pastsearch) ] pastMatches.extend([ userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(False, usersquery, pastsearch) ]) ################### ## now = time.time() ## timing_past_matches = now - start ## start = now ################### # dedup pastMatches = sorted(list(set(pastMatches)), key=pastMatches.index) output['autocomplete'] = pastMatches[:10] # just the 10 most recent output['autocomplete_match'] = usersearch ################### ## now = time.time() ## timing_sort_past_matches = now - start ## start = now ################### # new. only show next command if we aren't showing the args for the current command. # use will see next commands when they type "|" if atPipe: # keep only those that alias to themselves -- i.e., don't show aliases nextcommands = [x for x in nextcommands if aliasMap.get(x, '') == x] typeaheadcommands = [ x for x in typeaheadcommands if aliasMap.get(x, '') == x ] s = usersquery.strip() if '|' in s: s = s[:s.rindex('|')].strip() # make triplets of (command, description, replacement) output['autonexts'] = [(x, utils.getAttr(bnf, x, "shortdesc", ""), s + " | " + x) for x in typeaheadcommands] output['nexts'] = [(x, utils.getAttr(bnf, x, "shortdesc", ""), s + " | " + x) for x in nextcommands]