def reallyGetNextData(user, stanzas, sessionKey, namespace): commandResults = [] nextmap = {} commoncounts = {} argsmap = {} literals = describer.getLiterals(stanzas, user, namespace) bootstrapSearches, userSearches = getPastSearches(user, sessionKey, namespace) searches = bootstrapSearches + userSearches searches = searches[-MAX_HISTORY:] aliasMap = utils.getAliasMap(stanzas) badCommands = set() # for each search in file for search in searches: commandseqs = utils.getCommands(search, aliasMap) # for each sequency of commands for that search for j, commands in enumerate(commandseqs): commands.append((END,"")) # for each command for i, commandarg in enumerate(commands): command, arg = commandarg if command not in literals: if command != END: badCommands.add(command) arg = arg.strip() if command == END: break addCount(argsmap, command, arg) addCount(nextmap, command, commands[i+1][0]) addCommonCount(commoncounts, command) if len(badCommands) > 0: logger.warn("No searchbnf for these commands: %s!" % list(badCommands)) commandAndCounts = commoncounts.items() commandAndCounts.sort( lambda x, y: y[1] - x[1] ) for command,count in commandAndCounts: thisdata = {} commandResults.append(thisdata) thisdata['command'] = command thisdata['count'] = count thisargs = thisdata['args'] = [] thisnexts = thisdata['nextcommands'] = [] addSortedValueAndCounts(thisargs, argsmap[command]) addSortedValueAndCounts(thisnexts, nextmap[command]) return commandResults, userSearches
def reallyGetNextData(user, stanzas, sessionKey, namespace): commandResults = [] nextmap = {} commoncounts = {} argsmap = {} literals = describer.getLiterals(stanzas) bootstrapSearches, userSearches = getPastSearches(user, sessionKey, namespace) searches = bootstrapSearches + userSearches searches = searches[-MAX_HISTORY:] aliasMap = utils.getAliasMap(stanzas) badCommands = set() # for each search in file for search in searches: commandseqs = utils.getCommands(search, aliasMap) # for each sequency of commands for that search for j, commands in enumerate(commandseqs): commands.append((END,"")) # for each command for i, commandarg in enumerate(commands): command, arg = commandarg if command not in literals: if command != END: badCommands.add(command) arg = arg.strip() if command == END: break addCount(argsmap, command, arg) addCount(nextmap, command, commands[i+1][0]) addCommonCount(commoncounts, command) if len(badCommands) > 0: logger.warn("No searchbnf for these commands: %s!" % list(badCommands)) commandAndCounts = commoncounts.items() commandAndCounts.sort( lambda x, y: y[1] - x[1] ) for command,count in commandAndCounts: thisdata = {} commandResults.append(thisdata) thisdata['command'] = command thisdata['count'] = count thisargs = thisdata['args'] = [] thisnexts = thisdata['nextcommands'] = [] addSortedValueAndCounts(thisargs, argsmap[command]) addSortedValueAndCounts(thisnexts, nextmap[command]) return commandResults, userSearches
def doHelp(sessionKey, namespace, user, search, insertpos=None, earliest_time=None, latest_time=None, count=10, max_time=None, servers=None, useTypeahead=False, showCommandHelp=True, showCommandHistory=True, showFieldInfo=True): """ "did you mean ___?" "did you know ___?" "the 'sort' operator takes blah arguments and does blah" "you might also be interested in ___?" "the fields ___ can help narrow does these results" "these past searches are similar to your search" "these saved searches are similar to your search" "you are searching for ip and host and then deduplicating by host" "your search would be faster if you ..." """ originalsearch = search if insertpos == None: # no insertion point, use end insertpos = len(search) else: try: insertpos = int(insertpos) except: insertpos = len(search) search = search[:insertpos].strip() if search == "": search = "| search" elif not search.startswith("|"): search = "| " + search usersquery = originalsearch if usersquery.startswith("search "): usersquery = usersquery[len("search "):] queryprefix = utils.allButLast(usersquery) # defaults output = { 'notices': [], 'fields': [], 'args': [], 'nexts': [], 'autonexts':[], 'autocomplete':[], 'autocomplete_match':'', 'command':{}, 'typeahead': [], 'search': usersquery, 'searchprefix': queryprefix, 'allcommands': [], 'savedsearches': [], 'arg_typeahead':[], 'has_field_args':False} try: ## overallstart = start = time.time() bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace) ################### ## now = time.time() ## timing_bnf = now - start ## start = now ################### output['allcommands'] = utils.getAllCommands(bnf, user, namespace) ################### ## now = time.time() ## timing_allcommands = now - start ## start = now ################### aliasMap = utils.getAliasMap(bnf) ################### ## now = time.time() ## timing_aliasmap = now - start ## start = now ################### if (splunk.util.normalizeBoolean(useTypeahead)): suggestSearchTypeahead(output, search, usersquery, count, max_time, earliest_time, latest_time, servers, namespace, user) ################### ## now = time.time() ## timing_typeahead = now - start ## start = now ################### firstTermShouldBeCommand(output, search, aliasMap) ################### ## now = time.time() ## timing_firstterm = now - start ## start = now ################### didYouMean.help(output, bnf, sessionKey, namespace, user, search, usersquery) ################### ## now = time.time() ## timing_didyoumean = now - start ## start = now ################### didYouKnow.help(output, aliasMap, user, search) ################### ## now = time.time() ## timing_didyouknow = now - start ## start = now ################### relatedPastSearches(output, user, search) ################### ## now = time.time() ## timing_relatedpastsearches = now - start ## start = now ################### relatedSearches(output, sessionKey, namespace, user, search) ################### ## now = time.time() ## timing_relatedsearches = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showCommandHelp)): commandHelp(output, user, search, aliasMap, bnf) ################### ## now = time.time() ## timing_commandhelp = now - start ## start = now ################### nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, splunk.util.normalizeBoolean(showCommandHistory)) ################### ## now = time.time() ## timing_nextcommand = now - start ## start = now ################### relatedTerms(output, user, search) ################### ## now = time.time() ## timing_relatedterms = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showFieldInfo)): fieldInfo.usefulFields(output, sessionKey, namespace, user, usersquery) ################### ## now = time.time() ## timing_usefulfields = now - start ## start = now ################### describeSearch(output, user, search) ################### ## now = time.time() ## timing_describesearch = now - start ## start = now ################### suggestOptimizations(output, user, search) ################### ## now = time.time() ## timing_optimize = now - start ## start = now ################### argTypeahead(output, sessionKey, namespace, user, bnf, search) ################### ## now = time.time() ## timing_argtypeahead = now - start ## start = now ################### ## overall_time = now - overallstart ## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields) ## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg)) except Exception, e: msg = "! Error in search assistant: %s" % e msg += traceback.format_exc() output['notices'].insert(0,msg) logger.error(msg)
def doHelp(sessionKey, namespace, user, search, insertpos=None, earliest_time=None, latest_time=None, count=10, max_time=None, servers=None, useTypeahead=False, showCommandHelp=True, showCommandHistory=True, showFieldInfo=True): """ "did you mean ___?" "did you know ___?" "the 'sort' operator takes blah arguments and does blah" "you might also be interested in ___?" "the fields ___ can help narrow does these results" "these past searches are similar to your search" "these saved searches are similar to your search" "you are searching for ip and host and then deduplicating by host" "your search would be faster if you ..." """ originalsearch = search if insertpos == None: # no insertion point, use end insertpos = len(search) else: try: insertpos = int(insertpos) except: insertpos = len(search) search = search[:insertpos].strip() if search == "": search = "| search" elif not search.startswith("|"): search = "| " + search usersquery = originalsearch if usersquery.startswith("search "): usersquery = usersquery[len("search "):] queryprefix = utils.allButLast(usersquery) # defaults output = { 'notices': [], 'fields': [], 'args': [], 'nexts': [], 'autonexts': [], 'autocomplete': [], 'autocomplete_match': '', 'command': {}, 'typeahead': [], 'search': usersquery, 'searchprefix': queryprefix, 'allcommands': [], 'savedsearches': [], 'arg_typeahead': [], 'has_field_args': False } try: ## overallstart = start = time.time() bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace) ################### ## now = time.time() ## timing_bnf = now - start ## start = now ################### output['allcommands'] = utils.getAllCommands(bnf, user, namespace) ################### ## now = time.time() ## timing_allcommands = now - start ## start = now ################### aliasMap = utils.getAliasMap(bnf) ################### ## now = time.time() ## timing_aliasmap = now - start ## start = now ################### if (splunk.util.normalizeBoolean(useTypeahead)): suggestSearchTypeahead(output, search, usersquery, count, max_time, earliest_time, latest_time, servers, namespace, user) ################### ## now = time.time() ## timing_typeahead = now - start ## start = now ################### firstTermShouldBeCommand(output, search, aliasMap) ################### ## now = time.time() ## timing_firstterm = now - start ## start = now ################### didYouMean.help(output, bnf, sessionKey, namespace, user, search, usersquery) ################### ## now = time.time() ## timing_didyoumean = now - start ## start = now ################### didYouKnow.help(output, aliasMap, user, search) ################### ## now = time.time() ## timing_didyouknow = now - start ## start = now ################### relatedPastSearches(output, user, search) ################### ## now = time.time() ## timing_relatedpastsearches = now - start ## start = now ################### relatedSearches(output, sessionKey, namespace, user, search) ################### ## now = time.time() ## timing_relatedsearches = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showCommandHelp)): commandHelp(output, user, search, aliasMap, bnf) ################### ## now = time.time() ## timing_commandhelp = now - start ## start = now ################### nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, splunk.util.normalizeBoolean(showCommandHistory)) ################### ## now = time.time() ## timing_nextcommand = now - start ## start = now ################### relatedTerms(output, user, search) ################### ## now = time.time() ## timing_relatedterms = now - start ## start = now ################### if (splunk.util.normalizeBoolean(showFieldInfo)): fieldInfo.usefulFields(output, sessionKey, namespace, user, usersquery) ################### ## now = time.time() ## timing_usefulfields = now - start ## start = now ################### describeSearch(output, user, search) ################### ## now = time.time() ## timing_describesearch = now - start ## start = now ################### suggestOptimizations(output, user, search) ################### ## now = time.time() ## timing_optimize = now - start ## start = now ################### argTypeahead(output, sessionKey, namespace, user, bnf, search) ################### ## now = time.time() ## timing_argtypeahead = now - start ## start = now ################### ## overall_time = now - overallstart ## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields) ## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg)) except Exception, e: msg = "! Error in search assistant: %s" % e msg += traceback.format_exc() output['notices'].insert(0, msg) logger.error(msg)