Ejemplo n.º 1
0
def didYouMeanCommands(bnf, search):
    output = []
    # get list of public commands
    knownCommands = utils.getAllCommands(bnf)
    # get list of commands user entered
    userCommandsAndArgs = utils.getCommands(search, None)[-1:] # just last
    searchCommands = [c.strip() for search in userCommandsAndArgs for c, a in search ]
    # get mapping of tags to commands
    tagmap = getTagsToCommands(bnf, knownCommands)

    # for each command user entered 
    for searchCommand in searchCommands:
        # if not known, suggest something
        if not searchCommand in knownCommands:
            suggestion = getSuggestions(knownCommands, searchCommand, tagmap)
            if suggestion != "":
                output.extend(suggestion) #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion}))

    return output, knownCommands
Ejemplo n.º 2
0
def getLiterals(stanzas):
    literals = {}
    commands = utils.getAllCommands(stanzas)
    
    for command in commands:
        myliterals = set()
        cname = command+"-command"
        if cname not in stanzas:
            continue
        unexpanded = {}
        stanza = stanzas[cname]
        syntax = recurseSyntax(command, stanzas, stanza, unexpanded, False)
        syntax = cleanSyntax(syntax)
        uppers = re.findall("[A-Z][A-Z_-]+(?=[^a-zA-Z])", syntax)
        myliterals.update(uppers)
        quoted = re.findall('"(.*?)"', syntax)
        myliterals.update(quoted)
        terms = re.findall("(?<![<a-zA-Z0-9_-])([a-zA-Z0-9_-]+)", syntax)
        myliterals.update(terms)
        literals[command] = myliterals
    return literals
Ejemplo n.º 3
0
def didYouMeanCommands(bnf, search):
    output = []
    # get list of public commands
    knownCommands = utils.getAllCommands(bnf)
    # get list of commands user entered
    userCommandsAndArgs = utils.getCommands(search, None)[-1:]  # just last
    searchCommands = [
        c.strip() for search in userCommandsAndArgs for c, a in search
    ]
    # get mapping of tags to commands
    tagmap = getTagsToCommands(bnf, knownCommands)

    # for each command user entered
    for searchCommand in searchCommands:
        # if not known, suggest something
        if not searchCommand in knownCommands:
            suggestion = getSuggestions(knownCommands, searchCommand, tagmap)
            if suggestion != "":
                output.extend(
                    suggestion
                )  #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion}))

    return output, knownCommands
def doHelp(sessionKey, namespace, user, search, insertpos=None, earliest_time=None, latest_time=None, count=10, max_time=None, servers=None,
         useTypeahead=False, showCommandHelp=True, showCommandHistory=True, showFieldInfo=True):
    """
    "did you mean ___?"
    "did you know ___?"
    "the 'sort' operator takes blah arguments and does blah"
    "you might also be interested in ___?"
    "the fields ___ can help narrow does these results"
    "these past searches are similar to your search"
    "these saved searches are similar to your search"
    "you are searching for ip and host and then deduplicating by host"
    "your search would be faster if you ..."
    """

    originalsearch = search
    if insertpos == None: # no insertion point, use end
        insertpos = len(search)
    else:
        try:
            insertpos = int(insertpos)
        except:
            insertpos = len(search)

    search = search[:insertpos].strip()

    if search == "":
        search = "| search"
    elif not search.startswith("|"):
        search = "| " + search

    usersquery = originalsearch
    if usersquery.startswith("search "):
        usersquery = usersquery[len("search "):]
    queryprefix = utils.allButLast(usersquery)
    # defaults
    output = { 'notices': [], 'fields': [], 'args': [], 'nexts': [], 'autonexts':[], 'autocomplete':[], 'autocomplete_match':'', 'command':{}, 'typeahead': [],
               'search': usersquery, 'searchprefix': queryprefix, 'allcommands': [], 'savedsearches': [], 'arg_typeahead':[], 'has_field_args':False}
    try:
        
        ## overallstart = start = time.time()

        bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)

        ###################
        ## now = time.time()
        ## timing_bnf = now - start
        ## start = now
        ###################
        
        output['allcommands'] = utils.getAllCommands(bnf, user, namespace)

        ###################
        ## now = time.time()
        ## timing_allcommands = now - start
        ## start = now
        ###################
        
        aliasMap = utils.getAliasMap(bnf)

        ###################
        ## now = time.time()
        ## timing_aliasmap = now - start
        ## start = now
        ###################
        
        if (splunk.util.normalizeBoolean(useTypeahead)):
            suggestSearchTypeahead(output, search, usersquery, count, max_time, earliest_time, latest_time, servers, namespace, user)

        ###################
        ## now = time.time()
        ## timing_typeahead = now - start
        ## start = now
        ###################            
        
        firstTermShouldBeCommand(output, search, aliasMap)

        ###################
        ## now = time.time()
        ## timing_firstterm = now - start
        ## start = now
        ###################            
        
        didYouMean.help(output, bnf, sessionKey, namespace, user, search, usersquery)

        ###################
        ## now = time.time()
        ## timing_didyoumean = now - start
        ## start = now
        ###################            
        
        didYouKnow.help(output, aliasMap, user, search)

        ###################
        ## now = time.time()
        ## timing_didyouknow = now - start
        ## start = now
        ###################            
        
        relatedPastSearches(output, user, search)

        ###################
        ## now = time.time()
        ## timing_relatedpastsearches = now - start
        ## start = now
        ###################            
        
        relatedSearches(output, sessionKey, namespace, user, search)

        ###################
        ## now = time.time()
        ## timing_relatedsearches = now - start
        ## start = now
        ###################            

        if (splunk.util.normalizeBoolean(showCommandHelp)):
            commandHelp(output, user, search, aliasMap, bnf)

        ###################
        ## now = time.time()
        ## timing_commandhelp = now - start
        ## start = now
        ###################            
    
        nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, splunk.util.normalizeBoolean(showCommandHistory))

        ###################
        ## now = time.time()
        ## timing_nextcommand = now - start
        ## start = now
        ###################            
        
        relatedTerms(output, user, search)

        ###################
        ## now = time.time()
        ## timing_relatedterms = now - start
        ## start = now
        ###################            
        
        if (splunk.util.normalizeBoolean(showFieldInfo)):
            fieldInfo.usefulFields(output, sessionKey, namespace, user, usersquery)


        ###################
        ## now = time.time()
        ## timing_usefulfields = now - start
        ## start = now
        ###################            

            
        describeSearch(output, user, search)


        ###################
        ## now = time.time()
        ## timing_describesearch = now - start
        ## start = now
        ###################            


        suggestOptimizations(output, user, search)


        ###################
        ## now = time.time()
        ## timing_optimize = now - start
        ## start = now
        ###################            
        
        argTypeahead(output, sessionKey, namespace, user, bnf, search)

        ###################
        ## now = time.time()
        ## timing_argtypeahead = now - start
        ## start = now
        ###################            

        ## overall_time = now - overallstart
        ## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields)
        ## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg))
        
    except Exception, e:
        msg = "! Error in search assistant: %s" % e
        msg += traceback.format_exc()
        output['notices'].insert(0,msg)

        logger.error(msg)
def nextCommand(output, sessionKey, namespace, user, search, usersquery, queryprefix, aliasMap, bnf, showargs):

    ## overallstart = start = time.time()
    ## timing_last_command = 0
    ## timing_all_commands = 0
    ## timing_get_next_data = 0
    ## timing_get_args = 0
    ## timing_add_commands = 0
    ## timing_past_matches = 0
    ## timing_sort_past_matches = 0
    
    atPipe = False

    # if search ends in "|", don't give args for last command, but
    # give next information from previous commandd
    if search[-1] == "|":
        search = search[:-1]
        showargs = False
        atPipe = True
        
    nextcommands = []
    typeaheadcommands = []
    
    commandAndArgs = utils.getLastCommand(search, aliasMap)
    
    ###################
    ## now = time.time()
    ## timing_last_command = now - start
    ## start = now
    ###################

    
    if commandAndArgs == None:
        # list all generating commands.
        # make a list() copy so we don't trash it by adding search
        typeaheadcommands = list(utils.getAllCommands(bnf, user, namespace, True))

        ###################
        ## now = time.time()
        ## timing_all_commands = now - start
        ## start = now
        ###################

        
    else:        
        command, args = commandAndArgs
        data, pastsearches = next.getNextData(user, bnf, sessionKey, namespace)

        ###################
        ## now = time.time()
        ## timing_get_next_data = now - start
        ## start = now
        ###################

        
        for datum in data:
            if datum['command'] == command:
                typeaheadcommands = [x for x,y in datum['nextcommands'] if x != "<RUN>"]
                if showargs:

                    matchingargs = []
                    fs = fuzzSearch(usersquery)
                    for arg, perc in datum['args']:
                        replacement = "%s | %s %s" % (queryprefix, command, arg)
                        fr = fuzzSearch(replacement)
                        if fr.startswith(fs) and fr != fs:
                            matchingargs.append((arg,perc))
                    output['args'] =  matchingargs
                    #output['args'] =  datum['args']
                break

        ###################
        ## now = time.time()
        ## timing_get_args = now - start
        ## start = now
        ###################
            
        # now add in all commands that were not already added
        if command in aliasMap:
            # adding all the other commands not already added
            for thiscommand in utils.getAllCommands(bnf, user, namespace):
                if thiscommand not in nextcommands and thiscommand not in typeaheadcommands:
                    nextcommands.append(thiscommand)

        ###################
        ## now = time.time()
        ## timing_add_commands = now - start
        ## start = now
        ###################


        # look for pastsearches that the current search is a subset of  (like firefox url autocomplete looking for any term)
        usersearch = normalizeSearch(search)

        # if user didn't enter anything don't match on "search" or "search *", just get most recent
        if usersearch == "" or usersearch == "*":
            pastMatches = [userifySearch(p) for p in pastsearches]
        else:
            pastMatches = [userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(True, usersquery, pastsearch)]
            pastMatches.extend([userifySearch(pastsearch) for pastsearch in pastsearches if normalizedSearchMatch(False, usersquery, pastsearch)])

        ###################
        ## now = time.time()
        ## timing_past_matches = now - start
        ## start = now
        ###################
            
        # dedup
        pastMatches = sorted(list(set(pastMatches)), key=pastMatches.index)         
        output['autocomplete'] = pastMatches[:10] # just the 10 most recent
        output['autocomplete_match'] = usersearch

        ###################
        ## now = time.time()
        ## timing_sort_past_matches = now - start
        ## start = now
        ###################


    # new.  only show next command if we aren't showing the args for the current command.
    # use will see next commands when they type "|"
    if atPipe:
        # keep only those that alias to themselves -- i.e., don't show aliases
        nextcommands = [x for x in nextcommands if aliasMap.get(x, '') == x]
        typeaheadcommands = [x for x in typeaheadcommands if aliasMap.get(x, '') == x]
        s = usersquery.strip()
        if '|' in s:
            s = s[:s.rindex('|')].strip()
        # make triplets of (command, description, replacement)
        output['autonexts'] = [(x, utils.getAttr(bnf,x,"shortdesc",""), s + " | " + x) for x in typeaheadcommands]
        output['nexts'] = [(x, utils.getAttr(bnf,x,"shortdesc",""), s + " | " + x) for x in nextcommands]
def doHelp(sessionKey,
           namespace,
           user,
           search,
           insertpos=None,
           earliest_time=None,
           latest_time=None,
           count=10,
           max_time=None,
           servers=None,
           useTypeahead=False,
           showCommandHelp=True,
           showCommandHistory=True,
           showFieldInfo=True):
    """
    "did you mean ___?"
    "did you know ___?"
    "the 'sort' operator takes blah arguments and does blah"
    "you might also be interested in ___?"
    "the fields ___ can help narrow does these results"
    "these past searches are similar to your search"
    "these saved searches are similar to your search"
    "you are searching for ip and host and then deduplicating by host"
    "your search would be faster if you ..."
    """

    originalsearch = search
    if insertpos == None:  # no insertion point, use end
        insertpos = len(search)
    else:
        try:
            insertpos = int(insertpos)
        except:
            insertpos = len(search)

    search = search[:insertpos].strip()

    if search == "":
        search = "| search"
    elif not search.startswith("|"):
        search = "| " + search

    usersquery = originalsearch
    if usersquery.startswith("search "):
        usersquery = usersquery[len("search "):]
    queryprefix = utils.allButLast(usersquery)
    # defaults
    output = {
        'notices': [],
        'fields': [],
        'args': [],
        'nexts': [],
        'autonexts': [],
        'autocomplete': [],
        'autocomplete_match': '',
        'command': {},
        'typeahead': [],
        'search': usersquery,
        'searchprefix': queryprefix,
        'allcommands': [],
        'savedsearches': [],
        'arg_typeahead': [],
        'has_field_args': False
    }
    try:

        ## overallstart = start = time.time()

        bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)

        ###################
        ## now = time.time()
        ## timing_bnf = now - start
        ## start = now
        ###################

        output['allcommands'] = utils.getAllCommands(bnf, user, namespace)

        ###################
        ## now = time.time()
        ## timing_allcommands = now - start
        ## start = now
        ###################

        aliasMap = utils.getAliasMap(bnf)

        ###################
        ## now = time.time()
        ## timing_aliasmap = now - start
        ## start = now
        ###################

        if (splunk.util.normalizeBoolean(useTypeahead)):
            suggestSearchTypeahead(output, search, usersquery, count, max_time,
                                   earliest_time, latest_time, servers,
                                   namespace, user)

        ###################
        ## now = time.time()
        ## timing_typeahead = now - start
        ## start = now
        ###################

        firstTermShouldBeCommand(output, search, aliasMap)

        ###################
        ## now = time.time()
        ## timing_firstterm = now - start
        ## start = now
        ###################

        didYouMean.help(output, bnf, sessionKey, namespace, user, search,
                        usersquery)

        ###################
        ## now = time.time()
        ## timing_didyoumean = now - start
        ## start = now
        ###################

        didYouKnow.help(output, aliasMap, user, search)

        ###################
        ## now = time.time()
        ## timing_didyouknow = now - start
        ## start = now
        ###################

        relatedPastSearches(output, user, search)

        ###################
        ## now = time.time()
        ## timing_relatedpastsearches = now - start
        ## start = now
        ###################

        relatedSearches(output, sessionKey, namespace, user, search)

        ###################
        ## now = time.time()
        ## timing_relatedsearches = now - start
        ## start = now
        ###################

        if (splunk.util.normalizeBoolean(showCommandHelp)):
            commandHelp(output, user, search, aliasMap, bnf)

        ###################
        ## now = time.time()
        ## timing_commandhelp = now - start
        ## start = now
        ###################

        nextCommand(output, sessionKey, namespace, user, search, usersquery,
                    queryprefix, aliasMap, bnf,
                    splunk.util.normalizeBoolean(showCommandHistory))

        ###################
        ## now = time.time()
        ## timing_nextcommand = now - start
        ## start = now
        ###################

        relatedTerms(output, user, search)

        ###################
        ## now = time.time()
        ## timing_relatedterms = now - start
        ## start = now
        ###################

        if (splunk.util.normalizeBoolean(showFieldInfo)):
            fieldInfo.usefulFields(output, sessionKey, namespace, user,
                                   usersquery)

        ###################
        ## now = time.time()
        ## timing_usefulfields = now - start
        ## start = now
        ###################

        describeSearch(output, user, search)

        ###################
        ## now = time.time()
        ## timing_describesearch = now - start
        ## start = now
        ###################

        suggestOptimizations(output, user, search)

        ###################
        ## now = time.time()
        ## timing_optimize = now - start
        ## start = now
        ###################

        argTypeahead(output, sessionKey, namespace, user, bnf, search)

        ###################
        ## now = time.time()
        ## timing_argtypeahead = now - start
        ## start = now
        ###################

        ## overall_time = now - overallstart
        ## msg = "aliasmap=%6f, allcommands=%6f, argtypeahead=%6f, bnf=%6f, commandhelp=%6f, describesearch=%6f, didyouknow=%6f, didyoumean=%6f, firstterm=%6f, nextcommand=%6f, optimize=%6f, relatedpastsearches=%6f, relatedsearches=%6f, relatedterms=%6f, typeahead=%6f, usefulfields=%6f" % (timing_aliasmap, timing_allcommands, timing_argtypeahead, timing_bnf, timing_commandhelp, timing_describesearch, timing_didyouknow, timing_didyoumean, timing_firstterm, timing_nextcommand, timing_optimize, timing_relatedpastsearches, timing_relatedsearches, timing_relatedterms, timing_typeahead, timing_usefulfields)
        ## logger.error("SHELPER TIMING: %s overall=%6f -- %s" % (sessionKey, overall_time, msg))

    except Exception, e:
        msg = "! Error in search assistant: %s" % e
        msg += traceback.format_exc()
        output['notices'].insert(0, msg)

        logger.error(msg)
def nextCommand(output, sessionKey, namespace, user, search, usersquery,
                queryprefix, aliasMap, bnf, showargs):

    ## overallstart = start = time.time()
    ## timing_last_command = 0
    ## timing_all_commands = 0
    ## timing_get_next_data = 0
    ## timing_get_args = 0
    ## timing_add_commands = 0
    ## timing_past_matches = 0
    ## timing_sort_past_matches = 0

    atPipe = False

    # if search ends in "|", don't give args for last command, but
    # give next information from previous commandd
    if search[-1] == "|":
        search = search[:-1]
        showargs = False
        atPipe = True

    nextcommands = []
    typeaheadcommands = []

    commandAndArgs = utils.getLastCommand(search, aliasMap)

    ###################
    ## now = time.time()
    ## timing_last_command = now - start
    ## start = now
    ###################

    if commandAndArgs == None:
        # list all generating commands.
        # make a list() copy so we don't trash it by adding search
        typeaheadcommands = list(
            utils.getAllCommands(bnf, user, namespace, True))

        ###################
        ## now = time.time()
        ## timing_all_commands = now - start
        ## start = now
        ###################

    else:
        command, args = commandAndArgs
        data, pastsearches = next.getNextData(user, bnf, sessionKey, namespace)

        ###################
        ## now = time.time()
        ## timing_get_next_data = now - start
        ## start = now
        ###################

        for datum in data:
            if datum['command'] == command:
                typeaheadcommands = [
                    x for x, y in datum['nextcommands'] if x != "<RUN>"
                ]
                if showargs:

                    matchingargs = []
                    fs = fuzzSearch(usersquery)
                    for arg, perc in datum['args']:
                        replacement = "%s | %s %s" % (queryprefix, command,
                                                      arg)
                        fr = fuzzSearch(replacement)
                        if fr.startswith(fs) and fr != fs:
                            matchingargs.append((arg, perc))
                    output['args'] = matchingargs
                    #output['args'] =  datum['args']
                break

        ###################
        ## now = time.time()
        ## timing_get_args = now - start
        ## start = now
        ###################

        # now add in all commands that were not already added
        if command in aliasMap:
            # adding all the other commands not already added
            for thiscommand in utils.getAllCommands(bnf, user, namespace):
                if thiscommand not in nextcommands and thiscommand not in typeaheadcommands:
                    nextcommands.append(thiscommand)

        ###################
        ## now = time.time()
        ## timing_add_commands = now - start
        ## start = now
        ###################

        # look for pastsearches that the current search is a subset of  (like firefox url autocomplete looking for any term)
        usersearch = normalizeSearch(search)

        # if user didn't enter anything don't match on "search" or "search *", just get most recent
        if usersearch == "" or usersearch == "*":
            pastMatches = [userifySearch(p) for p in pastsearches]
        else:
            pastMatches = [
                userifySearch(pastsearch) for pastsearch in pastsearches
                if normalizedSearchMatch(True, usersquery, pastsearch)
            ]
            pastMatches.extend([
                userifySearch(pastsearch) for pastsearch in pastsearches
                if normalizedSearchMatch(False, usersquery, pastsearch)
            ])

        ###################
        ## now = time.time()
        ## timing_past_matches = now - start
        ## start = now
        ###################

        # dedup
        pastMatches = sorted(list(set(pastMatches)), key=pastMatches.index)
        output['autocomplete'] = pastMatches[:10]  # just the 10 most recent
        output['autocomplete_match'] = usersearch

        ###################
        ## now = time.time()
        ## timing_sort_past_matches = now - start
        ## start = now
        ###################

    # new.  only show next command if we aren't showing the args for the current command.
    # use will see next commands when they type "|"
    if atPipe:
        # keep only those that alias to themselves -- i.e., don't show aliases
        nextcommands = [x for x in nextcommands if aliasMap.get(x, '') == x]
        typeaheadcommands = [
            x for x in typeaheadcommands if aliasMap.get(x, '') == x
        ]
        s = usersquery.strip()
        if '|' in s:
            s = s[:s.rindex('|')].strip()
        # make triplets of (command, description, replacement)
        output['autonexts'] = [(x, utils.getAttr(bnf, x, "shortdesc",
                                                 ""), s + " | " + x)
                               for x in typeaheadcommands]
        output['nexts'] = [(x, utils.getAttr(bnf, x, "shortdesc",
                                             ""), s + " | " + x)
                           for x in nextcommands]