def safe(search):
commandinfo = utils.getCommands(search, None)
commands = [c.strip() for search in commandinfo for c,a in search ]
# don't even think about outsmarting a subsearches
if len(commandinfo) > 1:
return False
for cmd in commands:
if cmd not in SAFE_COMMANDS:
return False
return True
def help(output, aliasMap, user, search):
"""did you know ________?"""
# get list of commands user entered
userCommandsAndArgs = utils.getCommands(search, aliasMap)
# for each pipeline
for pipeline in userCommandsAndArgs:
searchCommands = [c.strip() for c,a in pipeline]
# a sort followed by a dedup can by combined into a "dedup .. sortby .." command
if 'dedup' in searchCommands and 'sort' in searchCommands and searchCommands.index('dedup') > searchCommands.index('sort') and isInteresting(user, "sortdedup"):
output['notices'].append(_('Consider using "%(suggestion)s" rather than "%(current)s"') % {
'suggestion': 'dedup %s sortby %s' % (ELLIPSE, ELLIPSE),
'current': 'sort %s | dedup %s' % (ELLIPSE,ELLIPSE)
})
# "count(_raw)" probably should be just "count" in the context of
# a stats/chart/timechart command
if "count(_raw)" in search and isInteresting(user, "countraw"):
output['notices'].append(_('Consider using "count()" rather than "count(_raw)"'))
# 'search ...| where ...' could be combined into a single search
# condition if the where condition is simple (i.e. if it is a
# simple comparision of a field to a literal value)
if 'search' in searchCommands and 'where' in searchCommands and searchCommands.index('where') > searchCommands.index('search') and isInteresting(user, "searchwhere"):
whereargs = pipeline[searchCommands.index('where')][1].strip()
searchargs = pipeline[searchCommands.index('search')][1].strip()
if whereargs != "": whereargs = " (%s)" % whereargs
if searchargs != "": searchargs = " (%s)" % searchargs
output['notices'].append(_('Consider combining the "where" condition%(whereargs)s into the search condition%(searchargs)s') % {'whereargs':whereargs, 'searchargs':searchargs})
# 'search ... | where ...' could be combined into a single search
# condition if the where condition is simple (i.e. if it is a
# simple comparision of a field to a literal value)
# for each command
for i, cmd in enumerate(searchCommands):
if cmd == 'search':
# get args
searchargs = pipeline[i][1].strip()
# has 10.*, 10.11.12.13, but not 10.10.10.10/16
IPish = re.findall("([0-9]{1,3}(?:\.[0-9*]{1,3}){1,3})(?![./0-9])", searchargs)
if len(IPish) > 0 and isInteresting(user, "searchip"):
output['notices'].append(_('Consider using CIDR support in the search operator (e.g., "host=10.0.0.1/16")'))
if " and " in searchargs or " not " in searchargs or " or " in searchargs:
output['notices'].append(_('Boolean operators must be uppercased (e.g., AND, OR, NOT); otherwise the search is looking for the terms "and", "or", and "not".'))
if "..." in searchargs:
output['notices'].append(_('Wildcards are supported with an asterisk ("*"), not an ellipsis ("...").'))
if len(searchCommands) == 1 and searchCommands[0] == "search" and not search.strip().endswith("|"):
#if (search == "| search" or search == "| search *") and isInteresting(user, "searchintro"):
output['notices'].append(_("***INTROTXT***"))
return output
def reallyGetNextData(user, stanzas, sessionKey, namespace):
commandResults = []
nextmap = {}
commoncounts = {}
argsmap = {}
literals = describer.getLiterals(stanzas, user, namespace)
bootstrapSearches, userSearches = getPastSearches(user, sessionKey, namespace)
searches = bootstrapSearches + userSearches
searches = searches[-MAX_HISTORY:]
aliasMap = utils.getAliasMap(stanzas)
badCommands = set()
# for each search in file
for search in searches:
commandseqs = utils.getCommands(search, aliasMap)
# for each sequency of commands for that search
for j, commands in enumerate(commandseqs):
commands.append((END,""))
# for each command
for i, commandarg in enumerate(commands):
command, arg = commandarg
if command not in literals:
if command != END:
badCommands.add(command)
arg = arg.strip()
if command == END:
break
addCount(argsmap, command, arg)
addCount(nextmap, command, commands[i+1][0])
addCommonCount(commoncounts, command)
if len(badCommands) > 0:
logger.warn("No searchbnf for these commands: %s!" % list(badCommands))
commandAndCounts = commoncounts.items()
commandAndCounts.sort( lambda x, y: y[1] - x[1] )
for command,count in commandAndCounts:
thisdata = {}
commandResults.append(thisdata)
thisdata['command'] = command
thisdata['count'] = count
thisargs = thisdata['args'] = []
thisnexts = thisdata['nextcommands'] = []
addSortedValueAndCounts(thisargs, argsmap[command])
addSortedValueAndCounts(thisnexts, nextmap[command])
return commandResults, userSearches
def reallyGetNextData(user, stanzas, sessionKey, namespace):
commandResults = []
nextmap = {}
commoncounts = {}
argsmap = {}
literals = describer.getLiterals(stanzas)
bootstrapSearches, userSearches = getPastSearches(user, sessionKey, namespace)
searches = bootstrapSearches + userSearches
searches = searches[-MAX_HISTORY:]
aliasMap = utils.getAliasMap(stanzas)
badCommands = set()
# for each search in file
for search in searches:
commandseqs = utils.getCommands(search, aliasMap)
# for each sequency of commands for that search
for j, commands in enumerate(commandseqs):
commands.append((END,""))
# for each command
for i, commandarg in enumerate(commands):
command, arg = commandarg
if command not in literals:
if command != END:
badCommands.add(command)
arg = arg.strip()
if command == END:
break
addCount(argsmap, command, arg)
addCount(nextmap, command, commands[i+1][0])
addCommonCount(commoncounts, command)
if len(badCommands) > 0:
logger.warn("No searchbnf for these commands: %s!" % list(badCommands))
commandAndCounts = commoncounts.items()
commandAndCounts.sort( lambda x, y: y[1] - x[1] )
for command,count in commandAndCounts:
thisdata = {}
commandResults.append(thisdata)
thisdata['command'] = command
thisdata['count'] = count
thisargs = thisdata['args'] = []
thisnexts = thisdata['nextcommands'] = []
addSortedValueAndCounts(thisargs, argsmap[command])
addSortedValueAndCounts(thisnexts, nextmap[command])
return commandResults, userSearches
def _main():
if len(sys.argv) > 1:
search = sys.argv[1]
cmds = utils.getCommands(search, None)
comms = [c.strip() for search in cmds for c,a in search ]
args = [a.strip() for search in cmds for c,a in search ]
print "Commands:", cmds
print "Commands: %s Args: %s" % (comms, args)
else:
user = "******"
sessionKey = utils.TEST_SESSION()
namespace = utils.TEST_NAMESPACE()
#print getPastSearches(user, None, sessionKey, namespace)
bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)
data, searches = getNextData(user, bnf, sessionKey, namespace)
for cmd in data:
print "\t%s" % cmd
def _main():
if len(sys.argv) > 1:
search = sys.argv[1]
cmds = utils.getCommands(search, None)
comms = [c.strip() for search in cmds for c,a in search ]
args = [a.strip() for search in cmds for c,a in search ]
print "Commands:", cmds
print "Commands: %s Args: %s" % (comms, args)
else:
user = "******"
sessionKey = utils.TEST_SESSION()
namespace = utils.TEST_NAMESPACE()
#print getPastSearches(user, None, sessionKey, namespace)
bnf = utils.getStanzas("searchbnf", sessionKey, user, namespace)
data, searches = getNextData(user, bnf, sessionKey, namespace)
for cmd in data:
print "\t%s" % cmd
def didYouMeanCommands(bnf, search):
output = []
# get list of public commands
knownCommands = utils.getAllCommands(bnf)
# get list of commands user entered
userCommandsAndArgs = utils.getCommands(search, None)[-1:] # just last
searchCommands = [c.strip() for search in userCommandsAndArgs for c, a in search ]
# get mapping of tags to commands
tagmap = getTagsToCommands(bnf, knownCommands)
# for each command user entered
for searchCommand in searchCommands:
# if not known, suggest something
if not searchCommand in knownCommands:
suggestion = getSuggestions(knownCommands, searchCommand, tagmap)
if suggestion != "":
output.extend(suggestion) #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion}))
return output, knownCommands
def didYouMeanCommands(bnf, search):
output = []
# get list of public commands
knownCommands = utils.getAllCommands(bnf)
# get list of commands user entered
userCommandsAndArgs = utils.getCommands(search, None)[-1:] # just last
searchCommands = [
c.strip() for search in userCommandsAndArgs for c, a in search
]
# get mapping of tags to commands
tagmap = getTagsToCommands(bnf, knownCommands)
# for each command user entered
for searchCommand in searchCommands:
# if not known, suggest something
if not searchCommand in knownCommands:
suggestion = getSuggestions(knownCommands, searchCommand, tagmap)
if suggestion != "":
output.extend(
suggestion
) #output.append(_('Unknown command "%(command)s". %(suggestion)s' % {'command':searchCommand, 'suggestion':suggestion}))
return output, knownCommands
def help(output, aliasMap, user, search):
"""did you know ________?"""
# get list of commands user entered
userCommandsAndArgs = utils.getCommands(search, aliasMap)
# for each pipeline
for pipeline in userCommandsAndArgs:
searchCommands = [c.strip() for c, a in pipeline]
# a sort followed by a dedup can by combined into a "dedup .. sortby .." command
if 'dedup' in searchCommands and 'sort' in searchCommands and searchCommands.index(
'dedup') > searchCommands.index('sort') and isInteresting(
user, "sortdedup"):
output['notices'].append(
_('Consider using "%(suggestion)s" rather than "%(current)s"')
% {
'suggestion': 'dedup %s sortby %s' % (ELLIPSE, ELLIPSE),
'current': 'sort %s | dedup %s' % (ELLIPSE, ELLIPSE)
})
# "count(_raw)" probably should be just "count" in the context of
# a stats/chart/timechart command
if "count(_raw)" in search and isInteresting(user, "countraw"):
output['notices'].append(
_('Consider using "count()" rather than "count(_raw)"'))
# 'search ...| where ...' could be combined into a single search
# condition if the where condition is simple (i.e. if it is a
# simple comparision of a field to a literal value)
if 'search' in searchCommands and 'where' in searchCommands and searchCommands.index(
'where') > searchCommands.index('search') and isInteresting(
user, "searchwhere"):
whereargs = pipeline[searchCommands.index('where')][1].strip()
searchargs = pipeline[searchCommands.index('search')][1].strip()
if whereargs != "": whereargs = " (%s)" % whereargs
if searchargs != "": searchargs = " (%s)" % searchargs
output['notices'].append(
_('Consider combining the "where" condition%(whereargs)s into the search condition%(searchargs)s'
) % {
'whereargs': whereargs,
'searchargs': searchargs
})
# 'search ... | where ...' could be combined into a single search
# condition if the where condition is simple (i.e. if it is a
# simple comparision of a field to a literal value)
# for each command
for i, cmd in enumerate(searchCommands):
if cmd == 'search':
# get args
searchargs = pipeline[i][1].strip()
# has 10.*, 10.11.12.13, but not 10.10.10.10/16
IPish = re.findall(
"([0-9]{1,3}(?:\.[0-9*]{1,3}){1,3})(?![./0-9])",
searchargs)
if len(IPish) > 0 and isInteresting(user, "searchip"):
output['notices'].append(
_('Consider using CIDR support in the search operator (e.g., "host=10.0.0.1/16")'
))
if " and " in searchargs or " not " in searchargs or " or " in searchargs:
output['notices'].append(
_('Boolean operators must be uppercased (e.g., AND, OR, NOT); otherwise the search is looking for the terms "and", "or", and "not".'
))
if "..." in searchargs:
output['notices'].append(
_('Wildcards are supported with an asterisk ("*"), not an ellipsis ("...").'
))
if len(searchCommands) == 1 and searchCommands[
0] == "search" and not search.strip().endswith("|"):
#if (search == "| search" or search == "| search *") and isInteresting(user, "searchintro"):
output['notices'].append(_("***INTROTXT***"))
return output
def __init__(self):
self.users = {}
self.commands = getCommands('utils/commands')
self.dbpool = txmongo.MongoConnectionPool()
self.db = self.dbpool.pychatserver
