def execute(): results = [] try: results, dummyresults, settings = si.getOrganizedResults() # default values args = {"namespace": "search"} # get commandline args keywords, options = si.getKeywordsAndOptions() # override default args with settings from search kernel args.update(settings) # override default args with commandline args args.update(options) sessionKey = args.get("sessionKey", None) owner = args.get("owner", "admin") namespace = args.get("namespace", None) if namespace.lower() == "none": namespace = None messages = {} if sessionKey == None: # this shouldn't happen, but it's useful for testing. try: sessionKey = sa.getSessionKey("admin", "changeme") si.addWarnMessage( messages, "No session given to 'tune' command. Using default admin account and password." ) except splunk.AuthenticationFailed, e: si.addErrorMessage(messages, "No session given to 'tune' command.") return if len(keywords) != 1: usage() # e.g., '/data/inputs/monitor' entity = keywords[0] logger.info("Entity: %s Args: %s" % (entity, args)) results = [] # we don't care about incoming results try: entitys = en.getEntities(entity, sessionKey=sessionKey, owner=owner, namespace=namespace, count=-1) for name, entity in entitys.items(): try: myapp = entity["eai:acl"]["app"] if namespace != None and myapp != namespace: continue except: continue # if no eai:acl/app, filter out result = entityToResult(name, entity) results.append(result) except splunk.ResourceNotFound, e2: pass
def main(): try: messages = {} keywords,options = si.getKeywordsAndOptions() DEFAULT_MAX_TYPES = 10 maxtypes = options.get('max', str(DEFAULT_MAX_TYPES)) error = None if not maxtypes.isdigit(): error = 'max must be an integer between 1-%s.' % MAXRESULTS else: maxtypes = int(maxtypes) if not (0 < maxtypes <= MAXRESULTS): error = 'max must be an integer between 1-%s.' % MAXRESULTS if error: si.generateErrorResults(error) return ignore_covered = 'notcovered' in keywords useraw = 'useraw' in keywords results,dummyresults,settings = si.getOrganizedResults() #for r in results: # for attr in r: # print attr, r[attr], len(r[attr]) if len(results) > MAXRESULTS: results = results[:MAXRESULTS] si.addWarnMessage(messages, "For performance reasons, the maximum number of results used to discover event types was capped at %s. Consider a more restrictive search." % MAXRESULTS) argc = len(sys.argv) argv = sys.argv sessionKey = settings.get("sessionKey", None) owner = settings.get("owner", None) namespace = settings.get("namespace", None) searchhead = '' try: searches = sutils.getCommands(settings.get("search", ''), None) firstcmd = searches[0][0][0] firstarg = searches[0][0][1].strip() if firstcmd == 'search' and firstarg != '*': searchhead = firstarg except Exception, e: pass results = discover(results, searchhead, maxtypes, ignore_covered, useraw) if len(results) == 0: si.addWarnMessage(messages, "Unable to isolate useful groups of events.")
def main(): if len(sys.argv) < 3: usage() tname = sys.argv[1] #log("args") #for v in sys.argv: # log(v) options = ["max_terms", "use_disjunct", "eventsonly"] srchargs = [] log("ARGS: %s" % sys.argv[2:]) for arg in sys.argv[2:]: for option in options: if arg.startswith(option): break else: srchargs.append(arg) if len(srchargs) == 0: usage() tsearch = ' '.join(srchargs) log("SEARCH: %s" % tsearch) results, dummyresults, settings = si.getOrganizedResults() results = [] # we don't care about incoming results ########TEST##################### if 'sessionKey' not in settings: settings['owner'] = 'admin' settings['password'] = '******' settings['namespace'] = 'search' settings['sessionKey'] = splunk.auth.getSessionKey('admin', 'changeme') ########TEST#################### kwargs = {} for f in ['owner', 'namespace', 'sessionKey', 'hostPath']: if f in settings: kwargs[f] = settings[f] messages = {} try: maxTerms = int(settings.get("max_terms", MAX_SEARCH_COMPLEXITY)) if maxTerms > MAX_SEARCH_COMPLEXITY or maxTerms < 1: si.addWarnMessage( messages, "max_terms must be between 1 and %s. Using default." % MAX_SEARCH_COMPLEXITY) maxTerms = MAX_SEARCH_COMPLEXITY except Exception, e: maxTerms = MAX_SEARCH_COMPLEXITY
def main(): if len(sys.argv) < 3: usage() tname = sys.argv[1] #log("args") #for v in sys.argv: # log(v) options = ["max_terms", "use_disjunct", "eventsonly"] srchargs = [] log("ARGS: %s" % sys.argv[2:]) for arg in sys.argv[2:]: for option in options: if arg.startswith(option): break else: srchargs.append(arg) if len(srchargs) == 0: usage() tsearch = ' '.join(srchargs) log("SEARCH: %s" % tsearch) results,dummyresults,settings = si.getOrganizedResults() results = [] # we don't care about incoming results ########TEST##################### if 'sessionKey' not in settings: settings['owner'] = 'admin' settings['password'] = '******' settings['namespace'] = 'search' settings['sessionKey'] = splunk.auth.getSessionKey('admin', 'changeme') ########TEST#################### kwargs = {} for f in ['owner','namespace','sessionKey','hostPath']: if f in settings: kwargs[f] = settings[f] messages = {} try: maxTerms = int(settings.get("max_terms", MAX_SEARCH_COMPLEXITY)) if maxTerms > MAX_SEARCH_COMPLEXITY or maxTerms < 1: si.addWarnMessage(messages, "max_terms must be between 1 and %s. Using default." % MAX_SEARCH_COMPLEXITY) maxTerms = MAX_SEARCH_COMPLEXITY except Exception, e: maxTerms = MAX_SEARCH_COMPLEXITY
def main(): if len(sys.argv) < 3: usage() tname = sys.argv[1] #log("args") #for v in sys.argv: # log(v) options = ["max_terms", "use_disjunct", "eventsonly"] srchargs = [] log("ARGS: %s" % sys.argv[2:]) for arg in sys.argv[2:]: for option in options: if arg.startswith(option): break else: srchargs.append(arg) if len(srchargs) == 0: usage() tsearch = ' '.join(srchargs) log("SEARCH: %s" % tsearch) results, dummyresults, settings = si.getOrganizedResults() results = [] # we don't care about incoming results ########TEST##################### if 'sessionKey' not in settings: settings['owner'] = 'admin' settings['password'] = '******' settings['namespace'] = 'search' settings['sessionKey'] = splunk.auth.getSessionKey('admin', 'changeme') ########TEST#################### kwargs = {} for f in ['owner', 'namespace', 'sessionKey', 'hostPath']: if f in settings: kwargs[f] = settings[f] messages = {} try: maxTerms = int(settings.get("max_terms", MAX_SEARCH_COMPLEXITY)) if maxTerms > MAX_SEARCH_COMPLEXITY or maxTerms < 1: si.addWarnMessage( messages, "max_terms must be between 1 and %s. Using default." % MAX_SEARCH_COMPLEXITY) maxTerms = MAX_SEARCH_COMPLEXITY except Exception as e: maxTerms = MAX_SEARCH_COMPLEXITY dummy, options = si.getKeywordsAndOptions() makeORs = isTrue(options.get("use_disjunct", "t")) eventsOnly = isTrue(options.get("eventsonly", "f")) log("MAXTERMS: %s MAKEORS: %s eventsOnly: %s" % (maxTerms, makeORs, eventsOnly)) log("tsearch: %s" % tsearch) results = [] try: results = findTransaction(tname, tsearch, makeORs, eventsOnly, maxTerms, messages, **kwargs) except Exception as e: error(e) events = [] log("RESULTS: %s" % len(results)) for result in results: # api fail event = {} for field in result: if field == '_time': event['_time'] = util.dt2epoch( util.parseISO(str(result['_time']))) else: event[field] = result[field] events.append(event) si.outputResults(events, messages)
def findTransaction(tname, tconstraint, useORs, eventsOnly, maxTerms, messages, **kwargs): base_search, fields, maxspan = getTransactionInfo(tname, **kwargs) if maxspan == None: si.addWarnMessage( messages, "Add a maxspan contraint to the %s transactiontype definition to improve performance. Searching over all time for transitive values." % tname) log("MAXSPAN: %s" % maxspan) # require one field in transaction definition fieldsearch = " OR ".join(["%s=*" % field for field in fields]) initialConstraint = tconstraint if useORs: ## forces an OR of terms. slow and unnessary ## initialConstraint = disjunctify(tconstraint) # get the most restrictive term in the search and use that as the initial constrait to find events restrictiveTerm = getMostRestrictiveTerm(tconstraint, **kwargs) log("MOST RESTRICTIVE: %s" % restrictiveTerm) initialConstraint = restrictiveTerm # e.g., "sourcetype=sendmail" + "from=amrit" + "(qid=* OR mid=* OR pid=*)" index_search = "search (%s) (%s) (%s)" % (base_search, initialConstraint, fieldsearch) log("INDEX SEARCH: %s" % index_search) field_list_str = " ".join(fields) max_combos = maxTerms / len(fields) log("MAX_COMBINATION: %s" % max_combos) needsTIME = "" if maxspan != None: needsTIME = "_time" # make search to get field value pairs. # # e.g. | stats values(qid) as qid values(mid) as mid values(pid) as pid # stats_search = "| stats " + " ".join("values(%s) as %s" % (field, field) for field in fields) # # use top # stats_search = '| fillnull value="%s" %s | top %s %s showperc=false | addcoltotals' % (NULL_VAL, field_list_str, MAX_FIELD_COMBOS, field_list_str) # # TODO: if transactiondefinition contains maxspan, consider making # first stats_search return time ranges to limit values of fields stats_search = '| table %s %s | fillnull value="%s" %s | dedup %s | head %d' % ( field_list_str, needsTIME, NULL_VAL, field_list_str, field_list_str, max_combos) seenFields = set() while True: search = index_search + stats_search log("running search: %s" % search) results = splunk.search.searchAll(search, **kwargs) ## generate an OR of ANDS of field combinations -- (qid=1 pid=2) OR (qid=3 pid=4)..." ors = [] # for each top permuation of field values for result in results: ands = [] # for each field for field in result: if field == '_time': # if we have time field we must have maxspan # if we have maxspan info about event, use it to limit window of events to +/- maxspan of window # we don't need float precision, because subseconds don't matter in maxpan spec eventtime = int( util.dt2epoch(util.parseISO(str(result['_time'])))) ands.append('_time>=%s' % (eventtime - maxspan)) ands.append('_time<=%s' % (eventtime + maxspan)) else: val = result[field] # ignore empty values if val != NULL_VAL: seenFields.add( field) # add to list of fields with a value ands.append('%s="%s"' % (field, escVal(result[field]))) ands_str = "(" + " ".join(ands) + ")" ors.append(ands_str) field_constraints = " OR ".join(ors) # e.g., "sourcetype=sendmail (qid=1 pid=2) OR (qid=3 pid=4)..." index_search = "search (%s) (%s)" % (base_search, field_constraints) log("INDEXSEARCH: %s" % index_search) if len(results) >= max_combos: si.addWarnMessage( messages, "Reached max complexity in trying to find transaction events with %s unique values per field. Preferring more recent values. A more detailed initial transaction constraint will allow more complete transactions" % max_combos) if seenFields == set(fields): log("SEEN VALUES FOR ALL FIELDS: %s" % fields) break if len(results) == 0: msg = "No results in searching for required fields" si.addWarnMessage(messages, msg) return [] # we've retrieved all the events we're going to with the last index_search! if eventsOnly: # no transaction search, just return the events transaction_search = "" else: # this is it, find the transactions! transaction_search = '| transaction name="%s" | search %s' % ( tname, tconstraint) search = index_search + transaction_search log("running final search! %s" % search) results = splunk.search.searchAll(search, **kwargs) return results
def findTransaction(tname, tconstraint, useORs, eventsOnly, maxTerms, messages, **kwargs): base_search, fields, maxspan = getTransactionInfo(tname, **kwargs) if maxspan == None: si.addWarnMessage(messages, "Add a maxspan contraint to the %s transactiontype definition to improve performance. Searching over all time for transitive values." % tname) log("MAXSPAN: %s" % maxspan) # require one field in transaction definition fieldsearch = " OR ".join(["%s=*" % field for field in fields]) initialConstraint = tconstraint if useORs: ## forces an OR of terms. slow and unnessary ## initialConstraint = disjunctify(tconstraint) # get the most restrictive term in the search and use that as the initial constrait to find events restrictiveTerm = getMostRestrictiveTerm(tconstraint, **kwargs) log("MOST RESTRICTIVE: %s" % restrictiveTerm) initialConstraint = restrictiveTerm # e.g., "sourcetype=sendmail" + "from=amrit" + "(qid=* OR mid=* OR pid=*)" index_search = "search (%s) (%s) (%s)" % (base_search, initialConstraint, fieldsearch) log("INDEX SEARCH: %s" % index_search) field_list_str = " ".join(fields) max_combos = maxTerms / len(fields) log("MAX_COMBINATION: %s" % max_combos) needsTIME = "" if maxspan != None: needsTIME = "_time" # make search to get field value pairs. # # e.g. | stats values(qid) as qid values(mid) as mid values(pid) as pid # stats_search = "| stats " + " ".join("values(%s) as %s" % (field, field) for field in fields) # # use top # stats_search = '| fillnull value="%s" %s | top %s %s showperc=false | addcoltotals' % (NULL_VAL, field_list_str, MAX_FIELD_COMBOS, field_list_str) # # TODO: if transactiondefinition contains maxspan, consider making # first stats_search return time ranges to limit values of fields stats_search = '| table %s %s | fillnull value="%s" %s | dedup %s | head %s' % (field_list_str, needsTIME, NULL_VAL, field_list_str, field_list_str, max_combos) seenFields = set() while True: search = index_search + stats_search log("running search: %s" % search) results = splunk.search.searchAll(search, **kwargs) ## generate an OR of ANDS of field combinations -- (qid=1 pid=2) OR (qid=3 pid=4)..." ors = [] # for each top permuation of field values for result in results: ands = [] # for each field for field in result: if field == '_time': # if we have time field we must have maxspan # if we have maxspan info about event, use it to limit window of events to +/- maxspan of window # we don't need float precision, because subseconds don't matter in maxpan spec eventtime = int(util.dt2epoch(util.parseISO(str(result['_time'])))) ands.append('_time>=%s' % (eventtime - maxspan)) ands.append('_time<=%s' % (eventtime + maxspan)) else: val = result[field] # ignore empty values if val != NULL_VAL: seenFields.add(field) # add to list of fields with a value ands.append('%s="%s"' % (field, escVal(result[field]))) ands_str = "(" + " ".join(ands) + ")" ors.append(ands_str) field_constraints = " OR ".join(ors) # e.g., "sourcetype=sendmail (qid=1 pid=2) OR (qid=3 pid=4)..." index_search = "search (%s) (%s)" % (base_search, field_constraints) log("INDEXSEARCH: %s" % index_search) if len(results) >= max_combos: si.addWarnMessage(messages, "Reached max complexity in trying to find transaction events with %s unique values per field. Preferring more recent values. A more detailed initial transaction constraint will allow more complete transactions" % max_combos) if seenFields == set(fields): log("SEEN VALUES FOR ALL FIELDS: %s" % fields) break if len(results) == 0: msg = "No results in searching for required fields" si.addWarnMessage(messages, msg) return [] # we've retrieved all the events we're going to with the last index_search! if eventsOnly: # no transaction search, just return the events transaction_search = "" else: # this is it, find the transactions! transaction_search = '| transaction name="%s" | search %s' % (tname, tconstraint) search = index_search + transaction_search log("running final search! %s" % search) results = splunk.search.searchAll(search, **kwargs) return results
def execute(): results = [] try: results, dummyresults, settings = si.getOrganizedResults() # default values args = {'namespace': 'search'} # get commandline args keywords, options = si.getKeywordsAndOptions() # override default args with settings from search kernel args.update(settings) # override default args with commandline args args.update(options) sessionKey = args.get("sessionKey", None) owner = args.get("owner", 'admin') namespace = args.get("namespace", None) if namespace.lower() == "none": namespace = None messages = {} if sessionKey == None: # this shouldn't happen, but it's useful for testing. try: sessionKey = sa.getSessionKey('admin', 'changeme') si.addWarnMessage( messages, "No session given to 'tune' command. Using default admin account and password." ) except splunk.AuthenticationFailed, e: si.addErrorMessage(messages, "No session given to 'tune' command.") return if len(keywords) != 1: usage() # e.g., '/data/inputs/monitor' entity = keywords[0] logger.info("Entity: %s Args: %s" % (entity, args)) results = [] # we don't care about incoming results try: entitys = en.getEntities(entity, sessionKey=sessionKey, owner=owner, namespace=namespace, count=-1) for name, entity in entitys.items(): try: myapp = entity["eai:acl"]["app"] if namespace != None and myapp != namespace: continue except: continue # if no eai:acl/app, filter out result = entityToResult(name, entity) results.append(result) except splunk.ResourceNotFound, e2: pass