def splunk_results_command(): jobs = service.jobs # type: ignore found = False res = [] for job in jobs: if job.sid == demisto.args()['sid']: rr = results.ResultsReader(job.results()) for result in rr: if isinstance(result, results.Message): demisto.results({ "Type": 1, "ContentsFormat": "json", "Contents": json.dumps(result.message) }) elif isinstance(result, dict): # Normal events are returned as dicts res.append(result) found = True if not found: demisto.results("Found no job for sid: " + demisto.args()['sid']) if found: demisto.results({ "Type": 1, "ContentsFormat": "json", "Contents": json.dumps(res) })
def parse_batch_of_results(current_batch_of_results, max_results_to_add): parsed_batch_results = [] batch_dbot_scores = [] results_reader = results.ResultsReader( io.BufferedReader(ResponseReaderWrapper(current_batch_of_results))) for item in results_reader: if isinstance(item, results.Message): if "Error in" in item.message: raise ValueError(item.message) parsed_batch_results.append(convert_to_str(item.message)) elif isinstance(item, dict): if demisto.get(item, 'host'): batch_dbot_scores.append({ 'Indicator': item['host'], 'Type': 'hostname', 'Vendor': 'Splunk', 'Score': 0, 'isTypedIndicator': True }) # Normal events are returned as dicts parsed_batch_results.append(item) if len(parsed_batch_results) >= max_results_to_add: break return parsed_batch_results, batch_dbot_scores
def events_over_time(self, event_name = "", time_range = TimeRange.MONTH, property = ""): query = 'search index=%s application=%s event="%s" | timechart span=%s count by %s | fields - _span*' % ( self.index, self.application_name, (event_name or "*"), time_range, (PROPERTY_PREFIX + property) if property else "event", ) job = self.splunk.jobs.create(query, exec_mode="blocking") over_time = {} reader = results.ResultsReader(job.results()) for result in reader: if isinstance(result, dict): # Get the time for this entry time = result["_time"] del result["_time"] # The rest is in the form of [event/property]:count # pairs, so we decode those for key,count in result.iteritems(): # Ignore internal ResultsReader properties if key.startswith("$"): continue entry = over_time.get(key, []) entry.append({ "count": int(count or 0), "time": time, }) over_time[key] = entry return over_time
def fetch_incidents(service): last_run = demisto.getLastRun() and demisto.getLastRun()['time'] search_offset = demisto.getLastRun().get('offset', 0) incidents = [] current_time_for_fetch = datetime.utcnow() if demisto.get(demisto.params(), 'timezone'): timezone = demisto.params()['timezone'] current_time_for_fetch = current_time_for_fetch + timedelta( minutes=int(timezone)) now = current_time_for_fetch.strftime(SPLUNK_TIME_FORMAT) if demisto.get(demisto.params(), 'useSplunkTime'): now = get_current_splunk_time(service) current_time_in_splunk = datetime.strptime(now, SPLUNK_TIME_FORMAT) current_time_for_fetch = current_time_in_splunk if len(last_run) == 0: fetch_time_in_minutes = parse_time_to_minutes() start_time_for_fetch = current_time_for_fetch - timedelta( minutes=fetch_time_in_minutes) last_run = start_time_for_fetch.strftime(SPLUNK_TIME_FORMAT) earliest_fetch_time_fieldname = demisto.params().get( "earliest_fetch_time_fieldname", "index_earliest") latest_fetch_time_fieldname = demisto.params().get( "latest_fetch_time_fieldname", "index_latest") kwargs_oneshot = { earliest_fetch_time_fieldname: last_run, latest_fetch_time_fieldname: now, "count": FETCH_LIMIT, 'offset': search_offset } searchquery_oneshot = demisto.params()['fetchQuery'] if demisto.get(demisto.params(), 'extractFields'): extractFields = demisto.params()['extractFields'] extra_raw_arr = extractFields.split(',') for field in extra_raw_arr: field_trimmed = field.strip() searchquery_oneshot = searchquery_oneshot + ' | eval ' + field_trimmed + '=' + field_trimmed oneshotsearch_results = service.jobs.oneshot( searchquery_oneshot, **kwargs_oneshot) # type: ignore reader = results.ResultsReader(oneshotsearch_results) for item in reader: inc = notable_to_incident(item) incidents.append(inc) demisto.incidents(incidents) if len(incidents) < FETCH_LIMIT: demisto.setLastRun({'time': now, 'offset': 0}) else: demisto.setLastRun({ 'time': last_run, 'offset': search_offset + FETCH_LIMIT })
def test_read_from_empty_result_set(self): job = self.service.jobs.create( "search index=_internal_does_not_exist | head 2") while not job.is_done(): sleep(0.5) self.assertEqual( 0, len(list(results.ResultsReader(io.BufferedReader(job.results())))))
def pretty(): reader = results.ResultsReader(sys.stdin) while True: kind = reader.read() if kind == None: break if kind == results.RESULT: event = reader.value pprint(event)
def get_data(response): reader = results.ResultsReader(response) # Running through results dictionary and making key value pairs for result in reader: for key, value in result.items(): print key + "=" + value, print
def pretty(response): reader = results.ResultsReader(response) while True: kind = reader.read() if kind == None: break if kind == results.RESULT: event = reader.value pprint(event)
def display(response): reader = results.ResultsReader(response) typer.secho("---- Failed Logins ---", fg=typer.colors.MAGENTA) for result in reader: if isinstance(result, dict): login = ("timestamp={2} user={0} src=:{1}").format( result["user"], result["src"], result["timestamp"]) typer.echo(login)
def splunk(self): service = client.connect(host=SPLUNK_HOST, port=SPLUNK_PORT, scheme=SPLUNK_SCHEME, username=SPLUNK_USERNAME, password=SPLUNK_PASSWORD) # 判断重连操作 self.redis_r = self.tryConnetRedis() if 'conf_splunklog_rule' in self.redis_r.hkeys("passive_config"): info = eval(self.redis_r.hget('passive_config', 'conf_splunklog_rule')) else: self.redis_r.hset('passive_config', 'conf_splunklog_rule', rule) info = rule query = info['query'] earliest_time = info['earliest_time'] max_time = info['max_time'] kwargs_normalsearch = {"exec_mode": "normal", "earliest_time": earliest_time, "latest_time": "now", "max_time": int(max_time), "timeout": 120} job = service.jobs.create(query=query, **kwargs_normalsearch) while True: while not job.is_ready(): pass stats = {"isDone": job["isDone"], "doneProgress": float(job["doneProgress"]) * 100, "scanCount": int(job["scanCount"]), "eventCount": int(job["eventCount"]), "resultCount": int(job["resultCount"])} status = ("\r%(doneProgress)03.1f%% %(scanCount)d scanned " "%(eventCount)d matched %(resultCount)d results") % stats sys.stdout.write(status) sys.stdout.flush() if stats["isDone"] == "1": break time.sleep(2) print "\ntotal time: %s" % job["runDuration"] r = self.tryConnetRedis() for result in results.ResultsReader(job.results()): # GET请求中ng_request_url_short然而并不会存在?,但是为了保持算法一致,还是进行分割一下 MD5 = self.md5('GET' + result['ng_request_url_short'].split('?')[0]) request_json = { 'method': 'GET' if result['ng_request_url_short'].strip().upper() == 'GET' else 'POST', 'protocol': 'http://', 'domain': result['ng_request_domain'], 'ng_request_url_short': result['ng_request_url_short'], 'arg': result['ng_query'] } self.redis_r.set('DataSort_' + MD5, request_json) r.execute_command("QUIT") job.cancel()
def run_modified_splunk_search(splunk_host, splunk_password, search1, search2, detection_name, detection_file, earliest_time, latest_time): try: service = client.connect(host=splunk_host, port=8089, username='******', password=splunk_password) except Exception as e: print("Unable to connect to Splunk instance: " + str(e)) return 1, {} if not search1.startswith('|'): search1 = 'search ' + search1 if not search2.startswith('|'): search2 = 'search ' + search2 kwargs = {"dispatch.earliest_time": "-1d", "dispatch.latest_time": "now"} try: job = service.jobs.export(search1, **kwargs) except Exception as e: print("Unable to execute detection: " + str(e)) return 1, {} reader = results.ResultsReader(job) results_search_malicious = [] for result in reader: if isinstance(result, dict): results_search_malicious.append(result) try: job = service.jobs.export(search2, **kwargs) except Exception as e: print("Unable to execute detection: " + str(e)) return 1, {} reader = results.ResultsReader(job) results_search_not_malicious = [] for result in reader: if isinstance(result, dict): results_search_not_malicious.append(result) return results_search_malicious, results_search_not_malicious
def __assert_transposed_results(self, service, index_name): search_results = service.jobs.oneshot( f"search index={index_name} | head 10 | transpose") reader = results.ResultsReader(search_results) for result in reader: assert result.get( "column" ) is not None, f"Attribute 'column' not found in a row {result}"
def _build_results_dict_from_sdk_response(response): """ Get results from the SDK and return them. """ reader = results.ResultsReader(response) resultset = [] for result in reader: resultset.append(result) return resultset
def _build_events_from_sdk_response(response): """ Get results from the SDK and return them. """ reader = results.ResultsReader(response) events = [] for result in reader: events.append(_build_event_from_results_reader(result)) return events
def __getIdFromDataRow(self,dataRow): result = None for result in results.ResultsReader(dataRow.results()): pass if result == None: return None else: return result[1]["id"]
def run_blocking_mode_search(splunk_service,search_string,payload={}): try: job = splunk_service.jobs.create(search_string,**payload) print(job.content) for result in results.ResultsReader(job.results()): print(result) except Exception as e: print(e)
def search(self, spl, mode="normal", search_args=None, verbose=False, days=None, start_time=None, end_time=None): ''' Search Splunk and return the results as a list of dicts. spl: A string containing the Splunk search in SPL form mode: A string specifying the type of Splunk search to run ("normal" or "realtime") search_args: A dict containing any additional search parameters to pass to the Splunk server. days: Search the past X days. If provided, this supercedes both start_time and end_time. start_time: A datetime() object representing the start of the search window, or a string in Splunk syntax (e.g., "-2d@d"). If used without end_time, the end of the search window is the current time. end_time: A datetime() object representing the end of the search window, or a string in Splunk syntax (e.g., "-2d@d"). If used without start_time, the search start will be the earliest timestamp in Splunk. verbose: If True, any errors, warnings or other messages encountered by the search process will be printed to stdout. The default is False (suppress these messages). ''' if not search_args or not isinstance(search_args, dict): search_args = dict() search_args["search_mode"] = mode if days: # Search from current time backwards search_args["earliest_time"] = "-%dd" % days else: if start_time: # convert to string if it's a datetime if isinstance(start_time, datetime): start_time = start_time.isoformat() search_args["earliest_time"] = start_time if end_time: # convert to string if it's a datetime if isinstance(end_time, datetime): end_time = end_time.isoformat() search_args["latest_time"] = end_time # Use the "export" job type, since that's the most reliable way to return possibly large result sets export_results = self.splunk_conn.jobs.export(spl, **search_args) reader = results.ResultsReader(export_results) for res in reader: if isinstance(res, dict): yield res elif isinstance(res, results.Message) and verbose: print("Message: %s" % res)
def get_mapping_fields_command(service): # Create the query to get unique objects # The logic is identical to the 'fetch_incidents' command type_field = demisto.params().get('type_field', 'source') total_parsed_results = [] search_offset = demisto.getLastRun().get('offset', 0) current_time_for_fetch = datetime.utcnow() dem_params = demisto.params() if demisto.get(dem_params, 'timezone'): timezone = dem_params['timezone'] current_time_for_fetch = current_time_for_fetch + timedelta( minutes=int(timezone)) now = current_time_for_fetch.strftime(SPLUNK_TIME_FORMAT) if demisto.get(dem_params, 'useSplunkTime'): now = get_current_splunk_time(service) current_time_in_splunk = datetime.strptime(now, SPLUNK_TIME_FORMAT) current_time_for_fetch = current_time_in_splunk fetch_time_in_minutes = parse_time_to_minutes() start_time_for_fetch = current_time_for_fetch - timedelta( minutes=fetch_time_in_minutes) last_run = start_time_for_fetch.strftime(SPLUNK_TIME_FORMAT) earliest_fetch_time_fieldname = dem_params.get( "earliest_fetch_time_fieldname", "earliest_time") latest_fetch_time_fieldname = dem_params.get("latest_fetch_time_fieldname", "latest_time") kwargs_oneshot = { earliest_fetch_time_fieldname: last_run, latest_fetch_time_fieldname: now, "count": FETCH_LIMIT, 'offset': search_offset } searchquery_oneshot = dem_params['fetchQuery'] if demisto.get(dem_params, 'extractFields'): extractFields = dem_params['extractFields'] extra_raw_arr = extractFields.split(',') for field in extra_raw_arr: field_trimmed = field.strip() searchquery_oneshot = searchquery_oneshot + ' | eval ' + field_trimmed + '=' + field_trimmed searchquery_oneshot = searchquery_oneshot + ' | dedup ' + type_field oneshotsearch_results = service.jobs.oneshot( searchquery_oneshot, **kwargs_oneshot) # type: ignore reader = results.ResultsReader(oneshotsearch_results) for item in reader: inc = notable_to_incident(item) total_parsed_results.append(inc) types_map = create_mapping_dict(total_parsed_results, type_field) demisto.results(types_map)
def check_in_splunk(troll): s = client.connect(host=SPLUNK_HOST, port="443", username=SPLUNK_USER, password=SPLUNK_PASS) q = f"search index=\"gameday-{TEAM}\" first_name=\"{troll['first_name']}\" last_name=\"{troll['last_name']}\"" r = s.jobs.oneshot(q) return [item for item in results.ResultsReader(r)]
def retrieving_splunk_events(self): jobs = self.service.jobs entries = list() kwargs_blockingsearch = {"exec_mode": "blocking"} searchquery_blocking = "search * | head 100" job = jobs.create(searchquery_blocking, **kwargs_blockingsearch) for result in results.ResultsReader(job.results()): print(result) entries.append(self.transform_to_logentry(result)) return entries
def timeit(): start = time.time() reader = results.ResultsReader(sys.stdin) count = 0 while True: kind = reader.read() if kind == None: break if kind == results.RESULT: count += 1 delta = time.time() - start print "%d results in %f secs = %f results/sec" % (count, delta, count/delta)
def test_export(self): jobs = self.service.jobs stream = jobs.export("search index=_internal earliest=-1m | head 3") result = results.ResultsReader(stream) ds = list(result) self.assertEqual(result.is_preview, False) self.assertTrue(isinstance(ds[0], dict) or \ isinstance(ds[0], results.Message)) nonmessages = [d for d in ds if isinstance(d, dict)] self.assertTrue(len(nonmessages) <= 3)
def counts(job, result_key): applications = [] reader = results.ResultsReader(job.results()) for result in reader: if isinstance(result, dict): applications.append({ "name": result[result_key], "count": int(result["count"] or 0) }) return applications
def check_accuracy(self, search_string, feature_fields, class_field): ''' check_accuracy(search_string, feature_fields, class_field) search_string: string to use in the splunk search to narrow events feature_fields: which fields to use to predict class_field: field to predict returns: accuracy of prediction notes: assumes that classifier is already trained. calls predict on each event. ''' # 1: check that classifier is trained: if not self.trained: raise 'classifier is not trained' # 2: predict the search string splunk_search = 'search %s | table *' % search_string search_kwargs = { 'timeout': 1000, 'exec_mode': 'blocking' } #required fields set to all - check API ("rf=*") job = self.jobs.create(splunk_search, **search_kwargs) result_count = int(job["resultCount"]) # 3: iterate and tally accuracy correct = 0 total = 0 offset = 0 count = 100 while (offset < result_count): print "offset: %s" % offset kwargs_paginate = {'count': count, 'offset': offset} search_results = job.results(**kwargs_paginate) for result in results.ResultsReader(search_results): try: if result[class_field] == self.predict( feature_fields, class_field, result): correct += 1 total += 1 else: total += 1 except: continue #tochange offset += count print "curr acc: %f" % (float(correct) / total) # 4: calculate percentage perc_correct = float(correct) / total self.accuracy = perc_correct self.accuracy_tested = True # 5: return return perc_correct
def collect_events(helper, ew): global_account = helper.get_arg('global_account') cloud = helper.get_arg('cloud') apikey = helper.get_arg('apikey') username = global_account['username'] password = global_account['password'] # Get list of MD5's pending detonation #session_key = helper.get_arg('session_key') #input_name, input_items = inputs.inputs.popitem() session_key = helper.context_meta['session_key'] #session_key = "dummy" md5List = get_md5_list(username, password, session_key) #Set envvars based on clear creds os.environ["ZIA_USERNAME"] = username os.environ["ZIA_PASSWORD"] = password os.environ["ZIA_API"] = apikey #API Login helper.log_info("Login to Zscaler API: %s" % username) z = zscaler_python_sdk.zscaler() z.get_zia_creds_from_env(True) z.set_cloud(cloud) z.authenticate_zia_api() helper.log_info("Login Success") # Get the results and display them using the ResultsReader reader = results.ResultsReader(md5List) for item in reader: if (item["md5"] == "none"): helper.log_info("STOP: No queued MD5") break helper.log_info("Checking Zscaler Sandbox for MD5 : %s" % item["md5"]) quota = z.check_sandbox_quota() #print(quota) helper.log_info("Sandbox current quota : %s" % quota) while quota['unused'] <= 0: quota = z.check_sandbox_quota() helper.log_info("waiting 1 sec...\tquota_left[" + str(quota['unused']) + "']") time.sleep(1) helper.log_info("Loading Zscaler Sandbox for MD5 : %s" % item["md5"]) report = z.get_sandbox_report(item["md5"], "full") #helper.log_info("Sandbox REPORT : %s" % report.text) #print(item["md5"]) event = Event() #event.stanza = input_name event.data = report.text ew.write_event(event)
def search_splunk(conn, search): jobs = conn.jobs kwargs = {'exec_mode': 'blocking'} job = jobs.create(search, **kwargs) rs = job.results(count=0) for result in results.ResultsReader( io.BufferedReader(ResponseReaderWrapper(rs))): yield result job.cancel()
def _exec_search(self, service, search): self._logger().info('action=about_to_execute_pending_queries_search search="%s"', search) job = service.jobs.create(search, **{'exec_mode': 'blocking'}) self._logger().debug('action=finish_pending_queries_search job="%s"', job.content) if job.content.get('messages') and job.content.get('messages').get('error'): raise RuntimeError(job.content.messages['error']) else: kwargs_paginate = {'count': 0} search_results = job.results(**kwargs_paginate) reader = results.ResultsReader(search_results) return list(reader)
def runQurey(keys, *queryname): print("runQuery function called") job = service.jobs.create(queryname[0], **kwargs_normalsearch) while True: if job.is_done() == True: print("running job") break output = results.ResultsReader(job.results(count=0)) job.cancel() print("calling dataframe function now") myDataframe(output, keys, *queryname)
def search(splunk_host, splunk_password, search_name, log): print('\nexecute savedsearch: ' + search_name + '\n') service = client.connect(host=splunk_host, port=8089, username='******', password=splunk_password) # Retrieve the new search mysavedsearch = service.saved_searches[search_name] kwargs = { "disabled": False, "dispatch.earliest_time": "-60m", "dispatch.latest_time": "now" } # Enable savedsearch and adapt the scheduling time mysavedsearch.update(**kwargs).refresh() # Run the saved search job = mysavedsearch.dispatch() # Create a small delay to allow time for the update between server and client sleep(2) # Wait for the job to finish--poll for completion and display stats while True: job.refresh() stats = { "isDone": job["isDone"], "doneProgress": float(job["doneProgress"]) * 100, "scanCount": int(job["scanCount"]), "eventCount": int(job["eventCount"]), "resultCount": int(job["resultCount"]) } status = ("\r%(doneProgress)03.1f%% %(scanCount)d scanned " "%(eventCount)d matched %(resultCount)d results") % stats sys.stdout.write(status) sys.stdout.flush() if stats["isDone"] == "1": break sleep(2) # Get the results and display them for result in results.ResultsReader(job.results()): print() print(result) # disable the savedsearch kwargs = {"disabled": True} mysavedsearch.update(**kwargs).refresh()
def getSearch(search, service): #print "/*Busqueda: " + search + "*/" # Retrieve the new search logCreator('INFO', 'Executing search of %s' % search) logCreator('DEBUG', 'Trying to get the new search') try: mysavedsearch = service.saved_searches[search] # Run the saved search job = mysavedsearch.dispatch() except: print "Error trying to connect Splunk" logCreator('ERROR', 'Splunk error: Splunk search %s not found' % search) sys.exit(1) logCreator('DEBUG', 'Search %s is OK' % search) # Create a small delay to allow time for the update between server and client sleep(2) # Wait for the job to finish--poll for completion and display stats while True: job.refresh() stats = { "isDone": job["isDone"], "doneProgress": float(job["doneProgress"]) * 100, "scanCount": int(job["scanCount"]), "eventCount": int(job["eventCount"]), "resultCount": int(job["resultCount"]) } status = ("\r%(doneProgress)03.1f%% %(scanCount)d scanned " "%(eventCount)d matched %(resultCount)d results") % stats if stats["isDone"] == "1": break sleep(2) # Display the search results now that the job is done try: jobresults = job.results() except: logCreator('ERROR', 'No valid answer from Splunk (Invalid licence?)') exit(1) #print jobresults # Finally, we got a reader and the number of results: answer = {} reader = results.ResultsReader(jobresults) answer['Reader'] = reader numResults = job["resultCount"] answer['Results'] = numResults return answer