def __init__(self): search = 'search * | sort _time' rc = loadrc() service = client.connect(**rc) job = service.jobs.create(search) while True: while not job.is_ready(): pass if job['isDone'] == '1': break self._job = job from splunklib.results import ResultsReader self._reader = ResultsReader(job.results()) self._buffer = self._reader.next()['_raw'] self._stringio = StringIO.StringIO(self._buffer)
def _parse_results(self, handle): """ Wraps output from Splunk searches with the Splunk ResultsReader. Splunk typically retrieves events debug statements, errors through the same stream. Debug/Info messages will be displayed and actual results :param handle: Splunk search job generator """ result_reader = ResultsReader(handle) for result in result_reader: # Diagnostic messages may be returned in the results if isinstance(result, Message): logger.debug('[{}] {}'.format(result.type, result.message)) # Normal events are returned as dicts elif isinstance(result, dict): result = dict(result) if '_time' in result: result['_time'] = SplunkAbstraction._to_datetime( result['_time']) yield { 'time': result['_time'] if '_time' in result else '', 'metadata': {k: v for k, v in result.items() if k.startswith('_')}, 'state': {k: v for k, v in result.items() if not k.startswith('_')} } else: logger.warning( 'Unknown result type in _parse_results: {}'.format(result)) assert result_reader.is_preview is False
def query_splunk(service, query: str, kwargs: dict) -> List[dict]: """Make a splunk search on `service`.""" job = service.jobs.create(query, **kwargs) query_results: List[dict] = [] while not job.is_done(): sleep(0.1) query_results = [r for r in ResultsReader(job.results())] job.cancel() return query_results
def example_main(): duration_keys = ["latencyMillis", "elapsedMillis", "duration", 'dur'] ls_component_key = "lightstep.component_name" join_guid = "guid:correlation_id" tags_to_rewrite = { "correlation_id": join_guid, "cid": join_guid, "component": ls_component_key, "host": ls_component_key } opts = utils.parse(sys.argv[1:], {}, ".splunkrc", usage="") if len(opts.args) != 1: utils.error("Search expression required", 2) search = opts.args[0] service = connect(**opts.kwargs) regex = re.compile( "(?P<start_time>.+) (?P<component>.+) (?P<operation>.+): (?P<tags>.*)") log_parser = LogParser(regex) log_parser.duration_keys = duration_keys log_parser.downcase_keys = True dict_parser = DictParser() dict_parser.downcase_keys = True dict_parser.operation_keys = ['activity'] dict_parser.duration_keys = duration_keys results = service.get("search/jobs/export", search=search, earliest_time="rt", latest_time="rt", search_mode="realtime") for result in ResultsReader(results.body): try: log = None # I'm not sure what will actually be used here since ResultReader can return # either a dictionary or message. I imagine a dict=>ParsedLog should be # straightforward to write. # https://github.com/splunk/splunk-sdk-python/blob/master/splunklib/results.py#L170 if result is None: break if isinstance(result, dict): parsed = dict_parser.parse_dict(result) else: parsed = log_parser.parse_line(result.message) parsed.rewrite_tags(tags_to_rewrite) #if int(parsed.tags["status"]) >= 300: # parsed.tags["error"] = True parsed.to_span() except Exception as e: print("Did not parse line: ", result, "(error: ", e, ")")
class splunkstream: """Input stream of Splunk log. Supports readline, tell, and seek methods""" def __init__(self): search = 'search * | sort _time' rc = loadrc() service = client.connect(**rc) job = service.jobs.create(search) while True: while not job.is_ready(): pass if job['isDone'] == '1': break self._job = job from splunklib.results import ResultsReader self._reader = ResultsReader(job.results()) self._buffer = self._reader.next()['_raw'] self._stringio = StringIO.StringIO(self._buffer) def readline(self): line = self._stringio.readline() if line == '': try: self._buffer = self._reader.next()['_raw'] except StopIteration: return '' self._stringio = StringIO.StringIO(self._buffer) line = self._stringio.readline() if not line.endswith('\n'): line = line + '\n' #print line, return line def close(self): self._job.cancel() def tell(self): return 0 def seek(self, offset, whence): raise Exception('not supported')
def __iter__(self): """Iterator of IResult objects""" _seq = [] for ordered_dict in ResultsReader(self.context): for key, value in ordered_dict.iteritems(): if isinstance(value, basestring): ordered_dict[key] = createObject(u'sparc.db.result_value', value) else: ordered_dict[key] = createObject( u'sparc.db.result_multi_value', value) alsoProvides(ordered_dict, ITabularResult) _seq.append(ordered_dict) return iter(_seq)
def _getOneshotResults(self, query, test_name): response = self.service.jobs.oneshot(query, app="searchcommands_app") reader = ResultsReader(response) actual = [] for result in reader: if isinstance(result, dict): actual += [u'Results: %s' % result] elif isinstance(result, Message): actual += [u'Message: %s' % result] actual = actual + [u'is_preview = %s' % reader.is_preview] actual = u'\n'.join(actual) with TestSearchCommandsApp._open_data_file( '_expected_results/%s.txt' % test_name, 'r') as expected_file: expected = u''.join(expected_file.readlines()) return actual, expected
def search( self, search_query: str, search_kwargs: Dict[str, str] = {} ) -> List[Dict[Any, Any]]: """Make a splunk search on `service`.""" if not search_kwargs: search_kwargs = self.search_defaults() job = self.client.jobs.create(search_query, **search_kwargs) query_results: List[Dict[Any, Any]] = [] while not job.is_done(): sleep(0.1) query_results = [r for r in ResultsReader(job.results())] job.cancel() return query_results
def main(): usage = "usage: %prog <search>" opts = utils.parse(sys.argv[1:], {}, ".splunkrc", usage=usage) if len(opts.args) != 1: utils.error("Search expression required", 2) search = opts.args[0] service = connect(**opts.kwargs) try: result = service.get("search/jobs/export", search=search, earliest_time="rt", latest_time="rt", search_mode="realtime") for result in ResultsReader(result.body): if result is not None: print pprint(result) except KeyboardInterrupt: print "\nInterrupted."