def __init__(self):
search = 'search * | sort _time'
rc = loadrc()
service = client.connect(**rc)
job = service.jobs.create(search)
while True:
while not job.is_ready():
pass
if job['isDone'] == '1':
break
self._job = job
from splunklib.results import ResultsReader
self._reader = ResultsReader(job.results())
self._buffer = self._reader.next()['_raw']
self._stringio = StringIO.StringIO(self._buffer)
def _parse_results(self, handle):
""" Wraps output from Splunk searches with the Splunk ResultsReader.
Splunk typically retrieves events debug statements, errors through the same stream.
Debug/Info messages will be displayed and actual results
:param handle: Splunk search job generator
"""
result_reader = ResultsReader(handle)
for result in result_reader:
# Diagnostic messages may be returned in the results
if isinstance(result, Message):
logger.debug('[{}] {}'.format(result.type, result.message))
# Normal events are returned as dicts
elif isinstance(result, dict):
result = dict(result)
if '_time' in result:
result['_time'] = SplunkAbstraction._to_datetime(
result['_time'])
yield {
'time': result['_time'] if '_time' in result else '',
'metadata':
{k: v
for k, v in result.items() if k.startswith('_')},
'state':
{k: v
for k, v in result.items() if not k.startswith('_')}
}
else:
logger.warning(
'Unknown result type in _parse_results: {}'.format(result))
assert result_reader.is_preview is False
def query_splunk(service, query: str, kwargs: dict) -> List[dict]:
"""Make a splunk search on `service`."""
job = service.jobs.create(query, **kwargs)
query_results: List[dict] = []
while not job.is_done():
sleep(0.1)
query_results = [r for r in ResultsReader(job.results())]
job.cancel()
return query_results
def example_main():
duration_keys = ["latencyMillis", "elapsedMillis", "duration", 'dur']
ls_component_key = "lightstep.component_name"
join_guid = "guid:correlation_id"
tags_to_rewrite = {
"correlation_id": join_guid,
"cid": join_guid,
"component": ls_component_key,
"host": ls_component_key
}
opts = utils.parse(sys.argv[1:], {}, ".splunkrc", usage="")
if len(opts.args) != 1:
utils.error("Search expression required", 2)
search = opts.args[0]
service = connect(**opts.kwargs)
regex = re.compile(
"(?P<start_time>.+) (?P<component>.+) (?P<operation>.+): (?P<tags>.*)")
log_parser = LogParser(regex)
log_parser.duration_keys = duration_keys
log_parser.downcase_keys = True
dict_parser = DictParser()
dict_parser.downcase_keys = True
dict_parser.operation_keys = ['activity']
dict_parser.duration_keys = duration_keys
results = service.get("search/jobs/export",
search=search,
earliest_time="rt",
latest_time="rt",
search_mode="realtime")
for result in ResultsReader(results.body):
try:
log = None
# I'm not sure what will actually be used here since ResultReader can return
# either a dictionary or message. I imagine a dict=>ParsedLog should be
# straightforward to write.
# https://github.com/splunk/splunk-sdk-python/blob/master/splunklib/results.py#L170
if result is None:
break
if isinstance(result, dict):
parsed = dict_parser.parse_dict(result)
else:
parsed = log_parser.parse_line(result.message)
parsed.rewrite_tags(tags_to_rewrite)
#if int(parsed.tags["status"]) >= 300:
# parsed.tags["error"] = True
parsed.to_span()
except Exception as e:
print("Did not parse line: ", result, "(error: ", e, ")")
class splunkstream:
"""Input stream of Splunk log. Supports readline, tell, and seek methods"""
def __init__(self):
search = 'search * | sort _time'
rc = loadrc()
service = client.connect(**rc)
job = service.jobs.create(search)
while True:
while not job.is_ready():
pass
if job['isDone'] == '1':
break
self._job = job
from splunklib.results import ResultsReader
self._reader = ResultsReader(job.results())
self._buffer = self._reader.next()['_raw']
self._stringio = StringIO.StringIO(self._buffer)
def readline(self):
line = self._stringio.readline()
if line == '':
try:
self._buffer = self._reader.next()['_raw']
except StopIteration:
return ''
self._stringio = StringIO.StringIO(self._buffer)
line = self._stringio.readline()
if not line.endswith('\n'):
line = line + '\n'
#print line,
return line
def close(self):
self._job.cancel()
def tell(self):
return 0
def seek(self, offset, whence):
raise Exception('not supported')
def __iter__(self):
"""Iterator of IResult objects"""
_seq = []
for ordered_dict in ResultsReader(self.context):
for key, value in ordered_dict.iteritems():
if isinstance(value, basestring):
ordered_dict[key] = createObject(u'sparc.db.result_value',
value)
else:
ordered_dict[key] = createObject(
u'sparc.db.result_multi_value', value)
alsoProvides(ordered_dict, ITabularResult)
_seq.append(ordered_dict)
return iter(_seq)
def _getOneshotResults(self, query, test_name):
response = self.service.jobs.oneshot(query, app="searchcommands_app")
reader = ResultsReader(response)
actual = []
for result in reader:
if isinstance(result, dict):
actual += [u'Results: %s' % result]
elif isinstance(result, Message):
actual += [u'Message: %s' % result]
actual = actual + [u'is_preview = %s' % reader.is_preview]
actual = u'\n'.join(actual)
with TestSearchCommandsApp._open_data_file(
'_expected_results/%s.txt' % test_name, 'r') as expected_file:
expected = u''.join(expected_file.readlines())
return actual, expected
def search(
self, search_query: str, search_kwargs: Dict[str, str] = {}
) -> List[Dict[Any, Any]]:
"""Make a splunk search on `service`."""
if not search_kwargs:
search_kwargs = self.search_defaults()
job = self.client.jobs.create(search_query, **search_kwargs)
query_results: List[Dict[Any, Any]] = []
while not job.is_done():
sleep(0.1)
query_results = [r for r in ResultsReader(job.results())]
job.cancel()
return query_results
def main():
usage = "usage: %prog <search>"
opts = utils.parse(sys.argv[1:], {}, ".splunkrc", usage=usage)
if len(opts.args) != 1:
utils.error("Search expression required", 2)
search = opts.args[0]
service = connect(**opts.kwargs)
try:
result = service.get("search/jobs/export",
search=search,
earliest_time="rt",
latest_time="rt",
search_mode="realtime")
for result in ResultsReader(result.body):
if result is not None:
print pprint(result)
except KeyboardInterrupt:
print "\nInterrupted."