def _compare_chunks(self, expected, output, time_sensitive=True):
expected = expected.strip()
output = output.strip()
if time_sensitive:
self.assertEqual(len(expected), len(output))
compare_csv_files = self._compare_csv_files_time_sensitive
else:
compare_csv_files = self._compare_csv_files_time_insensitive
chunks_1 = self._load_chunks(StringIO(expected))
chunks_2 = self._load_chunks(StringIO(output))
self.assertEqual(len(chunks_1), len(chunks_2))
n = 0
for chunk_1, chunk_2 in izip(chunks_1, chunks_2):
self.assertDictEqual(
chunk_1.metadata, chunk_2.metadata,
'Chunk {0}: metadata error: "{1}" != "{2}"'.format(
n, chunk_1.metadata, chunk_2.metadata))
compare_csv_files(chunk_1.body, chunk_2.body)
n += 1
return
def test_successful_validation(self):
"""Check that successful validation yield no text and a 0 exit value."""
# Override abstract methods
class NewScript(Script):
def get_scheme(self):
return None
def validate_input(self, definition):
# always succeed...
return
def stream_events(self, inputs, ew):
# unused
return
script = NewScript()
out = StringIO()
err = StringIO()
ew = EventWriter(out, err)
args = [TEST_SCRIPT_PATH, "--validate-arguments"]
return_value = script.run_script(args, ew,
data_open("data/validation.xml"))
self.assertEqual("", err.getvalue())
self.assertEqual("", out.getvalue())
self.assertEqual(0, return_value)
def _compare_csv_files_time_sensitive(self, expected, output):
self.assertEqual(len(expected), len(output))
skip_first_row = expected[0:2] == '\r\n'
expected = StringIO(expected)
output = StringIO(output)
line_number = 1
if skip_first_row:
self.assertEqual(expected.readline(), output.readline())
line_number += 1
expected = csv.DictReader(expected)
output = csv.DictReader(output)
for expected_row in expected:
output_row = next(output)
self.assertDictEqual(
expected_row, output_row,
'Error on line {0}: expected {1}, not {2}'.format(
line_number, expected_row, output_row))
line_number += 1
self.assertRaises(StopIteration, output.next)
return
def playback(self, path):
with open(path, 'rb') as f:
test_data = pickle.load(f)
self._output = StringIO()
self._recording = test_data['inputs']
self._recording_part = self._recording.popleft()
def get(self, method, *args, **kwargs):
return self._recording_part[method.__name__].popleft()
self.get = MethodType(get, self, self.__class__)
def next_part(self):
self._recording_part = self._recording.popleft()
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
self._test_case.assertEqual(test_data['results'],
self._output.getvalue())
self.stop = MethodType(stop, self, self.__class__)
return
def _compare_csv_files_time_insensitive(self, expected, output):
skip_first_row = expected[0:2] == '\r\n'
expected = StringIO(expected)
output = StringIO(output)
line_number = 1
if skip_first_row:
self.assertEqual(expected.readline(), output.readline())
line_number += 1
expected = csv.DictReader(expected)
output = csv.DictReader(output)
for expected_row in expected:
output_row = next(output)
try:
timestamp = float(output_row['_time'])
datetime.fromtimestamp(timestamp)
except BaseException as error:
self.fail(error)
else:
output_row['_time'] = expected_row['_time']
self.assertDictEqual(
expected_row, output_row,
'Error on line {0}: expected {1}, not {2}'.format(
line_number, expected_row, output_row))
line_number += 1
self.assertRaises(StopIteration, output.next)
return
def test_recorder(self):
if (python_version[0] == 2 and python_version[1] < 7):
print("Skipping test since we're on {1}".format("".join(python_version)))
pass
# Grab an input/output recording, the results of a prior countmatches run
recording = os.path.join(self._package_path, 'recordings', 'scpv2', 'Splunk-6.3', 'countmatches.')
with gzip.open(recording + 'input.gz', 'rb') as file_1:
with io.open(recording + 'output', 'rb') as file_2:
ifile = StringIO(file_1.read())
result = StringIO(file_2.read())
# Set up the input/output recorders that are under test
ifile = Recorder(mktemp(), ifile)
try:
ofile = Recorder(mktemp(), StringIO())
try:
# Read and then write a line
ifile.readline()
ofile.write(result.readline())
# Read and then write a block
ifile.read()
ofile.write(result.read())
# Verify that what we wrote is equivalent to the original recording, the result from a prior
# countmatches run
self.assertEqual(ofile.getvalue(), result.getvalue())
# Verify that we faithfully recorded the input and output files
ifile._recording.close()
ofile._recording.close()
with gzip.open(ifile._recording.name, 'rb') as file_1:
with gzip.open(ofile._recording.name, 'rb') as file_2:
self.assertEqual(file_1.read(), ifile._file.getvalue())
self.assertEqual(file_2.read(), ofile._file.getvalue())
finally:
ofile._recording.close()
os.remove(ofile._recording.name)
finally:
ifile._recording.close()
os.remove(ifile._recording.name)
return
def test_messages_header(self):
@Configuration()
class TestMessagesHeaderCommand(SearchCommand):
class ConfigurationSettings(SearchCommand.ConfigurationSettings):
@classmethod
def fix_up(cls, command_class):
pass
command = TestMessagesHeaderCommand()
command._protocol_version = 1
output_buffer = StringIO()
command._record_writer = RecordWriterV1(output_buffer)
messages = [(command.write_debug, 'debug_message'),
(command.write_error, 'error_message'),
(command.write_fatal, 'fatal_message'),
(command.write_info, 'info_message'),
(command.write_warning, 'warning_message')]
for write, message in messages:
write(message)
command.finish()
expected = ('debug_message=debug_message\r\n'
'error_message=error_message\r\n'
'error_message=fatal_message\r\n'
'info_message=info_message\r\n'
'warn_message=warning_message\r\n'
'\r\n')
self.assertEquals(output_buffer.getvalue(), expected)
return
def make_request(self, url, method, data, headers):
self.log_message("%s: %s", method, url)
try:
# Make the request
request = urllib.Request(url, data, headers)
request.get_method = lambda: method
response = urllib.urlopen(request)
# We were successful, so send the response code
self.send_response(response.code, message=response.msg)
for key, value in iteritems(dict(response.headers)):
# Optionally log the headers
#self.log_message("%s: %s" % (key, value))
self.send_header(key, value)
# Send the cross-domain headers
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods",
"PUT, POST, GET, DELETE, OPTIONS")
self.send_header("Access-Control-Allow-Headers",
"X-Redirect-URL, Authorization")
# We are done with the headers
self.end_headers()
# Copy the response to the output
self.copyfile(response, self.wfile)
except urllib.HTTPError as e:
# On errors, log the response code and message
self.log_message("Code: %s (%s)", e.code, e.msg)
for key, value in iteritems(dict(e.hdrs)):
# On errors, we always log the headers
self.log_message("%s: %s", key, value)
response_text = e.fp.read()
response_file = StringIO(response_text)
# On errors, we also log the response text
self.log_message("Response: %s", response_text)
# Send the error response code
self.send_response(e.code, message=e.msg)
# Send the cross-domain headers
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods",
"PUT, POST, GET, DELETE, OPTIONS")
self.send_header("Access-Control-Allow-Headers",
"X-Redirect-URL, Authorization")
# Send the other headers
self.send_header("Content-Type", self.error_content_type)
self.send_header('Connection', 'close')
self.end_headers()
# Finally, send the error itself
self.copyfile(response_file, self.wfile)
def record(self, path):
self._output = StringIO()
self._recording = deque()
self._recording_part = OrderedDict()
self._recording.append(self._recording_part)
def get(self, method, *args, **kwargs):
result = method(*args, **kwargs)
part = self._recording_part
key = method.__name__
try:
results = part[key]
except KeyError:
part[key] = results = deque()
results.append(result)
return result
self.get = MethodType(get, self, self.__class__)
def next_part(self):
part = OrderedDict()
self._recording_part = part
self._recording.append(part)
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
with io.open(path, 'wb') as f:
test = OrderedDict((('inputs', self._recording), ('results', self._output.getvalue())))
pickle.dump(test, f)
self.stop = MethodType(stop, self, self.__class__)
return
def __init__(self, ofile, maxresultrows=None):
self._maxresultrows = 50000 if maxresultrows is None else maxresultrows
self._ofile = set_binary_mode(ofile)
self._fieldnames = None
self._buffer = StringIO()
self._writer = csv.writer(self._buffer, dialect=CsvDialect)
self._writerow = self._writer.writerow
self._finished = False
self._flushed = False
self._inspector = OrderedDict()
self._chunk_count = 0
self._pending_record_count = 0
self._committed_record_count = 0
def test_error_on_script_with_null_scheme(self):
"""A script that returns a null scheme should generate no output on
stdout and an error on stderr saying that it the scheme was null."""
# Override abstract methods
class NewScript(Script):
def get_scheme(self):
return None
def stream_events(self, inputs, ew):
# not used
return
script = NewScript()
out = BytesIO()
err = BytesIO()
ew = EventWriter(out, err)
in_stream = StringIO()
args = [TEST_SCRIPT_PATH, "--scheme"]
return_value = script.run_script(args, ew, in_stream)
self.assertEqual(b"", out.getvalue())
self.assertEqual(
b"FATAL Modular input script returned a null scheme.\n",
err.getvalue())
self.assertNotEqual(0, return_value)
def test_event_without_enough_fields_fails(self):
"""Check that events without data throw an error"""
with self.assertRaises(ValueError):
event = Event()
stream = StringIO()
event.write_to(stream)
self.assertTrue(True)
def assertInfoEqual(self, output, expected):
reader = csv.reader(StringIO(output))
self.assertEqual([], next(reader))
fields = next(reader)
values = next(reader)
self.assertRaises(StopIteration, reader.next)
output = dict(izip(fields, values))
reader = csv.reader(StringIO(expected))
self.assertEqual([], next(reader))
fields = next(reader)
values = next(reader)
self.assertRaises(StopIteration, reader.next)
expected = dict(izip(fields, values))
self.assertDictEqual(expected, output)
def _execute_chunk_v2(self, process, chunk):
metadata, body = chunk
if len(body) <= 0:
return
records = self._read_csv_records(StringIO(body))
self._record_writer.write_records(process(records))
def _execute_chunk_v2(self, process, chunk):
metadata, body = chunk
if len(body) <= 0 and not self._allow_empty_input:
raise ValueError(
"No records found to process. Set allow_empty_input=True in dispatch function to move forward "
"with empty records.")
records = self._read_csv_records(StringIO(body))
self._record_writer.write_records(process(records))
def _records_protocol_v2(self, ifile):
istream = self._as_binary_stream(ifile)
while True:
result = self._read_chunk(istream)
if not result:
return
metadata, body = result
action = getattr(metadata, 'action', None)
if action != 'execute':
raise RuntimeError(
'Expected execute action, not {}'.format(action))
finished = getattr(metadata, 'finished', False)
self._record_writer.is_flushed = False
if len(body) > 0:
reader = csv.reader(StringIO(body), dialect=CsvDialect)
try:
fieldnames = next(reader)
except StopIteration:
return
mv_fieldnames = dict([(name, name[len('__mv_'):])
for name in fieldnames
if name.startswith('__mv_')])
if len(mv_fieldnames) == 0:
for values in reader:
yield OrderedDict(izip(fieldnames, values))
else:
for values in reader:
record = OrderedDict()
for fieldname, value in izip(fieldnames, values):
if fieldname.startswith('__mv_'):
if len(value) > 0:
record[mv_fieldnames[
fieldname]] = self._decode_list(value)
elif fieldname not in record:
record[fieldname] = value
yield record
if finished:
return
self.flush()
def record(self, path):
self._output = StringIO()
self._recording = deque()
self._recording_part = OrderedDict()
self._recording.append(self._recording_part)
def get(self, method, *args, **kwargs):
result = method(*args, **kwargs)
part = self._recording_part
key = method.__name__
try:
results = part[key]
except KeyError:
part[key] = results = deque()
results.append(result)
return result
self.get = MethodType(get, self, self.__class__)
def next_part(self):
part = OrderedDict()
self._recording_part = part
self._recording.append(part)
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
with io.open(path, 'wb') as f:
test = OrderedDict(
(('inputs', self._recording), ('results',
self._output.getvalue())))
pickle.dump(test, f)
self.stop = MethodType(stop, self, self.__class__)
return
def __init__(self, ofile, maxresultrows=None):
self._maxresultrows = 50000 if maxresultrows is None else maxresultrows
self._ofile = ofile
self._fieldnames = None
self._buffer = StringIO()
self._writer = csv.writer(self._buffer, dialect=CsvDialect)
self._writerow = self._writer.writerow
self._finished = False
self._flushed = False
self._inspector = OrderedDict()
self._chunk_count = 0
self._record_count = 0
self._total_record_count = 0
def playback(self, path):
with open(path, 'rb') as f:
test_data = pickle.load(f)
self._output = StringIO()
self._recording = test_data['inputs']
self._recording_part = self._recording.popleft()
def get(self, method, *args, **kwargs):
return self._recording_part[method.__name__].popleft()
self.get = MethodType(get, self, self.__class__)
def next_part(self):
self._recording_part = self._recording.popleft()
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
self._test_case.assertEqual(test_data['results'], self._output.getvalue())
self.stop = MethodType(stop, self, self.__class__)
return
def format(self, value):
output = StringIO()
writer = csv.writer(output, List.Dialect)
writer.writerow(value)
value = output.getvalue()
return value[:-1]
class TestRecorder(object):
def __init__(self, test_case):
self._test_case = test_case
self._output = None
self._recording = None
self._recording_part = None
def _not_implemented(self):
raise NotImplementedError(
'class {} is not in playback or record mode'.format(
self.__class__.__name__))
self.get = self.next_part = self.stop = MethodType(
_not_implemented, self, self.__class__)
return
@property
def output(self):
return self._output
def playback(self, path):
with open(path, 'rb') as f:
test_data = pickle.load(f)
self._output = StringIO()
self._recording = test_data['inputs']
self._recording_part = self._recording.popleft()
def get(self, method, *args, **kwargs):
return self._recording_part[method.__name__].popleft()
self.get = MethodType(get, self, self.__class__)
def next_part(self):
self._recording_part = self._recording.popleft()
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
self._test_case.assertEqual(test_data['results'],
self._output.getvalue())
self.stop = MethodType(stop, self, self.__class__)
return
def record(self, path):
self._output = StringIO()
self._recording = deque()
self._recording_part = OrderedDict()
self._recording.append(self._recording_part)
def get(self, method, *args, **kwargs):
result = method(*args, **kwargs)
part = self._recording_part
key = method.__name__
try:
results = part[key]
except KeyError:
part[key] = results = deque()
results.append(result)
return result
self.get = MethodType(get, self, self.__class__)
def next_part(self):
part = OrderedDict()
self._recording_part = part
self._recording.append(part)
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
with io.open(path, 'wb') as f:
test = OrderedDict(
(('inputs', self._recording), ('results',
self._output.getvalue())))
pickle.dump(test, f)
self.stop = MethodType(stop, self, self.__class__)
return
def test_record_writer_with_recordings(self):
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join(
(cls.__name__, method.__name__)))
for input_file in iglob(base_path + '*.input.gz'):
with gzip.open(input_file, 'rb') as ifile:
test_data = pickle.load(ifile)
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
write_record = writer.write_record
fieldnames = test_data['fieldnames']
for values in test_data['values']:
record = OrderedDict(izip(fieldnames, values))
try:
write_record(record)
except Exception as error:
self.fail(error)
for message_type, message_text in test_data['messages']:
writer.write_message(message_type, '{}', message_text)
for name, metric in six.iteritems(test_data['metrics']):
writer.write_metric(name, metric)
writer.flush(finished=True)
# Read expected data
expected_path = os.path.splitext(
os.path.splitext(input_file)[0])[0] + '.output'
with io.open(expected_path, 'rb') as ifile:
expected = ifile.read()
expected = self._load_chunks(StringIO(expected))
# Read observed data
ifile = writer._ofile
ifile.seek(0)
observed = self._load_chunks(ifile)
# Write observed data (as an aid to diagnostics)
observed_path = expected_path + '.observed'
observed_value = ifile.getvalue()
with io.open(observed_path, 'wb') as ifile:
ifile.write(observed_value)
self._compare_chunks(observed, expected)
return
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = [
'_serial', '_time', 'random_bytes', 'random_dict',
'random_integers', 'random_unicode'
]
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [
serial_number,
time(),
random_bytes(),
random_dict(),
random_integers(),
random_unicode()
]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [('debug', random_unicode()), ('error', random_unicode()),
('fatal', random_unicode()), ('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(
ifilter(lambda k_v: k_v[0].startswith('metric.'),
six.iteritems(writer._inspector))),
dict(
imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]),
six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError,
writer.flush,
finished=True,
partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(
self._recordings_path, '.'.join(
(cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def test_process_scpv1(self):
# TestCommand.process should complain if supports_getinfo == False
# We support dynamic configuration, not static
# The exception line number may change, so we're using a regex match instead of a string match
expected = re.compile(
r'error_message=RuntimeError at ".+search_command\.py", line \d\d\d : Command test appears to be '
r'statically configured for search command protocol version 1 and static configuration is unsupported by '
r'splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza:\n'
r'\[test\]\n'
r'filename = test.py\n'
r'enableheader = true\n'
r'outputheader = true\n'
r'requires_srinfo = true\n'
r'supports_getinfo = true\n'
r'supports_multivalues = true\n'
r'supports_rawargs = true')
argv = [
'test.py', 'not__GETINFO__or__EXECUTE__', 'option=value',
'fieldname'
]
command = TestCommand()
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ofile=result)
self.assertRegexpMatches(result.getvalue(), expected)
# TestCommand.process should return configuration settings on Getinfo probe
argv = [
'test.py', '__GETINFO__', 'required_option_1=value',
'required_option_2=value'
]
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertEqual(str(command.configuration), '')
if six.PY2:
expected = (
"[(u'clear_required_fields', None, [1]), (u'distributed', None, [2]), (u'generates_timeorder', None, [1]), "
"(u'generating', None, [1, 2]), (u'maxinputs', None, [2]), (u'overrides_timeorder', None, [1]), "
"(u'required_fields', None, [1, 2]), (u'requires_preop', None, [1]), (u'retainsevents', None, [1]), "
"(u'run_in_preview', None, [2]), (u'streaming', None, [1]), (u'streaming_preop', None, [1, 2]), "
"(u'type', None, [2])]")
else:
expected = (
"[('clear_required_fields', None, [1]), ('distributed', None, [2]), ('generates_timeorder', None, [1]), "
"('generating', None, [1, 2]), ('maxinputs', None, [2]), ('overrides_timeorder', None, [1]), "
"('required_fields', None, [1, 2]), ('requires_preop', None, [1]), ('retainsevents', None, [1]), "
"('run_in_preview', None, [2]), ('streaming', None, [1]), ('streaming_preop', None, [1, 2]), "
"('type', None, [2])]")
self.assertEqual(repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error, result.getvalue()))
self.assertEqual('\r\n\r\n\r\n', result.getvalue()
) # No message header and no configuration settings
ifile = StringIO('\n')
result = StringIO()
# We might also put this sort of code into our SearchCommand.prepare override ...
configuration = command.configuration
# SCP v1/v2 configuration settings
configuration.generating = True
configuration.required_fields = ['foo', 'bar']
configuration.streaming_preop = 'some streaming command'
# SCP v1 configuration settings
configuration.clear_required_fields = True
configuration.generates_timeorder = True
configuration.overrides_timeorder = True
configuration.requires_preop = True
configuration.retainsevents = True
configuration.streaming = True
# SCP v2 configuration settings (SCP v1 requires that maxinputs and run_in_preview are set in commands.conf)
configuration.distributed = True
configuration.maxinputs = 50000
configuration.run_in_preview = True
configuration.type = 'streaming'
if six.PY2:
expected = (
'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[u\'foo\', u\'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
else:
expected = (
'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[\'foo\', \'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
self.assertEqual(str(command.configuration), expected)
if six.PY2:
expected = (
"[(u'clear_required_fields', True, [1]), (u'distributed', True, [2]), (u'generates_timeorder', True, [1]), "
"(u'generating', True, [1, 2]), (u'maxinputs', 50000, [2]), (u'overrides_timeorder', True, [1]), "
"(u'required_fields', [u'foo', u'bar'], [1, 2]), (u'requires_preop', True, [1]), "
"(u'retainsevents', True, [1]), (u'run_in_preview', True, [2]), (u'streaming', True, [1]), "
"(u'streaming_preop', u'some streaming command', [1, 2]), (u'type', u'streaming', [2])]"
)
else:
expected = (
"[('clear_required_fields', True, [1]), ('distributed', True, [2]), ('generates_timeorder', True, [1]), "
"('generating', True, [1, 2]), ('maxinputs', 50000, [2]), ('overrides_timeorder', True, [1]), "
"('required_fields', ['foo', 'bar'], [1, 2]), ('requires_preop', True, [1]), "
"('retainsevents', True, [1]), ('run_in_preview', True, [2]), ('streaming', True, [1]), "
"('streaming_preop', 'some streaming command', [1, 2]), ('type', 'streaming', [2])]"
)
self.assertEqual(repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error, result.getvalue()))
result.seek(0)
reader = csv.reader(result)
self.assertEqual([], next(reader))
observed = dict(izip(next(reader), next(reader)))
self.assertRaises(StopIteration, lambda: next(reader))
expected = {
'clear_required_fields': '1',
'__mv_clear_required_fields': '',
'generating': '1',
'__mv_generating': '',
'generates_timeorder': '1',
'__mv_generates_timeorder': '',
'overrides_timeorder': '1',
'__mv_overrides_timeorder': '',
'requires_preop': '1',
'__mv_requires_preop': '',
'required_fields': 'foo,bar',
'__mv_required_fields': '',
'retainsevents': '1',
'__mv_retainsevents': '',
'streaming': '1',
'__mv_streaming': '',
'streaming_preop': 'some streaming command',
'__mv_streaming_preop': '',
}
self.assertDictEqual(
expected,
observed) # No message header and no configuration settings
for action in '__GETINFO__', '__EXECUTE__':
# TestCommand.process should produce an error record on parser errors
argv = [
'test.py', action, 'required_option_1=value',
'required_option_2=value', 'undefined_option=value',
'fieldname_1', 'fieldname_2'
]
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=Unrecognized test command option: undefined_option="value"\r\n\r\n',
result.getvalue())
# TestCommand.process should produce an error record when required options are missing
argv = [
'test.py', action, 'required_option_2=value', 'fieldname_1'
]
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=A value for test command option required_option_1 is required\r\n\r\n',
result.getvalue())
argv = ['test.py', action, 'fieldname_1']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=Values for these test command options are required: required_option_1, required_option_2'
'\r\n\r\n', result.getvalue())
# TestStreamingCommand.process should exit on processing exceptions
ifile = StringIO('\naction\r\nraise_error\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(error.code, 0)
self.assertRegexpMatches(
result.getvalue(),
r'^error_message=RuntimeError at ".+", line \d+ : Testing\r\n\r\n$'
)
except BaseException as error:
self.fail('Expected SystemExit, but caught {}: {}'.format(
type(error).__name__, error))
else:
self.fail('Expected SystemExit, but no exception was raised')
# Command.process should provide access to search results info
info_path = os.path.join(self._package_directory, 'recordings',
'scpv1', 'Splunk-6.3',
'countmatches.execute.dispatch_dir',
'externSearchResultsInfo.csv')
ifile = StringIO('infoPath:' + info_path +
'\n\naction\r\nget_search_results_info\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('Expected no exception, but caught {}: {}'.format(
type(error).__name__, error))
else:
self.assertRegexpMatches(
result.getvalue(), r'^\r\n'
r'('
r'data,__mv_data,_serial,__mv__serial\r\n'
r'"\{.*u\'is_summary_index\': 0, .+\}",,0,'
r'|'
r'_serial,__mv__serial,data,__mv_data\r\n'
r'0,,"\{.*u\'is_summary_index\': 0, .+\}",'
r')'
r'\r\n$')
# TestStreamingCommand.process should provide access to a service object when search results info is available
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command.search_results_info.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command.search_results_info.auth_token)
self.assertEqual(command.service.namespace.app,
command.search_results_info.ppc_app)
self.assertEqual(command.service.namespace.owner, None)
self.assertEqual(command.service.namespace.sharing, None)
# Command.process should not provide access to search results info or a service object when the 'infoPath'
# input header is unavailable
ifile = StringIO('\naction\r\nget_search_results_info')
argv = ['teststreaming.py', '__EXECUTE__']
command = TestStreamingCommand()
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
self.assertIsNone(command.search_results_info)
self.assertIsNone(command.service)
return
def test_process_scpv2(self):
# SearchCommand.process should
# 1. Recognize all standard options:
metadata = (
'{{'
'"action": "getinfo", "preview": false, "searchinfo": {{'
'"latest_time": "0",'
'"splunk_version": "20150522",'
'"username": "******",'
'"app": "searchcommands_app",'
'"args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"search": "Ａ%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",'
'"earliest_time": "0",'
'"session_key": "0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",'
'"owner": "admin",'
'"sid": "1433261372.158",'
'"splunkd_uri": "https://127.0.0.1:8089",'
'"dispatch_dir": {dispatch_dir},'
'"raw_args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"maxresultrows": 10,'
'"command": "countmatches"'
'}}'
'}}')
basedir = self._package_directory
default_logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'default', 'logging.conf')
dispatch_dir = os.path.join(basedir, 'recordings', 'scpv2', 'Splunk-6.3', 'countmatches.dispatch_dir')
logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'logging.conf')
logging_level = 'ERROR'
record = False
show_configuration = True
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true' if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO(
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['some-external-search-command.py']
self.assertEqual(command.logging_level, 'WARNING')
self.assertIs(command.record, None)
self.assertIs(command.show_configuration, None)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.fail('Unexpected exception: {}: {}'.format(type(error).__name__, error))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,68,0\n'
'{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
'chunked 1.0,17,23\n'
'{"finished":true}test,__mv_test\r\n'
'data,\r\n',
result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 2. Provide access to these properties:
# fieldnames
# input_header
# metadata
# search_results_info
# service
self.assertEqual([], command.fieldnames)
command_metadata = command.metadata
input_header = command.input_header
self.assertIsNone(input_header['allowStream'])
self.assertEqual(input_header['infoPath'], os.path.join(command_metadata.searchinfo.dispatch_dir, 'info.csv'))
self.assertIsNone(input_header['keywords'])
self.assertEqual(input_header['preview'], command_metadata.preview)
self.assertIs(input_header['realtime'], False)
self.assertEqual(input_header['search'], command_metadata.searchinfo.search)
self.assertEqual(input_header['sid'], command_metadata.searchinfo.sid)
self.assertEqual(input_header['splunkVersion'], command_metadata.searchinfo.splunk_version)
self.assertIsNone(input_header['truncated'])
self.assertEqual(command_metadata.preview, input_header['preview'])
self.assertEqual(command_metadata.searchinfo.app, 'searchcommands_app')
self.assertEqual(command_metadata.searchinfo.args, ['logging_configuration=' + logging_configuration, 'logging_level=ERROR', 'record=false', 'show_configuration=true', 'required_option_1=value_1', 'required_option_2=value_2'])
self.assertEqual(command_metadata.searchinfo.dispatch_dir, os.path.dirname(input_header['infoPath']))
self.assertEqual(command_metadata.searchinfo.earliest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.latest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.owner, 'admin')
self.assertEqual(command_metadata.searchinfo.raw_args, command_metadata.searchinfo.args)
self.assertEqual(command_metadata.searchinfo.search, 'Ａ| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw')
self.assertEqual(command_metadata.searchinfo.session_key, '0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^')
self.assertEqual(command_metadata.searchinfo.sid, '1433261372.158')
self.assertEqual(command_metadata.searchinfo.splunk_version, '20150522')
self.assertEqual(command_metadata.searchinfo.splunkd_uri, 'https://127.0.0.1:8089')
self.assertEqual(command_metadata.searchinfo.username, 'admin')
self.assertEqual(command_metadata.searchinfo.maxresultrows, 10)
self.assertEqual(command_metadata.searchinfo.command, 'countmatches')
command.search_results_info.search_metrics = command.search_results_info.search_metrics.__dict__
command.search_results_info.optional_fields_json = command.search_results_info.optional_fields_json.__dict__
self.maxDiff = None
self.assertDictEqual(command.search_results_info.__dict__, {
u'is_summary_index': 0,
u'bs_thread_count': 1,
u'rt_backfill': 0,
u'rtspan': '',
u'search_StartTime': 1433261392.934936,
u'read_raw': 1,
u'root_sid': '',
u'field_rendering': '',
u'query_finished': 1,
u'optional_fields_json': {},
u'group_list': '',
u'remoteServers': '',
u'rt_latest': '',
u'remote_log_download_mode': 'disabled',
u'reduce_search': '',
u'request_finalization': 0,
u'auth_token': 'UQZSgWwE2f9oIKrj1QG^kVhW^T_cR4H5Z65bPtMhwlHytS5jFrFYyH^dGzjTusDjVTgoBNeR7bvIzctHF7DrLJ1ANevgDOWEWRvABNj6d_k0koqxw9Io',
u'indexed_realtime': 0,
u'ppc_bs': '$SPLUNK_HOME/etc',
u'drop_count': 0,
u'datamodel_map': '',
u'search_can_be_event_type': 0,
u'search_StartUp_Spent': 0,
u'realtime': 0,
u'splunkd_uri': 'https://127.0.0.1:8089',
u'columnOrder': '',
u'kv_store_settings': 'hosts;127.0.0.1:8191\\;;local;127.0.0.1:8191;read_preference;958513E3-8716-4ABF-9559-DA0C9678437F;replica_set_name;958513E3-8716-4ABF-9559-DA0C9678437F;status;ready;',
u'label': '',
u'summary_maxtimespan': '',
u'indexed_realtime_offset': 0,
u'sid': 1433261392.159,
u'msg': [],
u'internal_only': 0,
u'summary_id': '',
u'orig_search_head': '',
u'ppc_app': 'chunked_searchcommands',
u'countMap': {
u'invocations.dispatch.writeStatus': u'1',
u'duration.dispatch.writeStatus': u'2',
u'duration.startup.handoff': u'79',
u'duration.startup.configuration': u'34',
u'invocations.startup.handoff': u'1',
u'invocations.startup.configuration': u'1'},
u'is_shc_mode': 0,
u'shp_id': '958513E3-8716-4ABF-9559-DA0C9678437F',
u'timestamp': 1433261392.936374, u'is_remote_sorted': 0,
u'remote_search': '',
u'splunkd_protocol': 'https',
u'site': '',
u'maxevents': 0,
u'keySet': '',
u'summary_stopped': 0,
u'search_metrics': {
u'ConsideredEvents': 0,
u'ConsideredBuckets': 0,
u'TotalSlicesInBuckets': 0,
u'EliminatedBuckets': 0,
u'DecompressedSlices': 0},
u'summary_mode': 'all', u'now': 1433261392.0,
u'splunkd_port': 8089, u'is_saved_search': 0,
u'rtoptions': '',
u'search': '| inputlookup random_data max=50000 | sum total=total value1 record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw',
u'bundle_version': 0,
u'generation_id': 0,
u'bs_thread_id': 0,
u'is_batch_mode': 0,
u'scan_count': 0,
u'rt_earliest': '',
u'default_group': '*',
u'tstats_reduce': '',
u'kv_store_additional_settings': 'hosts_guids;958513E3-8716-4ABF-9559-DA0C9678437F\\;;',
u'enable_event_stream': 0,
u'is_remote': 0,
u'is_scheduled': 0,
u'sample_ratio': 1,
u'ppc_user': '******',
u'sample_seed': 0})
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority, command_metadata.searchinfo.splunkd_uri)
self.assertEqual(command.service.scheme, command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port, command.search_results_info.splunkd_port)
self.assertEqual(command.service.token, command_metadata.searchinfo.session_key)
self.assertEqual(command.service.namespace.app, command.metadata.searchinfo.app)
self.assertIsNone(command.service.namespace.owner)
self.assertIsNone(command.service.namespace.sharing)
self.assertEqual(command.protocol_version, 2)
# 3. Produce an error message, log a debug message, and exit when invalid standard option values are encountered
# Note on loggers
# Loggers are global and can't be removed once they're created. We create loggers that are keyed by class name
# Each instance of a class thus created gets access to the same logger. We created one in the prior test and
# set it's level to ERROR. That level is retained in this test.
logging_configuration = 'non-existent-logging.conf'
logging_level = 'NON-EXISTENT-LOGGING-LEVEL'
record = 'Non-boolean value'
show_configuration = 'Non-boolean value'
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=record,
show_configuration=show_configuration)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO(
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
# noinspection PyTypeChecker
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, False)
self.assertEqual(command.show_configuration, False)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,287,0\n'
'{"inspector":{"messages":[["ERROR","Illegal value: logging_configuration=non-existent-logging.conf"],'
'["ERROR","Illegal value: logging_level=NON-EXISTENT-LOGGING-LEVEL"],'
'["ERROR","Illegal value: record=Non-boolean value"],'
'["ERROR","Illegal value: show_configuration=Non-boolean value"]]}}\n'
'chunked 1.0,17,0\n'
'{"finished":true}',
result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 4. Produce an error message, log an error message that includes a traceback, and exit when an exception is
# raised during command execution.
logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'logging.conf')
logging_level = 'WARNING'
record = False
show_configuration = False
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true' if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'action\r\nraise_exception\r\n'
ifile = StringIO(
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
try:
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(0, error.code)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue()))
else:
self.fail('Expected SystemExit, not a return from TestCommand.process: {}\n'.format(result.getvalue()))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, logging_level)
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
finished = r'"finished":true'
inspector = \
r'"inspector":\{"messages":\[\["ERROR","StandardError at \\".+\\", line \d+ : test ' \
r'logging_configuration=\\".+\\" logging_level=\\"WARNING\\" record=\\"f\\" ' \
r'required_option_1=\\"value_1\\" required_option_2=\\"value_2\\" show_configuration=\\"f\\""\]\]\}'
self.assertRegexpMatches(
result.getvalue(),
r'^chunked 1.0,2,0\n'
r'\{\}\n'
r'chunked 1.0,\d+,0\n'
r'\{(' + inspector + r',' + finished + r'|' + finished + r',' + inspector + r')\}')
self.assertEqual(command.protocol_version, 2)
return
class TestRecorder(object):
def __init__(self, test_case):
self._test_case = test_case
self._output = None
self._recording = None
self._recording_part = None
def _not_implemented(self):
raise NotImplementedError('class {} is not in playback or record mode'.format(self.__class__.__name__))
self.get = self.next_part = self.stop = MethodType(_not_implemented, self, self.__class__)
return
@property
def output(self):
return self._output
def playback(self, path):
with open(path, 'rb') as f:
test_data = pickle.load(f)
self._output = StringIO()
self._recording = test_data['inputs']
self._recording_part = self._recording.popleft()
def get(self, method, *args, **kwargs):
return self._recording_part[method.__name__].popleft()
self.get = MethodType(get, self, self.__class__)
def next_part(self):
self._recording_part = self._recording.popleft()
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
self._test_case.assertEqual(test_data['results'], self._output.getvalue())
self.stop = MethodType(stop, self, self.__class__)
return
def record(self, path):
self._output = StringIO()
self._recording = deque()
self._recording_part = OrderedDict()
self._recording.append(self._recording_part)
def get(self, method, *args, **kwargs):
result = method(*args, **kwargs)
part = self._recording_part
key = method.__name__
try:
results = part[key]
except KeyError:
part[key] = results = deque()
results.append(result)
return result
self.get = MethodType(get, self, self.__class__)
def next_part(self):
part = OrderedDict()
self._recording_part = part
self._recording.append(part)
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
with io.open(path, 'wb') as f:
test = OrderedDict((('inputs', self._recording), ('results', self._output.getvalue())))
pickle.dump(test, f)
self.stop = MethodType(stop, self, self.__class__)
return
class RecordWriter(object):
def __init__(self, ofile, maxresultrows=None):
self._maxresultrows = 50000 if maxresultrows is None else maxresultrows
self._ofile = set_binary_mode(ofile)
self._fieldnames = None
self._buffer = StringIO()
self._writer = csv.writer(self._buffer, dialect=CsvDialect)
self._writerow = self._writer.writerow
self._finished = False
self._flushed = False
self._inspector = OrderedDict()
self._chunk_count = 0
self._pending_record_count = 0
self._committed_record_count = 0
@property
def is_flushed(self):
return self._flushed
@is_flushed.setter
def is_flushed(self, value):
self._flushed = True if value else False
@property
def ofile(self):
return self._ofile
@ofile.setter
def ofile(self, value):
self._ofile = set_binary_mode(value)
@property
def pending_record_count(self):
return self._pending_record_count
@property
def _record_count(self):
warnings.warn(
"_record_count will be deprecated soon. Use pending_record_count instead.",
PendingDeprecationWarning)
return self.pending_record_count
@property
def committed_record_count(self):
return self._committed_record_count
@property
def _total_record_count(self):
warnings.warn(
"_total_record_count will be deprecated soon. Use committed_record_count instead.",
PendingDeprecationWarning)
return self.committed_record_count
def write(self, data):
bytes_type = bytes if sys.version_info >= (3, 0) else str
if not isinstance(data, bytes_type):
data = data.encode('utf-8')
self.ofile.write(data)
def flush(self, finished=None, partial=None):
assert finished is None or isinstance(finished, bool)
assert partial is None or isinstance(partial, bool)
assert not (finished is None and partial is None)
assert finished is None or partial is None
self._ensure_validity()
def write_message(self, message_type, message_text, *args, **kwargs):
self._ensure_validity()
self._inspector.setdefault('messages', []).append(
(message_type, message_text.format(*args, **kwargs)))
def write_record(self, record):
self._ensure_validity()
self._write_record(record)
def write_records(self, records):
self._ensure_validity()
write_record = self._write_record
for record in records:
write_record(record)
def _clear(self):
self._buffer.seek(0)
self._buffer.truncate()
self._inspector.clear()
self._pending_record_count = 0
def _ensure_validity(self):
if self._finished is True:
assert self._record_count == 0 and len(self._inspector) == 0
raise RuntimeError('I/O operation on closed record writer')
def _write_record(self, record):
fieldnames = self._fieldnames
if fieldnames is None:
self._fieldnames = fieldnames = list(record.keys())
value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)),
fieldnames)
self._writerow(list(chain.from_iterable(value_list)))
get_value = record.get
values = []
for fieldname in fieldnames:
value = get_value(fieldname, None)
if value is None:
values += (None, None)
continue
value_t = type(value)
if issubclass(value_t, (list, tuple)):
if len(value) == 0:
values += (None, None)
continue
if len(value) > 1:
value_list = value
sv = ''
mv = '$'
for value in value_list:
if value is None:
sv += '\n'
mv += '$;$'
continue
value_t = type(value)
if value_t is not bytes:
if value_t is bool:
value = str(value.real)
elif value_t is six.text_type:
value = value
elif isinstance(
value, six.integer_types
) or value_t is float or value_t is complex:
value = str(value)
elif issubclass(value_t, (dict, list, tuple)):
value = str(''.join(
RecordWriter._iterencode_json(value, 0)))
else:
value = repr(value).encode(
'utf-8', errors='backslashreplace')
sv += value + '\n'
mv += value.replace('$', '$$') + '$;$'
values += (sv[:-1], mv[:-2])
continue
value = value[0]
value_t = type(value)
if value_t is bool:
values += (str(value.real), None)
continue
if value_t is bytes:
values += (value, None)
continue
if value_t is six.text_type:
if six.PY2:
value = value.encode('utf-8')
values += (value, None)
continue
if isinstance(value, six.integer_types
) or value_t is float or value_t is complex:
values += (str(value), None)
continue
if issubclass(value_t, dict):
values += (str(''.join(RecordWriter._iterencode_json(value,
0))),
None)
continue
values += (repr(value), None)
self._writerow(values)
self._pending_record_count += 1
if self.pending_record_count >= self._maxresultrows:
self.flush(partial=True)
try:
# noinspection PyUnresolvedReferences
from _json import make_encoder
except ImportError:
# We may be running under PyPy 2.5 which does not include the _json module
_iterencode_json = JSONEncoder(separators=(',', ':')).iterencode
else:
# Creating _iterencode_json this way yields a two-fold performance improvement on Python 2.7.9 and 2.7.10
from json.encoder import encode_basestring_ascii
@staticmethod
def _default(o):
raise TypeError(repr(o) + ' is not JSON serializable')
_iterencode_json = make_encoder(
{}, # markers (for detecting circular references)
_default, # object_encoder
encode_basestring_ascii, # string_encoder
None, # indent
':',
',', # separators
False, # sort_keys
False, # skip_keys
True # allow_nan
)
del make_encoder
def test_input_header(self):
# No items
input_header = InputHeader()
with closing(StringIO('\r\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
# One unnamed single-line item (same as no items)
input_header = InputHeader()
with closing(
StringIO('this%20is%20an%20unnamed%20single-line%20item\n\n'.
encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
input_header = InputHeader()
with closing(
StringIO('this%20is%20an%20unnamed\nmulti-\nline%20item\n\n'.
encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
# One named single-line item
input_header = InputHeader()
with closing(
StringIO('Foo:this%20is%20a%20single-line%20item\n\n'.encode())
) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEquals(input_header['Foo'], 'this is a single-line item')
input_header = InputHeader()
with closing(StringIO('Bar:this is a\nmulti-\nline item\n\n'.encode())
) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEquals(input_header['Bar'], 'this is a\nmulti-\nline item')
# The infoPath item (which is the path to a file that we open for reads)
input_header = InputHeader()
with closing(StringIO(
'infoPath:non-existent.csv\n\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEqual(input_header['infoPath'], 'non-existent.csv')
# Set of named items
collection = {
'word_list': 'hello\nworld\n!',
'word_1': 'hello',
'word_2': 'world',
'word_3': '!',
'sentence': 'hello world!'
}
input_header = InputHeader()
text = reduce(
lambda value, item: value + '{}:{}\n'.format(item[0], item[1]),
six.iteritems(collection), '') + '\n'
with closing(StringIO(text.encode())) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Set of named items with an unnamed item at the beginning (the only place that an unnamed item can appear)
with closing(StringIO(
('unnamed item\n' + text).encode())) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Test iterators, indirectly through items, keys, and values
self.assertEqual(sorted(input_header.items()),
sorted(collection.items()))
self.assertEqual(sorted(input_header.keys()),
sorted(collection.keys()))
self.assertEqual(sorted(input_header.values()),
sorted(collection.values()))
return
def format(self, value):
output = StringIO()
writer = csv.writer(output, List.Dialect)
writer.writerow(value)
value = output.getvalue()
return value[:-1]
def test_logging_configuration(self):
# Test that logging is properly initialized when there is no logging configuration file
rebase_environment('app_without_logging_configuration')
self.assertIsNone(environment.logging_configuration)
self.assertTrue(
any(
isinstance(h, logging.StreamHandler)
for h in logging.root.handlers))
self.assertTrue('splunklib' in logging.Logger.manager.loggerDict)
self.assertEqual(environment.splunklib_logger,
logging.Logger.manager.loggerDict['splunklib'])
self.assertIsInstance(environment.splunklib_logger, logging.Logger)
command = StubbedSearchCommand()
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
self.assertEqual(len(command.logger.handlers), 0)
self.assertIsNone(command.logging_configuration)
self.assertIs(command.logger.root, logging.root)
root_handler = next(h for h in logging.root.handlers
if isinstance(h, logging.StreamHandler))
self.assertIsInstance(root_handler, logging.StreamHandler)
self.assertEqual(root_handler.stream, sys.stderr)
self.assertEqual(command.logging_level,
logging.getLevelName(logging.root.level))
root_handler.stream = StringIO()
message = 'Test that output is directed to stderr without formatting'
command.logger.warning(message)
self.assertEqual(root_handler.stream.getvalue(), message + '\n')
# A search command loads {local,default}/logging.conf when it is available
rebase_environment('app_with_logging_configuration')
command = StubbedSearchCommand()
self.assertEqual(
command.logging_configuration,
os.path.join(environment.app_root, 'default', 'logging.conf'))
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
# Setting logging_configuration loads a new logging configuration file relative to the app root
command.logging_configuration = 'alternative-logging.conf'
self.assertEqual(
command.logging_configuration,
os.path.join(environment.app_root, 'default',
'alternative-logging.conf'))
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
# Setting logging_configuration loads a new logging configuration file on an absolute path
app_root_logging_configuration = os.path.join(environment.app_root,
'logging.conf')
command.logging_configuration = app_root_logging_configuration
self.assertEqual(command.logging_configuration,
app_root_logging_configuration)
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
# logging_configuration raises a value error, if a non-existent logging configuration file is provided
try:
command.logging_configuration = 'foo'
except ValueError:
pass
except BaseException as e:
self.fail('Expected ValueError, but {} was raised'.format(type(e)))
else:
self.fail(
'Expected ValueError, but logging_configuration={}'.format(
command.logging_configuration))
try:
command.logging_configuration = os.path.join(
package_directory, 'non-existent.logging.conf')
except ValueError:
pass
except BaseException as e:
self.fail('Expected ValueError, but {} was raised'.format(type(e)))
else:
self.fail(
'Expected ValueError, but logging_configuration={}'.format(
command.logging_configuration))
def test_process_scpv1(self):
# TestCommand.process should complain if supports_getinfo == False
# We support dynamic configuration, not static
# The exception line number may change, so we're using a regex match instead of a string match
expected = re.compile(
r'error_message=RuntimeError at ".+search_command\.py", line \d\d\d : Command test appears to be '
r'statically configured for search command protocol version 1 and static configuration is unsupported by '
r'splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza:\n'
r'\[test\]\n'
r'filename = test.py\n'
r'enableheader = true\n'
r'outputheader = true\n'
r'requires_srinfo = true\n'
r'supports_getinfo = true\n'
r'supports_multivalues = true\n'
r'supports_rawargs = true')
argv = ['test.py', 'not__GETINFO__or__EXECUTE__', 'option=value', 'fieldname']
command = TestCommand()
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ofile=result)
self.assertRegexpMatches(result.getvalue(), expected)
# TestCommand.process should return configuration settings on Getinfo probe
argv = ['test.py', '__GETINFO__', 'required_option_1=value', 'required_option_2=value']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertEqual(str(command.configuration), '')
if six.PY2:
expected = ("[(u'clear_required_fields', None, [1]), (u'distributed', None, [2]), (u'generates_timeorder', None, [1]), "
"(u'generating', None, [1, 2]), (u'maxinputs', None, [2]), (u'overrides_timeorder', None, [1]), "
"(u'required_fields', None, [1, 2]), (u'requires_preop', None, [1]), (u'retainsevents', None, [1]), "
"(u'run_in_preview', None, [2]), (u'streaming', None, [1]), (u'streaming_preop', None, [1, 2]), "
"(u'type', None, [2])]")
else:
expected = ("[('clear_required_fields', None, [1]), ('distributed', None, [2]), ('generates_timeorder', None, [1]), "
"('generating', None, [1, 2]), ('maxinputs', None, [2]), ('overrides_timeorder', None, [1]), "
"('required_fields', None, [1, 2]), ('requires_preop', None, [1]), ('retainsevents', None, [1]), "
"('run_in_preview', None, [2]), ('streaming', None, [1]), ('streaming_preop', None, [1, 2]), "
"('type', None, [2])]")
self.assertEqual(
repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue()))
self.assertEqual('\r\n\r\n\r\n', result.getvalue()) # No message header and no configuration settings
ifile = StringIO('\n')
result = StringIO()
# We might also put this sort of code into our SearchCommand.prepare override ...
configuration = command.configuration
# SCP v1/v2 configuration settings
configuration.generating = True
configuration.required_fields = ['foo', 'bar']
configuration.streaming_preop = 'some streaming command'
# SCP v1 configuration settings
configuration.clear_required_fields = True
configuration.generates_timeorder = True
configuration.overrides_timeorder = True
configuration.requires_preop = True
configuration.retainsevents = True
configuration.streaming = True
# SCP v2 configuration settings (SCP v1 requires that maxinputs and run_in_preview are set in commands.conf)
configuration.distributed = True
configuration.maxinputs = 50000
configuration.run_in_preview = True
configuration.type = 'streaming'
if six.PY2:
expected = ('clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[u\'foo\', u\'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
else:
expected = ('clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[\'foo\', \'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
self.assertEqual(str(command.configuration), expected)
if six.PY2:
expected = ("[(u'clear_required_fields', True, [1]), (u'distributed', True, [2]), (u'generates_timeorder', True, [1]), "
"(u'generating', True, [1, 2]), (u'maxinputs', 50000, [2]), (u'overrides_timeorder', True, [1]), "
"(u'required_fields', [u'foo', u'bar'], [1, 2]), (u'requires_preop', True, [1]), "
"(u'retainsevents', True, [1]), (u'run_in_preview', True, [2]), (u'streaming', True, [1]), "
"(u'streaming_preop', u'some streaming command', [1, 2]), (u'type', u'streaming', [2])]")
else:
expected = ("[('clear_required_fields', True, [1]), ('distributed', True, [2]), ('generates_timeorder', True, [1]), "
"('generating', True, [1, 2]), ('maxinputs', 50000, [2]), ('overrides_timeorder', True, [1]), "
"('required_fields', ['foo', 'bar'], [1, 2]), ('requires_preop', True, [1]), "
"('retainsevents', True, [1]), ('run_in_preview', True, [2]), ('streaming', True, [1]), "
"('streaming_preop', 'some streaming command', [1, 2]), ('type', 'streaming', [2])]")
self.assertEqual(
repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue()))
result.seek(0)
reader = csv.reader(result)
self.assertEqual([], next(reader))
observed = dict(izip(next(reader), next(reader)))
self.assertRaises(StopIteration, lambda: next(reader))
expected = {
'clear_required_fields': '1', '__mv_clear_required_fields': '',
'generating': '1', '__mv_generating': '',
'generates_timeorder': '1', '__mv_generates_timeorder': '',
'overrides_timeorder': '1', '__mv_overrides_timeorder': '',
'requires_preop': '1', '__mv_requires_preop': '',
'required_fields': 'foo,bar', '__mv_required_fields': '',
'retainsevents': '1', '__mv_retainsevents': '',
'streaming': '1', '__mv_streaming': '',
'streaming_preop': 'some streaming command', '__mv_streaming_preop': '',
}
self.assertDictEqual(expected, observed) # No message header and no configuration settings
for action in '__GETINFO__', '__EXECUTE__':
# TestCommand.process should produce an error record on parser errors
argv = [
'test.py', action, 'required_option_1=value', 'required_option_2=value', 'undefined_option=value',
'fieldname_1', 'fieldname_2']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertTrue(
'error_message=Unrecognized test command option: undefined_option="value"\r\n\r\n',
result.getvalue())
# TestCommand.process should produce an error record when required options are missing
argv = ['test.py', action, 'required_option_2=value', 'fieldname_1']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertTrue(
'error_message=A value for test command option required_option_1 is required\r\n\r\n',
result.getvalue())
argv = ['test.py', action, 'fieldname_1']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertTrue(
'error_message=Values for these test command options are required: required_option_1, required_option_2'
'\r\n\r\n',
result.getvalue())
# TestStreamingCommand.process should exit on processing exceptions
ifile = StringIO('\naction\r\nraise_error\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(error.code, 0)
self.assertRegexpMatches(
result.getvalue(),
r'^error_message=RuntimeError at ".+", line \d+ : Testing\r\n\r\n$')
except BaseException as error:
self.fail('Expected SystemExit, but caught {}: {}'.format(type(error).__name__, error))
else:
self.fail('Expected SystemExit, but no exception was raised')
# Command.process should provide access to search results info
info_path = os.path.join(
self._package_directory, 'recordings', 'scpv1', 'Splunk-6.3', 'countmatches.execute.dispatch_dir',
'externSearchResultsInfo.csv')
ifile = StringIO('infoPath:' + info_path + '\n\naction\r\nget_search_results_info\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('Expected no exception, but caught {}: {}'.format(type(error).__name__, error))
else:
self.assertRegexpMatches(
result.getvalue(),
r'^\r\n'
r'('
r'data,__mv_data,_serial,__mv__serial\r\n'
r'"\{.*u\'is_summary_index\': 0, .+\}",,0,'
r'|'
r'_serial,__mv__serial,data,__mv_data\r\n'
r'0,,"\{.*u\'is_summary_index\': 0, .+\}",'
r')'
r'\r\n$'
)
# TestStreamingCommand.process should provide access to a service object when search results info is available
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command.search_results_info.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command.search_results_info.auth_token)
self.assertEqual(command.service.namespace.app,
command.search_results_info.ppc_app)
self.assertEqual(command.service.namespace.owner,
None)
self.assertEqual(command.service.namespace.sharing,
None)
# Command.process should not provide access to search results info or a service object when the 'infoPath'
# input header is unavailable
ifile = StringIO('\naction\r\nget_search_results_info')
argv = ['teststreaming.py', '__EXECUTE__']
command = TestStreamingCommand()
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
self.assertIsNone(command.search_results_info)
self.assertIsNone(command.service)
return
def test_recorder(self):
if (python_version[0] == 2 and python_version[1] < 7):
print("Skipping test since we're on {1}".format(
"".join(python_version)))
pass
# Grab an input/output recording, the results of a prior countmatches run
recording = os.path.join(self._package_path, 'recordings', 'scpv2',
'Splunk-6.3', 'countmatches.')
with gzip.open(recording + 'input.gz', 'rb') as file_1:
with io.open(recording + 'output', 'rb') as file_2:
ifile = StringIO(file_1.read())
result = StringIO(file_2.read())
# Set up the input/output recorders that are under test
ifile = Recorder(mktemp(), ifile)
try:
ofile = Recorder(mktemp(), StringIO())
try:
# Read and then write a line
ifile.readline()
ofile.write(result.readline())
# Read and then write a block
ifile.read()
ofile.write(result.read())
# Verify that what we wrote is equivalent to the original recording, the result from a prior
# countmatches run
self.assertEqual(ofile.getvalue(), result.getvalue())
# Verify that we faithfully recorded the input and output files
ifile._recording.close()
ofile._recording.close()
with gzip.open(ifile._recording.name, 'rb') as file_1:
with gzip.open(ofile._recording.name, 'rb') as file_2:
self.assertEqual(file_1.read(), ifile._file.getvalue())
self.assertEqual(file_2.read(), ofile._file.getvalue())
finally:
ofile._recording.close()
os.remove(ofile._recording.name)
finally:
ifile._recording.close()
os.remove(ifile._recording.name)
return
class RecordWriter(object):
def __init__(self, ofile, maxresultrows=None):
self._maxresultrows = 50000 if maxresultrows is None else maxresultrows
self._ofile = ofile
self._fieldnames = None
self._buffer = StringIO()
self._writer = csv.writer(self._buffer, dialect=CsvDialect)
self._writerow = self._writer.writerow
self._finished = False
self._flushed = False
self._inspector = OrderedDict()
self._chunk_count = 0
self._record_count = 0
self._total_record_count = 0
@property
def is_flushed(self):
return self._flushed
@is_flushed.setter
def is_flushed(self, value):
self._flushed = True if value else False
@property
def ofile(self):
return self._ofile
@ofile.setter
def ofile(self, value):
self._ofile = value
def flush(self, finished=None, partial=None):
assert finished is None or isinstance(finished, bool)
assert partial is None or isinstance(partial, bool)
assert not (finished is None and partial is None)
assert finished is None or partial is None
self._ensure_validity()
def write_message(self, message_type, message_text, *args, **kwargs):
self._ensure_validity()
self._inspector.setdefault('messages', []).append((message_type, message_text.format(*args, **kwargs)))
def write_record(self, record):
self._ensure_validity()
self._write_record(record)
def write_records(self, records):
self._ensure_validity()
write_record = self._write_record
for record in records:
write_record(record)
def _clear(self):
self._buffer.seek(0)
self._buffer.truncate()
self._inspector.clear()
self._record_count = 0
self._flushed = False
def _ensure_validity(self):
if self._finished is True:
assert self._record_count == 0 and len(self._inspector) == 0
raise RuntimeError('I/O operation on closed record writer')
def _write_record(self, record):
fieldnames = self._fieldnames
if fieldnames is None:
self._fieldnames = fieldnames = list(record.keys())
value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames)
self._writerow(list(chain.from_iterable(value_list)))
get_value = record.get
values = []
for fieldname in fieldnames:
value = get_value(fieldname, None)
if value is None:
values += (None, None)
continue
value_t = type(value)
if issubclass(value_t, (list, tuple)):
if len(value) == 0:
values += (None, None)
continue
if len(value) > 1:
value_list = value
sv = ''
mv = '$'
for value in value_list:
if value is None:
sv += '\n'
mv += '$;$'
continue
value_t = type(value)
if value_t is not bytes:
if value_t is bool:
value = str(value.real)
elif value_t is six.text_type:
value = value
elif value_t is int or value_t is int or value_t is float or value_t is complex:
value = str(value)
elif issubclass(value_t, (dict, list, tuple)):
value = str(''.join(RecordWriter._iterencode_json(value, 0)))
else:
value = repr(value).encode('utf-8', errors='backslashreplace')
sv += value + '\n'
mv += value.replace('$', '$$') + '$;$'
values += (sv[:-1], mv[:-2])
continue
value = value[0]
value_t = type(value)
if value_t is bool:
values += (str(value.real), None)
continue
if value_t is bytes:
values += (value, None)
continue
if value_t is six.text_type:
if six.PY2:
value = value.encode('utf-8')
values += (value, None)
continue
if value_t is int or value_t is int or value_t is float or value_t is complex:
values += (str(value), None)
continue
if issubclass(value_t, dict):
values += (str(''.join(RecordWriter._iterencode_json(value, 0))), None)
continue
values += (repr(value), None)
self._writerow(values)
self._record_count += 1
if self._record_count >= self._maxresultrows:
self.flush(partial=True)
try:
# noinspection PyUnresolvedReferences
from _json import make_encoder
except ImportError:
# We may be running under PyPy 2.5 which does not include the _json module
_iterencode_json = JSONEncoder(separators=(',', ':')).iterencode
else:
# Creating _iterencode_json this way yields a two-fold performance improvement on Python 2.7.9 and 2.7.10
from json.encoder import encode_basestring_ascii
@staticmethod
def _default(o):
raise TypeError(repr(o) + ' is not JSON serializable')
_iterencode_json = make_encoder(
{}, # markers (for detecting circular references)
_default, # object_encoder
encode_basestring_ascii, # string_encoder
None, # indent
':', ',', # separators
False, # sort_keys
False, # skip_keys
True # allow_nan
)
del make_encoder
def test_process_scpv2(self):
# SearchCommand.process should
# 1. Recognize all standard options:
metadata = (
'{{'
'"action": "getinfo", "preview": false, "searchinfo": {{'
'"latest_time": "0",'
'"splunk_version": "20150522",'
'"username": "******",'
'"app": "searchcommands_app",'
'"args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"search": "Ａ%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",'
'"earliest_time": "0",'
'"session_key": "0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",'
'"owner": "admin",'
'"sid": "1433261372.158",'
'"splunkd_uri": "https://127.0.0.1:8089",'
'"dispatch_dir": {dispatch_dir},'
'"raw_args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"maxresultrows": 10,'
'"command": "countmatches"'
'}}'
'}}')
basedir = self._package_directory
default_logging_configuration = os.path.join(
basedir, 'apps', 'app_with_logging_configuration', 'default',
'logging.conf')
dispatch_dir = os.path.join(basedir, 'recordings', 'scpv2',
'Splunk-6.3', 'countmatches.dispatch_dir')
logging_configuration = os.path.join(basedir, 'apps',
'app_with_logging_configuration',
'logging.conf')
logging_level = 'ERROR'
record = False
show_configuration = True
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true'
if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['some-external-search-command.py']
self.assertEqual(command.logging_level, 'WARNING')
self.assertIs(command.record, None)
self.assertIs(command.show_configuration, None)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.fail('Unexpected exception: {}: {}'.format(
type(error).__name__, error))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,68,0\n'
'{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
'chunked 1.0,17,23\n'
'{"finished":true}test,__mv_test\r\n'
'data,\r\n', result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 2. Provide access to these properties:
# fieldnames
# input_header
# metadata
# search_results_info
# service
self.assertEqual([], command.fieldnames)
command_metadata = command.metadata
input_header = command.input_header
self.assertIsNone(input_header['allowStream'])
self.assertEqual(
input_header['infoPath'],
os.path.join(command_metadata.searchinfo.dispatch_dir, 'info.csv'))
self.assertIsNone(input_header['keywords'])
self.assertEqual(input_header['preview'], command_metadata.preview)
self.assertIs(input_header['realtime'], False)
self.assertEqual(input_header['search'],
command_metadata.searchinfo.search)
self.assertEqual(input_header['sid'], command_metadata.searchinfo.sid)
self.assertEqual(input_header['splunkVersion'],
command_metadata.searchinfo.splunk_version)
self.assertIsNone(input_header['truncated'])
self.assertEqual(command_metadata.preview, input_header['preview'])
self.assertEqual(command_metadata.searchinfo.app, 'searchcommands_app')
self.assertEqual(command_metadata.searchinfo.args, [
'logging_configuration=' + logging_configuration,
'logging_level=ERROR', 'record=false', 'show_configuration=true',
'required_option_1=value_1', 'required_option_2=value_2'
])
self.assertEqual(command_metadata.searchinfo.dispatch_dir,
os.path.dirname(input_header['infoPath']))
self.assertEqual(command_metadata.searchinfo.earliest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.latest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.owner, 'admin')
self.assertEqual(command_metadata.searchinfo.raw_args,
command_metadata.searchinfo.args)
self.assertEqual(
command_metadata.searchinfo.search,
'Ａ| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw'
)
self.assertEqual(
command_metadata.searchinfo.session_key,
'0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^'
)
self.assertEqual(command_metadata.searchinfo.sid, '1433261372.158')
self.assertEqual(command_metadata.searchinfo.splunk_version,
'20150522')
self.assertEqual(command_metadata.searchinfo.splunkd_uri,
'https://127.0.0.1:8089')
self.assertEqual(command_metadata.searchinfo.username, 'admin')
self.assertEqual(command_metadata.searchinfo.maxresultrows, 10)
self.assertEqual(command_metadata.searchinfo.command, 'countmatches')
command.search_results_info.search_metrics = command.search_results_info.search_metrics.__dict__
command.search_results_info.optional_fields_json = command.search_results_info.optional_fields_json.__dict__
self.maxDiff = None
self.assertDictEqual(
command.search_results_info.__dict__, {
u'is_summary_index': 0,
u'bs_thread_count': 1,
u'rt_backfill': 0,
u'rtspan': '',
u'search_StartTime': 1433261392.934936,
u'read_raw': 1,
u'root_sid': '',
u'field_rendering': '',
u'query_finished': 1,
u'optional_fields_json': {},
u'group_list': '',
u'remoteServers': '',
u'rt_latest': '',
u'remote_log_download_mode': 'disabled',
u'reduce_search': '',
u'request_finalization': 0,
u'auth_token':
'UQZSgWwE2f9oIKrj1QG^kVhW^T_cR4H5Z65bPtMhwlHytS5jFrFYyH^dGzjTusDjVTgoBNeR7bvIzctHF7DrLJ1ANevgDOWEWRvABNj6d_k0koqxw9Io',
u'indexed_realtime': 0,
u'ppc_bs': '$SPLUNK_HOME/etc',
u'drop_count': 0,
u'datamodel_map': '',
u'search_can_be_event_type': 0,
u'search_StartUp_Spent': 0,
u'realtime': 0,
u'splunkd_uri': 'https://127.0.0.1:8089',
u'columnOrder': '',
u'kv_store_settings':
'hosts;127.0.0.1:8191\\;;local;127.0.0.1:8191;read_preference;958513E3-8716-4ABF-9559-DA0C9678437F;replica_set_name;958513E3-8716-4ABF-9559-DA0C9678437F;status;ready;',
u'label': '',
u'summary_maxtimespan': '',
u'indexed_realtime_offset': 0,
u'sid': 1433261392.159,
u'msg': [],
u'internal_only': 0,
u'summary_id': '',
u'orig_search_head': '',
u'ppc_app': 'chunked_searchcommands',
u'countMap': {
u'invocations.dispatch.writeStatus': u'1',
u'duration.dispatch.writeStatus': u'2',
u'duration.startup.handoff': u'79',
u'duration.startup.configuration': u'34',
u'invocations.startup.handoff': u'1',
u'invocations.startup.configuration': u'1'
},
u'is_shc_mode': 0,
u'shp_id': '958513E3-8716-4ABF-9559-DA0C9678437F',
u'timestamp': 1433261392.936374,
u'is_remote_sorted': 0,
u'remote_search': '',
u'splunkd_protocol': 'https',
u'site': '',
u'maxevents': 0,
u'keySet': '',
u'summary_stopped': 0,
u'search_metrics': {
u'ConsideredEvents': 0,
u'ConsideredBuckets': 0,
u'TotalSlicesInBuckets': 0,
u'EliminatedBuckets': 0,
u'DecompressedSlices': 0
},
u'summary_mode': 'all',
u'now': 1433261392.0,
u'splunkd_port': 8089,
u'is_saved_search': 0,
u'rtoptions': '',
u'search':
'| inputlookup random_data max=50000 | sum total=total value1 record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw',
u'bundle_version': 0,
u'generation_id': 0,
u'bs_thread_id': 0,
u'is_batch_mode': 0,
u'scan_count': 0,
u'rt_earliest': '',
u'default_group': '*',
u'tstats_reduce': '',
u'kv_store_additional_settings':
'hosts_guids;958513E3-8716-4ABF-9559-DA0C9678437F\\;;',
u'enable_event_stream': 0,
u'is_remote': 0,
u'is_scheduled': 0,
u'sample_ratio': 1,
u'ppc_user': '******',
u'sample_seed': 0
})
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command_metadata.searchinfo.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command_metadata.searchinfo.session_key)
self.assertEqual(command.service.namespace.app,
command.metadata.searchinfo.app)
self.assertIsNone(command.service.namespace.owner)
self.assertIsNone(command.service.namespace.sharing)
self.assertEqual(command.protocol_version, 2)
# 3. Produce an error message, log a debug message, and exit when invalid standard option values are encountered
# Note on loggers
# Loggers are global and can't be removed once they're created. We create loggers that are keyed by class name
# Each instance of a class thus created gets access to the same logger. We created one in the prior test and
# set it's level to ERROR. That level is retained in this test.
logging_configuration = 'non-existent-logging.conf'
logging_level = 'NON-EXISTENT-LOGGING-LEVEL'
record = 'Non-boolean value'
show_configuration = 'Non-boolean value'
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=record,
show_configuration=show_configuration)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
# noinspection PyTypeChecker
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, False)
self.assertEqual(command.show_configuration, False)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,287,0\n'
'{"inspector":{"messages":[["ERROR","Illegal value: logging_configuration=non-existent-logging.conf"],'
'["ERROR","Illegal value: logging_level=NON-EXISTENT-LOGGING-LEVEL"],'
'["ERROR","Illegal value: record=Non-boolean value"],'
'["ERROR","Illegal value: show_configuration=Non-boolean value"]]}}\n'
'chunked 1.0,17,0\n'
'{"finished":true}', result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 4. Produce an error message, log an error message that includes a traceback, and exit when an exception is
# raised during command execution.
logging_configuration = os.path.join(basedir, 'apps',
'app_with_logging_configuration',
'logging.conf')
logging_level = 'WARNING'
record = False
show_configuration = False
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true'
if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'action\r\nraise_exception\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
try:
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(0, error.code)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error, result.getvalue()))
else:
self.fail(
'Expected SystemExit, not a return from TestCommand.process: {}\n'
.format(result.getvalue()))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, logging_level)
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
finished = r'"finished":true'
inspector = \
r'"inspector":\{"messages":\[\["ERROR","StandardError at \\".+\\", line \d+ : test ' \
r'logging_configuration=\\".+\\" logging_level=\\"WARNING\\" record=\\"f\\" ' \
r'required_option_1=\\"value_1\\" required_option_2=\\"value_2\\" show_configuration=\\"f\\""\]\]\}'
self.assertRegexpMatches(
result.getvalue(), r'^chunked 1.0,2,0\n'
r'\{\}\n'
r'chunked 1.0,\d+,0\n'
r'\{(' + inspector + r',' + finished + r'|' + finished + r',' +
inspector + r')\}')
self.assertEqual(command.protocol_version, 2)
return