def _compare_chunks(self, expected, output, time_sensitive=True):
expected = expected.strip()
output = output.strip()
if time_sensitive:
self.assertEqual(len(expected), len(output))
compare_csv_files = self._compare_csv_files_time_sensitive
else:
compare_csv_files = self._compare_csv_files_time_insensitive
chunks_1 = self._load_chunks(StringIO(expected))
chunks_2 = self._load_chunks(StringIO(output))
self.assertEqual(len(chunks_1), len(chunks_2))
n = 0
for chunk_1, chunk_2 in izip(chunks_1, chunks_2):
self.assertDictEqual(
chunk_1.metadata, chunk_2.metadata,
'Chunk {0}: metadata error: "{1}" != "{2}"'.format(
n, chunk_1.metadata, chunk_2.metadata))
compare_csv_files(chunk_1.body, chunk_2.body)
n += 1
return
def _compare_csv_files_time_insensitive(self, expected, output):
skip_first_row = expected[0:2] == '\r\n'
expected = StringIO(expected)
output = StringIO(output)
line_number = 1
if skip_first_row:
self.assertEqual(expected.readline(), output.readline())
line_number += 1
expected = csv.DictReader(expected)
output = csv.DictReader(output)
for expected_row in expected:
output_row = next(output)
try:
timestamp = float(output_row['_time'])
datetime.fromtimestamp(timestamp)
except BaseException as error:
self.fail(error)
else:
output_row['_time'] = expected_row['_time']
self.assertDictEqual(
expected_row, output_row,
'Error on line {0}: expected {1}, not {2}'.format(
line_number, expected_row, output_row))
line_number += 1
self.assertRaises(StopIteration, output.next)
return
def _compare_csv_files_time_sensitive(self, expected, output):
self.assertEqual(len(expected), len(output))
skip_first_row = expected[0:2] == '\r\n'
expected = StringIO(expected)
output = StringIO(output)
line_number = 1
if skip_first_row:
self.assertEqual(expected.readline(), output.readline())
line_number += 1
expected = csv.DictReader(expected)
output = csv.DictReader(output)
for expected_row in expected:
output_row = next(output)
self.assertDictEqual(
expected_row, output_row,
'Error on line {0}: expected {1}, not {2}'.format(
line_number, expected_row, output_row))
line_number += 1
self.assertRaises(StopIteration, output.next)
return
def test_successful_validation(self):
"""Check that successful validation yield no text and a 0 exit value."""
# Override abstract methods
class NewScript(Script):
def get_scheme(self):
return None
def validate_input(self, definition):
# always succeed...
return
def stream_events(self, inputs, ew):
# unused
return
script = NewScript()
out = StringIO()
err = StringIO()
ew = EventWriter(out, err)
args = [TEST_SCRIPT_PATH, "--validate-arguments"]
return_value = script.run_script(args, ew,
data_open("data/validation.xml"))
self.assertEqual("", err.getvalue())
self.assertEqual("", out.getvalue())
self.assertEqual(0, return_value)
def test_recorder(self):
if (python_version[0] == 2 and python_version[1] < 7):
print("Skipping test since we're on {1}".format(
"".join(python_version)))
pass
# Grab an input/output recording, the results of a prior countmatches run
recording = os.path.join(self._package_path, 'recordings', 'scpv2',
'Splunk-6.3', 'countmatches.')
with gzip.open(recording + 'input.gz', 'rb') as file_1:
with io.open(recording + 'output', 'rb') as file_2:
ifile = StringIO(file_1.read())
result = StringIO(file_2.read())
# Set up the input/output recorders that are under test
ifile = Recorder(mktemp(), ifile)
try:
ofile = Recorder(mktemp(), StringIO())
try:
# Read and then write a line
ifile.readline()
ofile.write(result.readline())
# Read and then write a block
ifile.read()
ofile.write(result.read())
# Verify that what we wrote is equivalent to the original recording, the result from a prior
# countmatches run
self.assertEqual(ofile.getvalue(), result.getvalue())
# Verify that we faithfully recorded the input and output files
ifile._recording.close()
ofile._recording.close()
with gzip.open(ifile._recording.name, 'rb') as file_1:
with gzip.open(ofile._recording.name, 'rb') as file_2:
self.assertEqual(file_1.read(), ifile._file.getvalue())
self.assertEqual(file_2.read(), ofile._file.getvalue())
finally:
ofile._recording.close()
os.remove(ofile._recording.name)
finally:
ifile._recording.close()
os.remove(ifile._recording.name)
return
def assertInfoEqual(self, output, expected):
reader = csv.reader(StringIO(output))
self.assertEqual([], next(reader))
fields = next(reader)
values = next(reader)
self.assertRaises(StopIteration, reader.next)
output = dict(izip(fields, values))
reader = csv.reader(StringIO(expected))
self.assertEqual([], next(reader))
fields = next(reader)
values = next(reader)
self.assertRaises(StopIteration, reader.next)
expected = dict(izip(fields, values))
self.assertDictEqual(expected, output)
def make_request(self, url, method, data, headers):
self.log_message("%s: %s", method, url)
try:
# Make the request
request = urllib.Request(url, data, headers)
request.get_method = lambda: method
response = urllib.urlopen(request)
# We were successful, so send the response code
self.send_response(response.code, message=response.msg)
for key, value in iteritems(dict(response.headers)):
# Optionally log the headers
#self.log_message("%s: %s" % (key, value))
self.send_header(key, value)
# Send the cross-domain headers
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods",
"PUT, POST, GET, DELETE, OPTIONS")
self.send_header("Access-Control-Allow-Headers",
"X-Redirect-URL, Authorization")
# We are done with the headers
self.end_headers()
# Copy the response to the output
self.copyfile(response, self.wfile)
except urllib.HTTPError as e:
# On errors, log the response code and message
self.log_message("Code: %s (%s)", e.code, e.msg)
for key, value in iteritems(dict(e.hdrs)):
# On errors, we always log the headers
self.log_message("%s: %s", key, value)
response_text = e.fp.read()
response_file = StringIO(response_text)
# On errors, we also log the response text
self.log_message("Response: %s", response_text)
# Send the error response code
self.send_response(e.code, message=e.msg)
# Send the cross-domain headers
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods",
"PUT, POST, GET, DELETE, OPTIONS")
self.send_header("Access-Control-Allow-Headers",
"X-Redirect-URL, Authorization")
# Send the other headers
self.send_header("Content-Type", self.error_content_type)
self.send_header('Connection', 'close')
self.end_headers()
# Finally, send the error itself
self.copyfile(response_file, self.wfile)
def test_error_on_script_with_null_scheme(self):
"""A script that returns a null scheme should generate no output on
stdout and an error on stderr saying that it the scheme was null."""
# Override abstract methods
class NewScript(Script):
def get_scheme(self):
return None
def stream_events(self, inputs, ew):
# not used
return
script = NewScript()
out = BytesIO()
err = BytesIO()
ew = EventWriter(out, err)
in_stream = StringIO()
args = [TEST_SCRIPT_PATH, "--scheme"]
return_value = script.run_script(args, ew, in_stream)
self.assertEqual(b"", out.getvalue())
self.assertEqual(
b"FATAL Modular input script returned a null scheme.\n",
err.getvalue())
self.assertNotEqual(0, return_value)
def test_messages_header(self):
@Configuration()
class TestMessagesHeaderCommand(SearchCommand):
class ConfigurationSettings(SearchCommand.ConfigurationSettings):
@classmethod
def fix_up(cls, command_class):
pass
command = TestMessagesHeaderCommand()
command._protocol_version = 1
output_buffer = StringIO()
command._record_writer = RecordWriterV1(output_buffer)
messages = [(command.write_debug, 'debug_message'),
(command.write_error, 'error_message'),
(command.write_fatal, 'fatal_message'),
(command.write_info, 'info_message'),
(command.write_warning, 'warning_message')]
for write, message in messages:
write(message)
command.finish()
expected = ('debug_message=debug_message\r\n'
'error_message=error_message\r\n'
'error_message=fatal_message\r\n'
'info_message=info_message\r\n'
'warn_message=warning_message\r\n'
'\r\n')
self.assertEquals(output_buffer.getvalue(), expected)
return
def playback(self, path):
with open(path, 'rb') as f:
test_data = pickle.load(f)
self._output = StringIO()
self._recording = test_data['inputs']
self._recording_part = self._recording.popleft()
def get(self, method, *args, **kwargs):
return self._recording_part[method.__name__].popleft()
self.get = MethodType(get, self, self.__class__)
def next_part(self):
self._recording_part = self._recording.popleft()
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
self._test_case.assertEqual(test_data['results'],
self._output.getvalue())
self.stop = MethodType(stop, self, self.__class__)
return
def test_event_without_enough_fields_fails(self):
"""Check that events without data throw an error"""
with self.assertRaises(ValueError):
event = Event()
stream = StringIO()
event.write_to(stream)
self.assertTrue(True)
def _execute_chunk_v2(self, process, chunk):
metadata, body = chunk
if len(body) <= 0:
return
records = self._read_csv_records(StringIO(body))
self._record_writer.write_records(process(records))
def _execute_chunk_v2(self, process, chunk):
metadata, body = chunk
if len(body) <= 0 and not self._allow_empty_input:
raise ValueError(
"No records found to process. Set allow_empty_input=True in dispatch function to move forward "
"with empty records.")
records = self._read_csv_records(StringIO(body))
self._record_writer.write_records(process(records))
def _records_protocol_v2(self, ifile):
istream = self._as_binary_stream(ifile)
while True:
result = self._read_chunk(istream)
if not result:
return
metadata, body = result
action = getattr(metadata, 'action', None)
if action != 'execute':
raise RuntimeError(
'Expected execute action, not {}'.format(action))
finished = getattr(metadata, 'finished', False)
self._record_writer.is_flushed = False
if len(body) > 0:
reader = csv.reader(StringIO(body), dialect=CsvDialect)
try:
fieldnames = next(reader)
except StopIteration:
return
mv_fieldnames = dict([(name, name[len('__mv_'):])
for name in fieldnames
if name.startswith('__mv_')])
if len(mv_fieldnames) == 0:
for values in reader:
yield OrderedDict(izip(fieldnames, values))
else:
for values in reader:
record = OrderedDict()
for fieldname, value in izip(fieldnames, values):
if fieldname.startswith('__mv_'):
if len(value) > 0:
record[mv_fieldnames[
fieldname]] = self._decode_list(value)
elif fieldname not in record:
record[fieldname] = value
yield record
if finished:
return
self.flush()
def __init__(self, ofile, maxresultrows=None):
self._maxresultrows = 50000 if maxresultrows is None else maxresultrows
self._ofile = set_binary_mode(ofile)
self._fieldnames = None
self._buffer = StringIO()
self._writer = csv.writer(self._buffer, dialect=CsvDialect)
self._writerow = self._writer.writerow
self._finished = False
self._flushed = False
self._inspector = OrderedDict()
self._chunk_count = 0
self._pending_record_count = 0
self._committed_record_count = 0
def record(self, path):
self._output = StringIO()
self._recording = deque()
self._recording_part = OrderedDict()
self._recording.append(self._recording_part)
def get(self, method, *args, **kwargs):
result = method(*args, **kwargs)
part = self._recording_part
key = method.__name__
try:
results = part[key]
except KeyError:
part[key] = results = deque()
results.append(result)
return result
self.get = MethodType(get, self, self.__class__)
def next_part(self):
part = OrderedDict()
self._recording_part = part
self._recording.append(part)
self.next_part = MethodType(next_part, self, self.__class__)
def stop(self):
with io.open(path, 'wb') as f:
test = OrderedDict(
(('inputs', self._recording), ('results',
self._output.getvalue())))
pickle.dump(test, f)
self.stop = MethodType(stop, self, self.__class__)
return
def format(self, value):
output = StringIO()
writer = csv.writer(output, List.Dialect)
writer.writerow(value)
value = output.getvalue()
return value[:-1]
def test_logging_configuration(self):
# Test that logging is properly initialized when there is no logging configuration file
rebase_environment('app_without_logging_configuration')
self.assertIsNone(environment.logging_configuration)
self.assertTrue(
any(
isinstance(h, logging.StreamHandler)
for h in logging.root.handlers))
self.assertTrue('splunklib' in logging.Logger.manager.loggerDict)
self.assertEqual(environment.splunklib_logger,
logging.Logger.manager.loggerDict['splunklib'])
self.assertIsInstance(environment.splunklib_logger, logging.Logger)
command = StubbedSearchCommand()
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
self.assertEqual(len(command.logger.handlers), 0)
self.assertIsNone(command.logging_configuration)
self.assertIs(command.logger.root, logging.root)
root_handler = next(h for h in logging.root.handlers
if isinstance(h, logging.StreamHandler))
self.assertIsInstance(root_handler, logging.StreamHandler)
self.assertEqual(root_handler.stream, sys.stderr)
self.assertEqual(command.logging_level,
logging.getLevelName(logging.root.level))
root_handler.stream = StringIO()
message = 'Test that output is directed to stderr without formatting'
command.logger.warning(message)
self.assertEqual(root_handler.stream.getvalue(), message + '\n')
# A search command loads {local,default}/logging.conf when it is available
rebase_environment('app_with_logging_configuration')
command = StubbedSearchCommand()
self.assertEqual(
command.logging_configuration,
os.path.join(environment.app_root, 'default', 'logging.conf'))
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
# Setting logging_configuration loads a new logging configuration file relative to the app root
command.logging_configuration = 'alternative-logging.conf'
self.assertEqual(
command.logging_configuration,
os.path.join(environment.app_root, 'default',
'alternative-logging.conf'))
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
# Setting logging_configuration loads a new logging configuration file on an absolute path
app_root_logging_configuration = os.path.join(environment.app_root,
'logging.conf')
command.logging_configuration = app_root_logging_configuration
self.assertEqual(command.logging_configuration,
app_root_logging_configuration)
self.assertIs(command.logger,
logging.getLogger('StubbedSearchCommand'))
# logging_configuration raises a value error, if a non-existent logging configuration file is provided
try:
command.logging_configuration = 'foo'
except ValueError:
pass
except BaseException as e:
self.fail('Expected ValueError, but {} was raised'.format(type(e)))
else:
self.fail(
'Expected ValueError, but logging_configuration={}'.format(
command.logging_configuration))
try:
command.logging_configuration = os.path.join(
package_directory, 'non-existent.logging.conf')
except ValueError:
pass
except BaseException as e:
self.fail('Expected ValueError, but {} was raised'.format(type(e)))
else:
self.fail(
'Expected ValueError, but logging_configuration={}'.format(
command.logging_configuration))
def test_record_writer_with_recordings(self):
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join(
(cls.__name__, method.__name__)))
for input_file in iglob(base_path + '*.input.gz'):
with gzip.open(input_file, 'rb') as ifile:
test_data = pickle.load(ifile)
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
write_record = writer.write_record
fieldnames = test_data['fieldnames']
for values in test_data['values']:
record = OrderedDict(izip(fieldnames, values))
try:
write_record(record)
except Exception as error:
self.fail(error)
for message_type, message_text in test_data['messages']:
writer.write_message(message_type, '{}', message_text)
for name, metric in six.iteritems(test_data['metrics']):
writer.write_metric(name, metric)
writer.flush(finished=True)
# Read expected data
expected_path = os.path.splitext(
os.path.splitext(input_file)[0])[0] + '.output'
with io.open(expected_path, 'rb') as ifile:
expected = ifile.read()
expected = self._load_chunks(StringIO(expected))
# Read observed data
ifile = writer._ofile
ifile.seek(0)
observed = self._load_chunks(ifile)
# Write observed data (as an aid to diagnostics)
observed_path = expected_path + '.observed'
observed_value = ifile.getvalue()
with io.open(observed_path, 'wb') as ifile:
ifile.write(observed_value)
self._compare_chunks(observed, expected)
return
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = [
'_serial', '_time', 'random_bytes', 'random_dict',
'random_integers', 'random_unicode'
]
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [
serial_number,
time(),
random_bytes(),
random_dict(),
random_integers(),
random_unicode()
]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [('debug', random_unicode()), ('error', random_unicode()),
('fatal', random_unicode()), ('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(
ifilter(lambda k_v: k_v[0].startswith('metric.'),
six.iteritems(writer._inspector))),
dict(
imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]),
six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError,
writer.flush,
finished=True,
partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(
self._recordings_path, '.'.join(
(cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def test_input_header(self):
# No items
input_header = InputHeader()
with closing(StringIO('\r\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
# One unnamed single-line item (same as no items)
input_header = InputHeader()
with closing(
StringIO('this%20is%20an%20unnamed%20single-line%20item\n\n'.
encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
input_header = InputHeader()
with closing(
StringIO('this%20is%20an%20unnamed\nmulti-\nline%20item\n\n'.
encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 0)
# One named single-line item
input_header = InputHeader()
with closing(
StringIO('Foo:this%20is%20a%20single-line%20item\n\n'.encode())
) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEquals(input_header['Foo'], 'this is a single-line item')
input_header = InputHeader()
with closing(StringIO('Bar:this is a\nmulti-\nline item\n\n'.encode())
) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEquals(input_header['Bar'], 'this is a\nmulti-\nline item')
# The infoPath item (which is the path to a file that we open for reads)
input_header = InputHeader()
with closing(StringIO(
'infoPath:non-existent.csv\n\n'.encode())) as input_file:
input_header.read(input_file)
self.assertEquals(len(input_header), 1)
self.assertEqual(input_header['infoPath'], 'non-existent.csv')
# Set of named items
collection = {
'word_list': 'hello\nworld\n!',
'word_1': 'hello',
'word_2': 'world',
'word_3': '!',
'sentence': 'hello world!'
}
input_header = InputHeader()
text = reduce(
lambda value, item: value + '{}:{}\n'.format(item[0], item[1]),
six.iteritems(collection), '') + '\n'
with closing(StringIO(text.encode())) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Set of named items with an unnamed item at the beginning (the only place that an unnamed item can appear)
with closing(StringIO(
('unnamed item\n' + text).encode())) as input_file:
input_header.read(input_file)
self.assertDictEqual(input_header, collection)
# Test iterators, indirectly through items, keys, and values
self.assertEqual(sorted(input_header.items()),
sorted(collection.items()))
self.assertEqual(sorted(input_header.keys()),
sorted(collection.keys()))
self.assertEqual(sorted(input_header.values()),
sorted(collection.values()))
return
def test_process_scpv1(self):
# TestCommand.process should complain if supports_getinfo == False
# We support dynamic configuration, not static
# The exception line number may change, so we're using a regex match instead of a string match
expected = re.compile(
r'error_message=RuntimeError at ".+search_command\.py", line \d\d\d : Command test appears to be '
r'statically configured for search command protocol version 1 and static configuration is unsupported by '
r'splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza:\n'
r'\[test\]\n'
r'filename = test.py\n'
r'enableheader = true\n'
r'outputheader = true\n'
r'requires_srinfo = true\n'
r'supports_getinfo = true\n'
r'supports_multivalues = true\n'
r'supports_rawargs = true')
argv = [
'test.py', 'not__GETINFO__or__EXECUTE__', 'option=value',
'fieldname'
]
command = TestCommand()
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ofile=result)
self.assertRegexpMatches(result.getvalue(), expected)
# TestCommand.process should return configuration settings on Getinfo probe
argv = [
'test.py', '__GETINFO__', 'required_option_1=value',
'required_option_2=value'
]
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertEqual(str(command.configuration), '')
if six.PY2:
expected = (
"[(u'clear_required_fields', None, [1]), (u'distributed', None, [2]), (u'generates_timeorder', None, [1]), "
"(u'generating', None, [1, 2]), (u'maxinputs', None, [2]), (u'overrides_timeorder', None, [1]), "
"(u'required_fields', None, [1, 2]), (u'requires_preop', None, [1]), (u'retainsevents', None, [1]), "
"(u'run_in_preview', None, [2]), (u'streaming', None, [1]), (u'streaming_preop', None, [1, 2]), "
"(u'type', None, [2])]")
else:
expected = (
"[('clear_required_fields', None, [1]), ('distributed', None, [2]), ('generates_timeorder', None, [1]), "
"('generating', None, [1, 2]), ('maxinputs', None, [2]), ('overrides_timeorder', None, [1]), "
"('required_fields', None, [1, 2]), ('requires_preop', None, [1]), ('retainsevents', None, [1]), "
"('run_in_preview', None, [2]), ('streaming', None, [1]), ('streaming_preop', None, [1, 2]), "
"('type', None, [2])]")
self.assertEqual(repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error, result.getvalue()))
self.assertEqual('\r\n\r\n\r\n', result.getvalue()
) # No message header and no configuration settings
ifile = StringIO('\n')
result = StringIO()
# We might also put this sort of code into our SearchCommand.prepare override ...
configuration = command.configuration
# SCP v1/v2 configuration settings
configuration.generating = True
configuration.required_fields = ['foo', 'bar']
configuration.streaming_preop = 'some streaming command'
# SCP v1 configuration settings
configuration.clear_required_fields = True
configuration.generates_timeorder = True
configuration.overrides_timeorder = True
configuration.requires_preop = True
configuration.retainsevents = True
configuration.streaming = True
# SCP v2 configuration settings (SCP v1 requires that maxinputs and run_in_preview are set in commands.conf)
configuration.distributed = True
configuration.maxinputs = 50000
configuration.run_in_preview = True
configuration.type = 'streaming'
if six.PY2:
expected = (
'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[u\'foo\', u\'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
else:
expected = (
'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[\'foo\', \'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
self.assertEqual(str(command.configuration), expected)
if six.PY2:
expected = (
"[(u'clear_required_fields', True, [1]), (u'distributed', True, [2]), (u'generates_timeorder', True, [1]), "
"(u'generating', True, [1, 2]), (u'maxinputs', 50000, [2]), (u'overrides_timeorder', True, [1]), "
"(u'required_fields', [u'foo', u'bar'], [1, 2]), (u'requires_preop', True, [1]), "
"(u'retainsevents', True, [1]), (u'run_in_preview', True, [2]), (u'streaming', True, [1]), "
"(u'streaming_preop', u'some streaming command', [1, 2]), (u'type', u'streaming', [2])]"
)
else:
expected = (
"[('clear_required_fields', True, [1]), ('distributed', True, [2]), ('generates_timeorder', True, [1]), "
"('generating', True, [1, 2]), ('maxinputs', 50000, [2]), ('overrides_timeorder', True, [1]), "
"('required_fields', ['foo', 'bar'], [1, 2]), ('requires_preop', True, [1]), "
"('retainsevents', True, [1]), ('run_in_preview', True, [2]), ('streaming', True, [1]), "
"('streaming_preop', 'some streaming command', [1, 2]), ('type', 'streaming', [2])]"
)
self.assertEqual(repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error, result.getvalue()))
result.seek(0)
reader = csv.reader(result)
self.assertEqual([], next(reader))
observed = dict(izip(next(reader), next(reader)))
self.assertRaises(StopIteration, lambda: next(reader))
expected = {
'clear_required_fields': '1',
'__mv_clear_required_fields': '',
'generating': '1',
'__mv_generating': '',
'generates_timeorder': '1',
'__mv_generates_timeorder': '',
'overrides_timeorder': '1',
'__mv_overrides_timeorder': '',
'requires_preop': '1',
'__mv_requires_preop': '',
'required_fields': 'foo,bar',
'__mv_required_fields': '',
'retainsevents': '1',
'__mv_retainsevents': '',
'streaming': '1',
'__mv_streaming': '',
'streaming_preop': 'some streaming command',
'__mv_streaming_preop': '',
}
self.assertDictEqual(
expected,
observed) # No message header and no configuration settings
for action in '__GETINFO__', '__EXECUTE__':
# TestCommand.process should produce an error record on parser errors
argv = [
'test.py', action, 'required_option_1=value',
'required_option_2=value', 'undefined_option=value',
'fieldname_1', 'fieldname_2'
]
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=Unrecognized test command option: undefined_option="value"\r\n\r\n',
result.getvalue())
# TestCommand.process should produce an error record when required options are missing
argv = [
'test.py', action, 'required_option_2=value', 'fieldname_1'
]
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=A value for test command option required_option_1 is required\r\n\r\n',
result.getvalue())
argv = ['test.py', action, 'fieldname_1']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=Values for these test command options are required: required_option_1, required_option_2'
'\r\n\r\n', result.getvalue())
# TestStreamingCommand.process should exit on processing exceptions
ifile = StringIO('\naction\r\nraise_error\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(error.code, 0)
self.assertRegexpMatches(
result.getvalue(),
r'^error_message=RuntimeError at ".+", line \d+ : Testing\r\n\r\n$'
)
except BaseException as error:
self.fail('Expected SystemExit, but caught {}: {}'.format(
type(error).__name__, error))
else:
self.fail('Expected SystemExit, but no exception was raised')
# Command.process should provide access to search results info
info_path = os.path.join(self._package_directory, 'recordings',
'scpv1', 'Splunk-6.3',
'countmatches.execute.dispatch_dir',
'externSearchResultsInfo.csv')
ifile = StringIO('infoPath:' + info_path +
'\n\naction\r\nget_search_results_info\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('Expected no exception, but caught {}: {}'.format(
type(error).__name__, error))
else:
self.assertRegexpMatches(
result.getvalue(), r'^\r\n'
r'('
r'data,__mv_data,_serial,__mv__serial\r\n'
r'"\{.*u\'is_summary_index\': 0, .+\}",,0,'
r'|'
r'_serial,__mv__serial,data,__mv_data\r\n'
r'0,,"\{.*u\'is_summary_index\': 0, .+\}",'
r')'
r'\r\n$')
# TestStreamingCommand.process should provide access to a service object when search results info is available
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command.search_results_info.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command.search_results_info.auth_token)
self.assertEqual(command.service.namespace.app,
command.search_results_info.ppc_app)
self.assertEqual(command.service.namespace.owner, None)
self.assertEqual(command.service.namespace.sharing, None)
# Command.process should not provide access to search results info or a service object when the 'infoPath'
# input header is unavailable
ifile = StringIO('\naction\r\nget_search_results_info')
argv = ['teststreaming.py', '__EXECUTE__']
command = TestStreamingCommand()
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
self.assertIsNone(command.search_results_info)
self.assertIsNone(command.service)
return
def test_process_scpv2(self):
# SearchCommand.process should
# 1. Recognize all standard options:
metadata = (
'{{'
'"action": "getinfo", "preview": false, "searchinfo": {{'
'"latest_time": "0",'
'"splunk_version": "20150522",'
'"username": "******",'
'"app": "searchcommands_app",'
'"args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"search": "Ａ%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",'
'"earliest_time": "0",'
'"session_key": "0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",'
'"owner": "admin",'
'"sid": "1433261372.158",'
'"splunkd_uri": "https://127.0.0.1:8089",'
'"dispatch_dir": {dispatch_dir},'
'"raw_args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"maxresultrows": 10,'
'"command": "countmatches"'
'}}'
'}}')
basedir = self._package_directory
default_logging_configuration = os.path.join(
basedir, 'apps', 'app_with_logging_configuration', 'default',
'logging.conf')
dispatch_dir = os.path.join(basedir, 'recordings', 'scpv2',
'Splunk-6.3', 'countmatches.dispatch_dir')
logging_configuration = os.path.join(basedir, 'apps',
'app_with_logging_configuration',
'logging.conf')
logging_level = 'ERROR'
record = False
show_configuration = True
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true'
if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['some-external-search-command.py']
self.assertEqual(command.logging_level, 'WARNING')
self.assertIs(command.record, None)
self.assertIs(command.show_configuration, None)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.fail('Unexpected exception: {}: {}'.format(
type(error).__name__, error))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,68,0\n'
'{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
'chunked 1.0,17,23\n'
'{"finished":true}test,__mv_test\r\n'
'data,\r\n', result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 2. Provide access to these properties:
# fieldnames
# input_header
# metadata
# search_results_info
# service
self.assertEqual([], command.fieldnames)
command_metadata = command.metadata
input_header = command.input_header
self.assertIsNone(input_header['allowStream'])
self.assertEqual(
input_header['infoPath'],
os.path.join(command_metadata.searchinfo.dispatch_dir, 'info.csv'))
self.assertIsNone(input_header['keywords'])
self.assertEqual(input_header['preview'], command_metadata.preview)
self.assertIs(input_header['realtime'], False)
self.assertEqual(input_header['search'],
command_metadata.searchinfo.search)
self.assertEqual(input_header['sid'], command_metadata.searchinfo.sid)
self.assertEqual(input_header['splunkVersion'],
command_metadata.searchinfo.splunk_version)
self.assertIsNone(input_header['truncated'])
self.assertEqual(command_metadata.preview, input_header['preview'])
self.assertEqual(command_metadata.searchinfo.app, 'searchcommands_app')
self.assertEqual(command_metadata.searchinfo.args, [
'logging_configuration=' + logging_configuration,
'logging_level=ERROR', 'record=false', 'show_configuration=true',
'required_option_1=value_1', 'required_option_2=value_2'
])
self.assertEqual(command_metadata.searchinfo.dispatch_dir,
os.path.dirname(input_header['infoPath']))
self.assertEqual(command_metadata.searchinfo.earliest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.latest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.owner, 'admin')
self.assertEqual(command_metadata.searchinfo.raw_args,
command_metadata.searchinfo.args)
self.assertEqual(
command_metadata.searchinfo.search,
'Ａ| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw'
)
self.assertEqual(
command_metadata.searchinfo.session_key,
'0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^'
)
self.assertEqual(command_metadata.searchinfo.sid, '1433261372.158')
self.assertEqual(command_metadata.searchinfo.splunk_version,
'20150522')
self.assertEqual(command_metadata.searchinfo.splunkd_uri,
'https://127.0.0.1:8089')
self.assertEqual(command_metadata.searchinfo.username, 'admin')
self.assertEqual(command_metadata.searchinfo.maxresultrows, 10)
self.assertEqual(command_metadata.searchinfo.command, 'countmatches')
command.search_results_info.search_metrics = command.search_results_info.search_metrics.__dict__
command.search_results_info.optional_fields_json = command.search_results_info.optional_fields_json.__dict__
self.maxDiff = None
self.assertDictEqual(
command.search_results_info.__dict__, {
u'is_summary_index': 0,
u'bs_thread_count': 1,
u'rt_backfill': 0,
u'rtspan': '',
u'search_StartTime': 1433261392.934936,
u'read_raw': 1,
u'root_sid': '',
u'field_rendering': '',
u'query_finished': 1,
u'optional_fields_json': {},
u'group_list': '',
u'remoteServers': '',
u'rt_latest': '',
u'remote_log_download_mode': 'disabled',
u'reduce_search': '',
u'request_finalization': 0,
u'auth_token':
'UQZSgWwE2f9oIKrj1QG^kVhW^T_cR4H5Z65bPtMhwlHytS5jFrFYyH^dGzjTusDjVTgoBNeR7bvIzctHF7DrLJ1ANevgDOWEWRvABNj6d_k0koqxw9Io',
u'indexed_realtime': 0,
u'ppc_bs': '$SPLUNK_HOME/etc',
u'drop_count': 0,
u'datamodel_map': '',
u'search_can_be_event_type': 0,
u'search_StartUp_Spent': 0,
u'realtime': 0,
u'splunkd_uri': 'https://127.0.0.1:8089',
u'columnOrder': '',
u'kv_store_settings':
'hosts;127.0.0.1:8191\\;;local;127.0.0.1:8191;read_preference;958513E3-8716-4ABF-9559-DA0C9678437F;replica_set_name;958513E3-8716-4ABF-9559-DA0C9678437F;status;ready;',
u'label': '',
u'summary_maxtimespan': '',
u'indexed_realtime_offset': 0,
u'sid': 1433261392.159,
u'msg': [],
u'internal_only': 0,
u'summary_id': '',
u'orig_search_head': '',
u'ppc_app': 'chunked_searchcommands',
u'countMap': {
u'invocations.dispatch.writeStatus': u'1',
u'duration.dispatch.writeStatus': u'2',
u'duration.startup.handoff': u'79',
u'duration.startup.configuration': u'34',
u'invocations.startup.handoff': u'1',
u'invocations.startup.configuration': u'1'
},
u'is_shc_mode': 0,
u'shp_id': '958513E3-8716-4ABF-9559-DA0C9678437F',
u'timestamp': 1433261392.936374,
u'is_remote_sorted': 0,
u'remote_search': '',
u'splunkd_protocol': 'https',
u'site': '',
u'maxevents': 0,
u'keySet': '',
u'summary_stopped': 0,
u'search_metrics': {
u'ConsideredEvents': 0,
u'ConsideredBuckets': 0,
u'TotalSlicesInBuckets': 0,
u'EliminatedBuckets': 0,
u'DecompressedSlices': 0
},
u'summary_mode': 'all',
u'now': 1433261392.0,
u'splunkd_port': 8089,
u'is_saved_search': 0,
u'rtoptions': '',
u'search':
'| inputlookup random_data max=50000 | sum total=total value1 record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw',
u'bundle_version': 0,
u'generation_id': 0,
u'bs_thread_id': 0,
u'is_batch_mode': 0,
u'scan_count': 0,
u'rt_earliest': '',
u'default_group': '*',
u'tstats_reduce': '',
u'kv_store_additional_settings':
'hosts_guids;958513E3-8716-4ABF-9559-DA0C9678437F\\;;',
u'enable_event_stream': 0,
u'is_remote': 0,
u'is_scheduled': 0,
u'sample_ratio': 1,
u'ppc_user': '******',
u'sample_seed': 0
})
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command_metadata.searchinfo.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command_metadata.searchinfo.session_key)
self.assertEqual(command.service.namespace.app,
command.metadata.searchinfo.app)
self.assertIsNone(command.service.namespace.owner)
self.assertIsNone(command.service.namespace.sharing)
self.assertEqual(command.protocol_version, 2)
# 3. Produce an error message, log a debug message, and exit when invalid standard option values are encountered
# Note on loggers
# Loggers are global and can't be removed once they're created. We create loggers that are keyed by class name
# Each instance of a class thus created gets access to the same logger. We created one in the prior test and
# set it's level to ERROR. That level is retained in this test.
logging_configuration = 'non-existent-logging.conf'
logging_level = 'NON-EXISTENT-LOGGING-LEVEL'
record = 'Non-boolean value'
show_configuration = 'Non-boolean value'
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=record,
show_configuration=show_configuration)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
# noinspection PyTypeChecker
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, False)
self.assertEqual(command.show_configuration, False)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,287,0\n'
'{"inspector":{"messages":[["ERROR","Illegal value: logging_configuration=non-existent-logging.conf"],'
'["ERROR","Illegal value: logging_level=NON-EXISTENT-LOGGING-LEVEL"],'
'["ERROR","Illegal value: record=Non-boolean value"],'
'["ERROR","Illegal value: show_configuration=Non-boolean value"]]}}\n'
'chunked 1.0,17,0\n'
'{"finished":true}', result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 4. Produce an error message, log an error message that includes a traceback, and exit when an exception is
# raised during command execution.
logging_configuration = os.path.join(basedir, 'apps',
'app_with_logging_configuration',
'logging.conf')
logging_level = 'WARNING'
record = False
show_configuration = False
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true'
if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'action\r\nraise_exception\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
try:
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(0, error.code)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error, result.getvalue()))
else:
self.fail(
'Expected SystemExit, not a return from TestCommand.process: {}\n'
.format(result.getvalue()))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, logging_level)
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
finished = r'"finished":true'
inspector = \
r'"inspector":\{"messages":\[\["ERROR","StandardError at \\".+\\", line \d+ : test ' \
r'logging_configuration=\\".+\\" logging_level=\\"WARNING\\" record=\\"f\\" ' \
r'required_option_1=\\"value_1\\" required_option_2=\\"value_2\\" show_configuration=\\"f\\""\]\]\}'
self.assertRegexpMatches(
result.getvalue(), r'^chunked 1.0,2,0\n'
r'\{\}\n'
r'chunked 1.0,\d+,0\n'
r'\{(' + inspector + r',' + finished + r'|' + finished + r',' +
inspector + r')\}')
self.assertEqual(command.protocol_version, 2)
return