def _one_chunk_of_records(self, ifile):
self._finished = True
metadata_length, body_length = self._start_chunk(ifile)
if metadata_length is None:
self.logger.info("No chunk; exiting...")
return
self.logger.info('Start data chunk...({},{})'.format(metadata_length, body_length))
metadata = self._read_metadata(ifile, metadata_length)
action = getattr(metadata, 'action', None)
if action != 'execute':
raise RuntimeError('Expected execute action, not {}'.format(action))
finished = getattr(metadata, 'finished', False)
self._record_writer.is_flushed = False
if body_length is 0:
return
copy_input = True
if copy_input:
self._capture_input(ifile, body_length)
reader = csv.reader(self._icopy, dialect=CsvDialect)
else:
reader = csv.reader(ChunkedInput(ifile, body_length), dialect=CsvDialect)
try:
fieldnames = next(reader)
except StopIteration:
raise RuntimeError('CSV header malformed')
self.logger.debug('Read records...')
mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')])
if len(mv_fieldnames) == 0:
for values in reader:
yield OrderedDict(izip(fieldnames, values))
else:
for values in reader:
record = OrderedDict()
for fieldname, value in izip(fieldnames, values):
if fieldname.startswith('__mv_'):
if len(value) > 0:
record[mv_fieldnames[fieldname]] = self._decode_list(value)
elif fieldname not in record:
record[fieldname] = value
yield record
if not self._icopy is None:
self._icopy.close()
os.remove(self._icopy_path)
if finished:
return
self._finished = False
def _records_protocol_v1(self, ifile):
reader = csv.reader(ifile, dialect=CsvDialect)
try:
fieldnames = next(reader)
except StopIteration:
return
mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')])
if len(mv_fieldnames) == 0:
for values in reader:
yield OrderedDict(izip(fieldnames, values))
return
for values in reader:
record = OrderedDict()
for fieldname, value in izip(fieldnames, values):
if fieldname.startswith('__mv_'):
if len(value) > 0:
record[mv_fieldnames[fieldname]] = self._decode_list(value)
elif fieldname not in record:
record[fieldname] = value
yield record
def _read_csv_records(self, ifile):
reader = csv.reader(ifile, dialect=CsvDialect)
try:
fieldnames = next(reader)
except StopIteration:
return
mv_fieldnames = dict([(name, name[len('__mv_'):])
for name in fieldnames
if name.startswith('__mv_')])
if len(mv_fieldnames) == 0:
for values in reader:
yield OrderedDict(izip(fieldnames, values))
return
for values in reader:
record = OrderedDict()
for fieldname, value in izip(fieldnames, values):
if fieldname.startswith('__mv_'):
if len(value) > 0:
record[mv_fieldnames[fieldname]] = self._decode_list(
value)
elif fieldname not in record:
record[fieldname] = value
yield record
def _records_protocol_v2(self, ifile):
istream = self._as_binary_stream(ifile)
while True:
result = self._read_chunk(istream)
if not result:
return
metadata, body = result
action = getattr(metadata, 'action', None)
if action != 'execute':
raise RuntimeError(
'Expected execute action, not {}'.format(action))
finished = getattr(metadata, 'finished', False)
self._record_writer.is_flushed = False
if len(body) > 0:
reader = csv.reader(StringIO(body), dialect=CsvDialect)
try:
fieldnames = next(reader)
except StopIteration:
return
mv_fieldnames = dict([(name, name[len('__mv_'):])
for name in fieldnames
if name.startswith('__mv_')])
if len(mv_fieldnames) == 0:
for values in reader:
yield OrderedDict(izip(fieldnames, values))
else:
for values in reader:
record = OrderedDict()
for fieldname, value in izip(fieldnames, values):
if fieldname.startswith('__mv_'):
if len(value) > 0:
record[mv_fieldnames[
fieldname]] = self._decode_list(value)
elif fieldname not in record:
record[fieldname] = value
yield record
if finished:
return
self.flush()
def __init__(self, path):
self._dispatch_dir = path + '.dispatch_dir'
self._search = None
if os.path.exists(self._dispatch_dir):
with io.open(os.path.join(self._dispatch_dir,
'request.csv')) as ifile:
reader = csv.reader(ifile)
for name, value in izip(next(reader), next(reader)):
if name == 'search':
self._search = value
break
assert self._search is not None
splunk_cmd = path + '.splunk_cmd'
try:
with io.open(splunk_cmd, 'r') as f:
self._args = f.readline().encode().split(
None, 5) # ['splunk', 'cmd', <filename>, <action>, <args>]
except IOError as error:
if error.errno != 2:
raise
self._args = ['splunk', 'cmd', 'python', None]
self._input_file = path + '.input.gz'
self._output_file = path + '.output'
if six.PY3 and os.path.isfile(self._output_file + '.py3'):
self._output_file = self._output_file + '.py3'
# Remove the "splunk cmd" portion
self._args = self._args[2:]
def assertInfoEqual(self, output, expected):
reader = csv.reader(StringIO(output))
self.assertEqual([], next(reader))
fields = next(reader)
values = next(reader)
self.assertRaises(StopIteration, reader.next)
output = dict(izip(fields, values))
reader = csv.reader(StringIO(expected))
self.assertEqual([], next(reader))
fields = next(reader)
values = next(reader)
self.assertRaises(StopIteration, reader.next)
expected = dict(izip(fields, values))
self.assertDictEqual(expected, output)
def _records_protocol_v2(self, ifile):
while True:
result = self._read_chunk(ifile)
if not result:
return
metadata, body = result
action = getattr(metadata, 'action', None)
if action != 'execute':
raise RuntimeError('Expected execute action, not {}'.format(action))
finished = getattr(metadata, 'finished', False)
self._record_writer.is_flushed = False
if len(body) > 0:
reader = csv.reader(StringIO(body), dialect=CsvDialect)
try:
fieldnames = next(reader)
except StopIteration:
return
mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')])
if len(mv_fieldnames) == 0:
for values in reader:
yield OrderedDict(izip(fieldnames, values))
else:
for values in reader:
record = OrderedDict()
for fieldname, value in izip(fieldnames, values):
if fieldname.startswith('__mv_'):
if len(value) > 0:
record[mv_fieldnames[fieldname]] = self._decode_list(value)
elif fieldname not in record:
record[fieldname] = value
yield record
if finished:
return
self.flush()
def test_record_writer_with_recordings(self):
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__)))
for input_file in iglob(base_path + '*.input.gz'):
with gzip.open(input_file, 'rb') as ifile:
test_data = pickle.load(ifile)
writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test
write_record = writer.write_record
fieldnames = test_data['fieldnames']
for values in test_data['values']:
record = OrderedDict(izip(fieldnames, values))
try:
write_record(record)
except Exception as error:
self.fail(error)
for message_type, message_text in test_data['messages']:
writer.write_message(message_type, '{}', message_text)
for name, metric in six.iteritems(test_data['metrics']):
writer.write_metric(name, metric)
writer.flush(finished=True)
# Read expected data
expected_path = os.path.splitext(os.path.splitext(input_file)[0])[0] + '.output'
with io.open(expected_path, 'rb') as ifile:
expected = ifile.read()
expected = self._load_chunks(StringIO(expected))
# Read observed data
ifile = writer._ofile
ifile.seek(0)
observed = self._load_chunks(ifile)
# Write observed data (as an aid to diagnostics)
observed_path = expected_path + '.observed'
observed_value = ifile.getvalue()
with io.open(observed_path, 'wb') as ifile:
ifile.write(observed_value)
self._compare_chunks(observed, expected)
return
def _compare_chunks(self, chunks_1, chunks_2):
self.assertEqual(len(chunks_1), len(chunks_2))
n = 0
for chunk_1, chunk_2 in izip(chunks_1, chunks_2):
self.assertDictEqual(
chunk_1.metadata, chunk_2.metadata,
'Chunk {0}: metadata error: "{1}" != "{2}"'.format(n, chunk_1.metadata, chunk_2.metadata))
self.assertMultiLineEqual(chunk_1.body, chunk_2.body, 'Chunk {0}: data error'.format(n))
n += 1
return
def _run(self):
writer = RecordWriterV2(self.recorder.output, maxresultrows=10)
write_record = writer.write_record
names = self.fieldnames
for self._serial_number in range(0, 31):
record = OrderedDict(izip(names, self.row))
write_record(record)
return
def _run(self):
writer = RecordWriterV2(self.recorder.output, maxresultrows=10)
write_record = writer.write_record
names = self.fieldnames
for self._serial_number in range(0, 31):
record = OrderedDict(izip(names, self.row))
write_record(record)
return
def _compare_chunks(self, chunks_1, chunks_2):
self.assertEqual(len(chunks_1), len(chunks_2))
n = 0
for chunk_1, chunk_2 in izip(chunks_1, chunks_2):
self.assertDictEqual(
chunk_1.metadata, chunk_2.metadata,
'Chunk {0}: metadata error: "{1}" != "{2}"'.format(
n, chunk_1.metadata, chunk_2.metadata))
self.assertMultiLineEqual(chunk_1.body, chunk_2.body,
'Chunk {0}: data error'.format(n))
n += 1
return
def _compare_chunks(self, expected, output, time_sensitive=True):
expected = expected.strip()
output = output.strip()
if time_sensitive:
compare_csv_files = self._compare_csv_files_time_sensitive
else:
compare_csv_files = self._compare_csv_files_time_insensitive
chunks_1 = self._load_chunks(StringIO(expected))
chunks_2 = self._load_chunks(StringIO(output))
self.assertEqual(len(chunks_1), len(chunks_2))
n = 0
for chunk_1, chunk_2 in izip(chunks_1, chunks_2):
self.assertDictEqual(
chunk_1.metadata, chunk_2.metadata,
'Chunk {0}: metadata error: "{1}" != "{2}"'.format(
n, chunk_1.metadata, chunk_2.metadata))
compare_csv_files(chunk_1.body, chunk_2.body)
n += 1
return
def search_results_info(self):
""" Returns the search results info for this command invocation.
The search results info object is created from the search results info file associated with the command
invocation.
:return: Search results info:const:`None`, if the search results info file associated with the command
invocation is inaccessible.
:rtype: SearchResultsInfo or NoneType
"""
if self._search_results_info is not None:
return self._search_results_info
if self._protocol_version == 1:
try:
path = self._input_header['infoPath']
except KeyError:
return None
else:
assert self._protocol_version == 2
try:
dispatch_dir = self._metadata.searchinfo.dispatch_dir
except AttributeError:
return None
path = os.path.join(dispatch_dir, 'info.csv')
try:
with io.open(path, 'r') as f:
reader = csv.reader(f, dialect=CsvDialect)
fields = next(reader)
values = next(reader)
except IOError as error:
if error.errno == 2:
self.logger.error(
'Search results info file {} does not exist.'.format(
json_encode_string(path)))
return
raise
def convert_field(field):
return (field[1:] if field[0] == '_' else field).replace('.', '_')
decode = MetadataDecoder().decode
def convert_value(value):
try:
return decode(value) if len(value) > 0 else value
except ValueError:
return value
info = ObjectView(
dict(
imap(
lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])),
izip(fields, values))))
try:
count_map = info.countMap
except AttributeError:
pass
else:
count_map = count_map.split(';')
n = len(count_map)
info.countMap = dict(
izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2)))
try:
msg_type = info.msgType
msg_text = info.msg
except AttributeError:
pass
else:
messages = ifilter(
lambda t_m: t_m[0] or t_m[1],
izip(msg_type.split('\n'), msg_text.split('\n')))
info.msg = [Message(message) for message in messages]
del info.msgType
try:
info.vix_families = ElementTree.fromstring(info.vix_families)
except AttributeError:
pass
self._search_results_info = info
return info
def test_process_scpv1(self):
# TestCommand.process should complain if supports_getinfo == False
# We support dynamic configuration, not static
# The exception line number may change, so we're using a regex match instead of a string match
expected = re.compile(
r'error_message=RuntimeError at ".+search_command\.py", line \d\d\d : Command test appears to be '
r'statically configured for search command protocol version 1 and static configuration is unsupported by '
r'splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza:\n'
r'\[test\]\n'
r'filename = test.py\n'
r'enableheader = true\n'
r'outputheader = true\n'
r'requires_srinfo = true\n'
r'supports_getinfo = true\n'
r'supports_multivalues = true\n'
r'supports_rawargs = true')
argv = ['test.py', 'not__GETINFO__or__EXECUTE__', 'option=value', 'fieldname']
command = TestCommand()
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ofile=result)
self.assertRegexpMatches(result.getvalue(), expected)
# TestCommand.process should return configuration settings on Getinfo probe
argv = ['test.py', '__GETINFO__', 'required_option_1=value', 'required_option_2=value']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertEqual(str(command.configuration), '')
if six.PY2:
expected = ("[(u'clear_required_fields', None, [1]), (u'distributed', None, [2]), (u'generates_timeorder', None, [1]), "
"(u'generating', None, [1, 2]), (u'maxinputs', None, [2]), (u'overrides_timeorder', None, [1]), "
"(u'required_fields', None, [1, 2]), (u'requires_preop', None, [1]), (u'retainsevents', None, [1]), "
"(u'run_in_preview', None, [2]), (u'streaming', None, [1]), (u'streaming_preop', None, [1, 2]), "
"(u'type', None, [2])]")
else:
expected = ("[('clear_required_fields', None, [1]), ('distributed', None, [2]), ('generates_timeorder', None, [1]), "
"('generating', None, [1, 2]), ('maxinputs', None, [2]), ('overrides_timeorder', None, [1]), "
"('required_fields', None, [1, 2]), ('requires_preop', None, [1]), ('retainsevents', None, [1]), "
"('run_in_preview', None, [2]), ('streaming', None, [1]), ('streaming_preop', None, [1, 2]), "
"('type', None, [2])]")
self.assertEqual(
repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue()))
self.assertEqual('\r\n\r\n\r\n', result.getvalue()) # No message header and no configuration settings
ifile = StringIO('\n')
result = StringIO()
# We might also put this sort of code into our SearchCommand.prepare override ...
configuration = command.configuration
# SCP v1/v2 configuration settings
configuration.generating = True
configuration.required_fields = ['foo', 'bar']
configuration.streaming_preop = 'some streaming command'
# SCP v1 configuration settings
configuration.clear_required_fields = True
configuration.generates_timeorder = True
configuration.overrides_timeorder = True
configuration.requires_preop = True
configuration.retainsevents = True
configuration.streaming = True
# SCP v2 configuration settings (SCP v1 requires that maxinputs and run_in_preview are set in commands.conf)
configuration.distributed = True
configuration.maxinputs = 50000
configuration.run_in_preview = True
configuration.type = 'streaming'
if six.PY2:
expected = ('clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[u\'foo\', u\'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
else:
expected = ('clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[\'foo\', \'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
self.assertEqual(str(command.configuration), expected)
if six.PY2:
expected = ("[(u'clear_required_fields', True, [1]), (u'distributed', True, [2]), (u'generates_timeorder', True, [1]), "
"(u'generating', True, [1, 2]), (u'maxinputs', 50000, [2]), (u'overrides_timeorder', True, [1]), "
"(u'required_fields', [u'foo', u'bar'], [1, 2]), (u'requires_preop', True, [1]), "
"(u'retainsevents', True, [1]), (u'run_in_preview', True, [2]), (u'streaming', True, [1]), "
"(u'streaming_preop', u'some streaming command', [1, 2]), (u'type', u'streaming', [2])]")
else:
expected = ("[('clear_required_fields', True, [1]), ('distributed', True, [2]), ('generates_timeorder', True, [1]), "
"('generating', True, [1, 2]), ('maxinputs', 50000, [2]), ('overrides_timeorder', True, [1]), "
"('required_fields', ['foo', 'bar'], [1, 2]), ('requires_preop', True, [1]), "
"('retainsevents', True, [1]), ('run_in_preview', True, [2]), ('streaming', True, [1]), "
"('streaming_preop', 'some streaming command', [1, 2]), ('type', 'streaming', [2])]")
self.assertEqual(
repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue()))
result.seek(0)
reader = csv.reader(result)
self.assertEqual([], next(reader))
observed = dict(izip(next(reader), next(reader)))
self.assertRaises(StopIteration, lambda: next(reader))
expected = {
'clear_required_fields': '1', '__mv_clear_required_fields': '',
'generating': '1', '__mv_generating': '',
'generates_timeorder': '1', '__mv_generates_timeorder': '',
'overrides_timeorder': '1', '__mv_overrides_timeorder': '',
'requires_preop': '1', '__mv_requires_preop': '',
'required_fields': 'foo,bar', '__mv_required_fields': '',
'retainsevents': '1', '__mv_retainsevents': '',
'streaming': '1', '__mv_streaming': '',
'streaming_preop': 'some streaming command', '__mv_streaming_preop': '',
}
self.assertDictEqual(expected, observed) # No message header and no configuration settings
for action in '__GETINFO__', '__EXECUTE__':
# TestCommand.process should produce an error record on parser errors
argv = [
'test.py', action, 'required_option_1=value', 'required_option_2=value', 'undefined_option=value',
'fieldname_1', 'fieldname_2']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertTrue(
'error_message=Unrecognized test command option: undefined_option="value"\r\n\r\n',
result.getvalue())
# TestCommand.process should produce an error record when required options are missing
argv = ['test.py', action, 'required_option_2=value', 'fieldname_1']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertTrue(
'error_message=A value for test command option required_option_1 is required\r\n\r\n',
result.getvalue())
argv = ['test.py', action, 'fieldname_1']
command = TestCommand()
ifile = StringIO('\n')
result = StringIO()
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertTrue(
'error_message=Values for these test command options are required: required_option_1, required_option_2'
'\r\n\r\n',
result.getvalue())
# TestStreamingCommand.process should exit on processing exceptions
ifile = StringIO('\naction\r\nraise_error\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(error.code, 0)
self.assertRegexpMatches(
result.getvalue(),
r'^error_message=RuntimeError at ".+", line \d+ : Testing\r\n\r\n$')
except BaseException as error:
self.fail('Expected SystemExit, but caught {}: {}'.format(type(error).__name__, error))
else:
self.fail('Expected SystemExit, but no exception was raised')
# Command.process should provide access to search results info
info_path = os.path.join(
self._package_directory, 'recordings', 'scpv1', 'Splunk-6.3', 'countmatches.execute.dispatch_dir',
'externSearchResultsInfo.csv')
ifile = StringIO('infoPath:' + info_path + '\n\naction\r\nget_search_results_info\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = StringIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('Expected no exception, but caught {}: {}'.format(type(error).__name__, error))
else:
self.assertRegexpMatches(
result.getvalue(),
r'^\r\n'
r'('
r'data,__mv_data,_serial,__mv__serial\r\n'
r'"\{.*u\'is_summary_index\': 0, .+\}",,0,'
r'|'
r'_serial,__mv__serial,data,__mv_data\r\n'
r'0,,"\{.*u\'is_summary_index\': 0, .+\}",'
r')'
r'\r\n$'
)
# TestStreamingCommand.process should provide access to a service object when search results info is available
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command.search_results_info.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command.search_results_info.auth_token)
self.assertEqual(command.service.namespace.app,
command.search_results_info.ppc_app)
self.assertEqual(command.service.namespace.owner,
None)
self.assertEqual(command.service.namespace.sharing,
None)
# Command.process should not provide access to search results info or a service object when the 'infoPath'
# input header is unavailable
ifile = StringIO('\naction\r\nget_search_results_info')
argv = ['teststreaming.py', '__EXECUTE__']
command = TestStreamingCommand()
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
self.assertIsNone(command.search_results_info)
self.assertIsNone(command.service)
return
def test_process_scpv1(self):
# TestCommand.process should complain if supports_getinfo == False
# We support dynamic configuration, not static
# The exception line number may change, so we're using a regex match instead of a string match
expected = re.compile(
r'error_message=RuntimeError at ".+search_command\.py", line \d\d\d : Command test appears to be '
r'statically configured for search command protocol version 1 and static configuration is unsupported by '
r'splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza:\n'
r'\[test\]\n'
r'filename = test.py\n'
r'enableheader = true\n'
r'outputheader = true\n'
r'requires_srinfo = true\n'
r'supports_getinfo = true\n'
r'supports_multivalues = true\n'
r'supports_rawargs = true')
argv = [
'test.py', 'not__GETINFO__or__EXECUTE__', 'option=value',
'fieldname'
]
command = TestCommand()
result = BytesIO()
self.assertRaises(SystemExit, command.process, argv, ofile=result)
self.assertRegexpMatches(result.getvalue().decode('UTF-8'), expected)
# TestCommand.process should return configuration settings on Getinfo probe
argv = [
'test.py', '__GETINFO__', 'required_option_1=value',
'required_option_2=value'
]
command = TestCommand()
ifile = StringIO('\n')
result = BytesIO()
self.assertEqual(str(command.configuration), '')
if six.PY2:
expected = (
"[(u'clear_required_fields', None, [1]), (u'distributed', None, [2]), (u'generates_timeorder', None, [1]), "
"(u'generating', None, [1, 2]), (u'maxinputs', None, [2]), (u'overrides_timeorder', None, [1]), "
"(u'required_fields', None, [1, 2]), (u'requires_preop', None, [1]), (u'retainsevents', None, [1]), "
"(u'run_in_preview', None, [2]), (u'streaming', None, [1]), (u'streaming_preop', None, [1, 2]), "
"(u'type', None, [2])]")
else:
expected = (
"[('clear_required_fields', None, [1]), ('distributed', None, [2]), ('generates_timeorder', None, [1]), "
"('generating', None, [1, 2]), ('maxinputs', None, [2]), ('overrides_timeorder', None, [1]), "
"('required_fields', None, [1, 2]), ('requires_preop', None, [1]), ('retainsevents', None, [1]), "
"('run_in_preview', None, [2]), ('streaming', None, [1]), ('streaming_preop', None, [1, 2]), "
"('type', None, [2])]")
self.assertEqual(repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error,
result.getvalue().decode('UTF-8')))
self.assertEqual(
'\r\n\r\n\r\n',
result.getvalue().decode(
'UTF-8')) # No message header and no configuration settings
ifile = StringIO('\n')
result = BytesIO()
# We might also put this sort of code into our SearchCommand.prepare override ...
configuration = command.configuration
# SCP v1/v2 configuration settings
configuration.generating = True
configuration.required_fields = ['foo', 'bar']
configuration.streaming_preop = 'some streaming command'
# SCP v1 configuration settings
configuration.clear_required_fields = True
configuration.generates_timeorder = True
configuration.overrides_timeorder = True
configuration.requires_preop = True
configuration.retainsevents = True
configuration.streaming = True
# SCP v2 configuration settings (SCP v1 requires that maxinputs and run_in_preview are set in commands.conf)
configuration.distributed = True
configuration.maxinputs = 50000
configuration.run_in_preview = True
configuration.type = 'streaming'
if six.PY2:
expected = (
'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[u\'foo\', u\'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
else:
expected = (
'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", '
'required_fields="[\'foo\', \'bar\']", requires_preop="True", retainsevents="True", streaming="True", '
'streaming_preop="some streaming command"')
self.assertEqual(str(command.configuration), expected)
if six.PY2:
expected = (
"[(u'clear_required_fields', True, [1]), (u'distributed', True, [2]), (u'generates_timeorder', True, [1]), "
"(u'generating', True, [1, 2]), (u'maxinputs', 50000, [2]), (u'overrides_timeorder', True, [1]), "
"(u'required_fields', [u'foo', u'bar'], [1, 2]), (u'requires_preop', True, [1]), "
"(u'retainsevents', True, [1]), (u'run_in_preview', True, [2]), (u'streaming', True, [1]), "
"(u'streaming_preop', u'some streaming command', [1, 2]), (u'type', u'streaming', [2])]"
)
else:
expected = (
"[('clear_required_fields', True, [1]), ('distributed', True, [2]), ('generates_timeorder', True, [1]), "
"('generating', True, [1, 2]), ('maxinputs', 50000, [2]), ('overrides_timeorder', True, [1]), "
"('required_fields', ['foo', 'bar'], [1, 2]), ('requires_preop', True, [1]), "
"('retainsevents', True, [1]), ('run_in_preview', True, [2]), ('streaming', True, [1]), "
"('streaming_preop', 'some streaming command', [1, 2]), ('type', 'streaming', [2])]"
)
self.assertEqual(repr(command.configuration), expected)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error,
result.getvalue().decode('UTF-8')))
result.seek(0)
reader = csv.reader(codecs.iterdecode(result, 'UTF-8'))
self.assertEqual([], next(reader))
observed = dict(izip(next(reader), next(reader)))
self.assertRaises(StopIteration, lambda: next(reader))
expected = {
'clear_required_fields': '1',
'__mv_clear_required_fields': '',
'generating': '1',
'__mv_generating': '',
'generates_timeorder': '1',
'__mv_generates_timeorder': '',
'overrides_timeorder': '1',
'__mv_overrides_timeorder': '',
'requires_preop': '1',
'__mv_requires_preop': '',
'required_fields': 'foo,bar',
'__mv_required_fields': '',
'retainsevents': '1',
'__mv_retainsevents': '',
'streaming': '1',
'__mv_streaming': '',
'streaming_preop': 'some streaming command',
'__mv_streaming_preop': '',
}
self.assertDictEqual(
expected,
observed) # No message header and no configuration settings
for action in '__GETINFO__', '__EXECUTE__':
# TestCommand.process should produce an error record on parser errors
argv = [
'test.py', action, 'required_option_1=value',
'required_option_2=value', 'undefined_option=value',
'fieldname_1', 'fieldname_2'
]
command = TestCommand()
ifile = StringIO('\n')
result = BytesIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=Unrecognized test command option: undefined_option="value"\r\n\r\n',
result.getvalue().decode('UTF-8'))
# TestCommand.process should produce an error record when required options are missing
argv = [
'test.py', action, 'required_option_2=value', 'fieldname_1'
]
command = TestCommand()
ifile = StringIO('\n')
result = BytesIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=A value for test command option required_option_1 is required\r\n\r\n',
result.getvalue().decode('UTF-8'))
argv = ['test.py', action, 'fieldname_1']
command = TestCommand()
ifile = StringIO('\n')
result = BytesIO()
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertTrue(
'error_message=Values for these test command options are required: required_option_1, required_option_2'
'\r\n\r\n',
result.getvalue().decode('UTF-8'))
# TestStreamingCommand.process should exit on processing exceptions
ifile = StringIO('\naction\r\nraise_error\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = BytesIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(error.code, 0)
self.assertRegexpMatches(
result.getvalue().decode('UTF-8'),
r'^error_message=RuntimeError at ".+", line \d+ : Testing\r\n\r\n$'
)
except BaseException as error:
self.fail('Expected SystemExit, but caught {}: {}'.format(
type(error).__name__, error))
else:
self.fail('Expected SystemExit, but no exception was raised')
# Command.process should provide access to search results info
info_path = os.path.join(self._package_directory, 'recordings',
'scpv1', 'Splunk-6.3',
'countmatches.execute.dispatch_dir',
'externSearchResultsInfo.csv')
ifile = StringIO('infoPath:' + info_path +
'\n\naction\r\nget_search_results_info\r\n')
argv = ['test.py', '__EXECUTE__']
command = TestStreamingCommand()
result = BytesIO()
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except BaseException as error:
self.fail('Expected no exception, but caught {}: {}'.format(
type(error).__name__, error))
else:
self.assertRegexpMatches(
result.getvalue().decode('UTF-8'), r'^\r\n'
r'('
r'data,__mv_data,_serial,__mv__serial\r\n'
r'\"\{.*u\'is_summary_index\': 0, .+\}\",,0,'
r'|'
r'_serial,__mv__serial,data,__mv_data\r\n'
r'0,,\"\{.*\'is_summary_index\': 0, .+\}\",'
r')'
r'\r\n$')
# TestStreamingCommand.process should provide access to a service object when search results info is available
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command.search_results_info.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command.search_results_info.auth_token)
self.assertEqual(command.service.namespace.app,
command.search_results_info.ppc_app)
self.assertEqual(command.service.namespace.owner, None)
self.assertEqual(command.service.namespace.sharing, None)
# Command.process should not provide access to search results info or a service object when the 'infoPath'
# input header is unavailable
ifile = StringIO('\naction\r\nget_search_results_info')
argv = ['teststreaming.py', '__EXECUTE__']
command = TestStreamingCommand()
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
self.assertIsNone(command.search_results_info)
self.assertIsNone(command.service)
return
def test_record_writer_with_recordings(self):
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join(
(cls.__name__, method.__name__)))
for input_file in iglob(base_path + '*.input.gz'):
with gzip.open(input_file, 'rb') as ifile:
test_data = pickle.load(ifile)
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
write_record = writer.write_record
fieldnames = test_data['fieldnames']
for values in test_data['values']:
record = OrderedDict(izip(fieldnames, values))
try:
write_record(record)
except Exception as error:
self.fail(error)
for message_type, message_text in test_data['messages']:
writer.write_message(message_type, '{}', message_text)
for name, metric in six.iteritems(test_data['metrics']):
writer.write_metric(name, metric)
writer.flush(finished=True)
# Read expected data
expected_path = os.path.splitext(
os.path.splitext(input_file)[0])[0] + '.output'
with io.open(expected_path, 'rb') as ifile:
expected = ifile.read()
expected = self._load_chunks(StringIO(expected))
# Read observed data
ifile = writer._ofile
ifile.seek(0)
observed = self._load_chunks(ifile)
# Write observed data (as an aid to diagnostics)
observed_path = expected_path + '.observed'
observed_value = ifile.getvalue()
with io.open(observed_path, 'wb') as ifile:
ifile.write(observed_value)
self._compare_chunks(observed, expected)
return
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(
StringIO(),
maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = [
'_serial', '_time', 'random_bytes', 'random_dict',
'random_integers', 'random_unicode'
]
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [
serial_number,
time(),
random_bytes(),
random_dict(),
random_integers(),
random_unicode()
]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [('debug', random_unicode()), ('error', random_unicode()),
('fatal', random_unicode()), ('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(
ifilter(lambda k_v: k_v[0].startswith('metric.'),
six.iteritems(writer._inspector))),
dict(
imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]),
six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError,
writer.flush,
finished=True,
partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(
self._recordings_path, '.'.join(
(cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def test_command_line_parser(self):
@Configuration()
class TestCommandLineParserCommand(SearchCommand):
required_option = Option(validate=Boolean(), require=True)
unnecessary_option = Option(validate=Boolean(),
default=True,
require=False)
class ConfigurationSettings(SearchCommand.ConfigurationSettings):
@classmethod
def fix_up(cls, command_class):
pass
# Command line without fieldnames
options = ['required_option=true', 'unnecessary_option=false']
command = TestCommandLineParserCommand()
CommandLineParser.parse(command, options)
for option in six.itervalues(command.options):
if option.name in [
'logging_configuration', 'logging_level', 'record',
'show_configuration'
]:
self.assertFalse(option.is_set)
continue
self.assertTrue(option.is_set)
expected = 'testcommandlineparser required_option="t" unnecessary_option="f"'
self.assertEqual(expected, str(command))
self.assertEqual(command.fieldnames, [])
# Command line with fieldnames
fieldnames = ['field_1', 'field_2', 'field_3']
command = TestCommandLineParserCommand()
CommandLineParser.parse(command, options + fieldnames)
for option in six.itervalues(command.options):
if option.name in [
'logging_configuration', 'logging_level', 'record',
'show_configuration'
]:
self.assertFalse(option.is_set)
continue
self.assertTrue(option.is_set)
expected = 'testcommandlineparser required_option="t" unnecessary_option="f" field_1 field_2 field_3'
self.assertEqual(expected, str(command))
self.assertEqual(command.fieldnames, fieldnames)
# Command line without any unnecessary options
command = TestCommandLineParserCommand()
CommandLineParser.parse(command, ['required_option=true'] + fieldnames)
for option in six.itervalues(command.options):
if option.name in [
'unnecessary_option', 'logging_configuration',
'logging_level', 'record', 'show_configuration'
]:
self.assertFalse(option.is_set)
continue
self.assertTrue(option.is_set)
expected = 'testcommandlineparser required_option="t" field_1 field_2 field_3'
self.assertEqual(expected, str(command))
self.assertEqual(command.fieldnames, fieldnames)
# Command line with missing required options, with or without fieldnames or unnecessary options
options = ['unnecessary_option=true']
self.assertRaises(ValueError, CommandLineParser.parse, command,
options + fieldnames)
self.assertRaises(ValueError, CommandLineParser.parse, command,
options)
self.assertRaises(ValueError, CommandLineParser.parse, command, [])
# Command line with unrecognized options
self.assertRaises(
ValueError, CommandLineParser.parse, command,
['unrecognized_option_1=foo', 'unrecognized_option_2=bar'])
# Command line with a variety of quoted/escaped text options
@Configuration()
class TestCommandLineParserCommand(SearchCommand):
text = Option()
class ConfigurationSettings(SearchCommand.ConfigurationSettings):
@classmethod
def fix_up(cls, command_class):
pass
strings = [
r'"foo bar"', r'"foo/bar"', r'"foo\\bar"', r'"""foo bar"""',
r'"\"foo bar\""', r'Hello\ World!', r'\"Hello\ World!\"'
]
expected_values = [
r'foo bar', r'foo/bar', r'foo\bar', r'"foo bar"', r'"foo bar"',
r'Hello World!', r'"Hello World!"'
]
for string, expected_value in izip(strings, expected_values):
command = TestCommandLineParserCommand()
argv = ['text', '=', string]
CommandLineParser.parse(command, argv)
self.assertEqual(command.text, expected_value)
for string, expected_value in izip(strings, expected_values):
command = TestCommandLineParserCommand()
argv = [string]
CommandLineParser.parse(command, argv)
self.assertEqual(command.fieldnames[0], expected_value)
for string, expected_value in izip(strings, expected_values):
command = TestCommandLineParserCommand()
argv = ['text', '=', string] + strings
CommandLineParser.parse(command, argv)
self.assertEqual(command.text, expected_value)
self.assertEqual(command.fieldnames, expected_values)
strings = [
'some\\ string\\', r'some\ string"', r'"some string',
r'some"string'
]
for string in strings:
command = TestCommandLineParserCommand()
argv = [string]
self.assertRaises(SyntaxError, CommandLineParser.parse, command,
argv)
return
def test_record_writer_with_random_data(self, save_recording=False):
# Confirmed: [minint, maxint) covers the full range of values that xrange allows
# RecordWriter writes apps in units of maxresultrows records. Default: 50,0000.
# Partial results are written when the record count reaches maxresultrows.
writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test
test_data = OrderedDict()
fieldnames = ['_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode']
test_data['fieldnames'] = fieldnames
test_data['values'] = []
write_record = writer.write_record
for serial_number in range(0, 31):
values = [serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode()]
record = OrderedDict(izip(fieldnames, values))
#try:
write_record(record)
#except Exception as error:
# self.fail(error)
test_data['values'].append(values)
# RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result
# is produced or we're finished
messages = [
('debug', random_unicode()),
('error', random_unicode()),
('fatal', random_unicode()),
('info', random_unicode()),
('warn', random_unicode())]
test_data['messages'] = messages
for message_type, message_text in messages:
writer.write_message(message_type, '{}', message_text)
metrics = {
'metric-1': SearchMetric(1, 2, 3, 4),
'metric-2': SearchMetric(5, 6, 7, 8)
}
test_data['metrics'] = metrics
for name, metric in six.iteritems(metrics):
writer.write_metric(name, metric)
self.assertEqual(writer._chunk_count, 3)
self.assertEqual(writer._record_count, 1)
self.assertGreater(writer._buffer.tell(), 0)
self.assertEqual(writer._total_record_count, 30)
self.assertListEqual(writer._fieldnames, fieldnames)
self.assertListEqual(writer._inspector['messages'], messages)
self.assertDictEqual(
dict(ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))),
dict(imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics))))
writer.flush(finished=True)
self.assertEqual(writer._chunk_count, 4)
self.assertEqual(writer._record_count, 0)
self.assertEqual(writer._buffer.tell(), 0)
self.assertEqual(writer._buffer.getvalue(), '')
self.assertEqual(writer._total_record_count, 31)
self.assertRaises(AssertionError, writer.flush, finished=True, partial=True)
self.assertRaises(AssertionError, writer.flush, finished='non-boolean')
self.assertRaises(AssertionError, writer.flush, partial='non-boolean')
self.assertRaises(AssertionError, writer.flush)
self.assertRaises(RuntimeError, writer.write_record, {})
self.assertFalse(writer._ofile.closed)
self.assertIsNone(writer._fieldnames)
self.assertDictEqual(writer._inspector, OrderedDict())
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling
# RecordWriter.flush(partial=True).
# P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling
# RecordWriter.flush(finish=True).
if save_recording:
cls = self.__class__
method = cls.test_record_writer_with_recordings
base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__, six.text_type(time()))))
with gzip.open(base_path + '.input.gz', 'wb') as f:
pickle.dump(test_data, f)
with open(base_path + '.output', 'wb') as f:
f.write(writer._ofile.getvalue())
return
def search_results_info(self):
""" Returns the search results info for this command invocation.
The search results info object is created from the search results info file associated with the command
invocation.
:return: Search results info:const:`None`, if the search results info file associated with the command
invocation is inaccessible.
:rtype: SearchResultsInfo or NoneType
"""
if self._search_results_info is not None:
return self._search_results_info
if self._protocol_version == 1:
try:
path = self._input_header['infoPath']
except KeyError:
return None
else:
assert self._protocol_version == 2
try:
dispatch_dir = self._metadata.searchinfo.dispatch_dir
except AttributeError:
return None
path = os.path.join(dispatch_dir, 'info.csv')
try:
with io.open(path, 'r') as f:
reader = csv.reader(f, dialect=CsvDialect)
fields = next(reader)
values = next(reader)
except IOError as error:
if error.errno == 2:
self.logger.error('Search results info file {} does not exist.'.format(json_encode_string(path)))
return
raise
def convert_field(field):
return (field[1:] if field[0] == '_' else field).replace('.', '_')
decode = MetadataDecoder().decode
def convert_value(value):
try:
return decode(value) if len(value) > 0 else value
except ValueError:
return value
info = ObjectView(dict(imap(lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values))))
try:
count_map = info.countMap
except AttributeError:
pass
else:
count_map = count_map.split(';')
n = len(count_map)
info.countMap = dict(izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2)))
try:
msg_type = info.msgType
msg_text = info.msg
except AttributeError:
pass
else:
messages = ifilter(lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n')))
info.msg = [Message(message) for message in messages]
del info.msgType
try:
info.vix_families = ElementTree.fromstring(info.vix_families)
except AttributeError:
pass
self._search_results_info = info
return info
