def _one_chunk_of_records(self, ifile): self._finished = True metadata_length, body_length = self._start_chunk(ifile) if metadata_length is None: self.logger.info("No chunk; exiting...") return self.logger.info('Start data chunk...({},{})'.format(metadata_length, body_length)) metadata = self._read_metadata(ifile, metadata_length) action = getattr(metadata, 'action', None) if action != 'execute': raise RuntimeError('Expected execute action, not {}'.format(action)) finished = getattr(metadata, 'finished', False) self._record_writer.is_flushed = False if body_length is 0: return copy_input = True if copy_input: self._capture_input(ifile, body_length) reader = csv.reader(self._icopy, dialect=CsvDialect) else: reader = csv.reader(ChunkedInput(ifile, body_length), dialect=CsvDialect) try: fieldnames = next(reader) except StopIteration: raise RuntimeError('CSV header malformed') self.logger.debug('Read records...') mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')]) if len(mv_fieldnames) == 0: for values in reader: yield OrderedDict(izip(fieldnames, values)) else: for values in reader: record = OrderedDict() for fieldname, value in izip(fieldnames, values): if fieldname.startswith('__mv_'): if len(value) > 0: record[mv_fieldnames[fieldname]] = self._decode_list(value) elif fieldname not in record: record[fieldname] = value yield record if not self._icopy is None: self._icopy.close() os.remove(self._icopy_path) if finished: return self._finished = False
def _records_protocol_v1(self, ifile): reader = csv.reader(ifile, dialect=CsvDialect) try: fieldnames = next(reader) except StopIteration: return mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')]) if len(mv_fieldnames) == 0: for values in reader: yield OrderedDict(izip(fieldnames, values)) return for values in reader: record = OrderedDict() for fieldname, value in izip(fieldnames, values): if fieldname.startswith('__mv_'): if len(value) > 0: record[mv_fieldnames[fieldname]] = self._decode_list(value) elif fieldname not in record: record[fieldname] = value yield record
def _read_csv_records(self, ifile): reader = csv.reader(ifile, dialect=CsvDialect) try: fieldnames = next(reader) except StopIteration: return mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')]) if len(mv_fieldnames) == 0: for values in reader: yield OrderedDict(izip(fieldnames, values)) return for values in reader: record = OrderedDict() for fieldname, value in izip(fieldnames, values): if fieldname.startswith('__mv_'): if len(value) > 0: record[mv_fieldnames[fieldname]] = self._decode_list( value) elif fieldname not in record: record[fieldname] = value yield record
def _records_protocol_v2(self, ifile): istream = self._as_binary_stream(ifile) while True: result = self._read_chunk(istream) if not result: return metadata, body = result action = getattr(metadata, 'action', None) if action != 'execute': raise RuntimeError( 'Expected execute action, not {}'.format(action)) finished = getattr(metadata, 'finished', False) self._record_writer.is_flushed = False if len(body) > 0: reader = csv.reader(StringIO(body), dialect=CsvDialect) try: fieldnames = next(reader) except StopIteration: return mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')]) if len(mv_fieldnames) == 0: for values in reader: yield OrderedDict(izip(fieldnames, values)) else: for values in reader: record = OrderedDict() for fieldname, value in izip(fieldnames, values): if fieldname.startswith('__mv_'): if len(value) > 0: record[mv_fieldnames[ fieldname]] = self._decode_list(value) elif fieldname not in record: record[fieldname] = value yield record if finished: return self.flush()
def __init__(self, path): self._dispatch_dir = path + '.dispatch_dir' self._search = None if os.path.exists(self._dispatch_dir): with io.open(os.path.join(self._dispatch_dir, 'request.csv')) as ifile: reader = csv.reader(ifile) for name, value in izip(next(reader), next(reader)): if name == 'search': self._search = value break assert self._search is not None splunk_cmd = path + '.splunk_cmd' try: with io.open(splunk_cmd, 'r') as f: self._args = f.readline().encode().split( None, 5) # ['splunk', 'cmd', <filename>, <action>, <args>] except IOError as error: if error.errno != 2: raise self._args = ['splunk', 'cmd', 'python', None] self._input_file = path + '.input.gz' self._output_file = path + '.output' if six.PY3 and os.path.isfile(self._output_file + '.py3'): self._output_file = self._output_file + '.py3' # Remove the "splunk cmd" portion self._args = self._args[2:]
def assertInfoEqual(self, output, expected): reader = csv.reader(StringIO(output)) self.assertEqual([], next(reader)) fields = next(reader) values = next(reader) self.assertRaises(StopIteration, reader.next) output = dict(izip(fields, values)) reader = csv.reader(StringIO(expected)) self.assertEqual([], next(reader)) fields = next(reader) values = next(reader) self.assertRaises(StopIteration, reader.next) expected = dict(izip(fields, values)) self.assertDictEqual(expected, output)
def _records_protocol_v2(self, ifile): while True: result = self._read_chunk(ifile) if not result: return metadata, body = result action = getattr(metadata, 'action', None) if action != 'execute': raise RuntimeError('Expected execute action, not {}'.format(action)) finished = getattr(metadata, 'finished', False) self._record_writer.is_flushed = False if len(body) > 0: reader = csv.reader(StringIO(body), dialect=CsvDialect) try: fieldnames = next(reader) except StopIteration: return mv_fieldnames = dict([(name, name[len('__mv_'):]) for name in fieldnames if name.startswith('__mv_')]) if len(mv_fieldnames) == 0: for values in reader: yield OrderedDict(izip(fieldnames, values)) else: for values in reader: record = OrderedDict() for fieldname, value in izip(fieldnames, values): if fieldname.startswith('__mv_'): if len(value) > 0: record[mv_fieldnames[fieldname]] = self._decode_list(value) elif fieldname not in record: record[fieldname] = value yield record if finished: return self.flush()
def test_record_writer_with_recordings(self): cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__))) for input_file in iglob(base_path + '*.input.gz'): with gzip.open(input_file, 'rb') as ifile: test_data = pickle.load(ifile) writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test write_record = writer.write_record fieldnames = test_data['fieldnames'] for values in test_data['values']: record = OrderedDict(izip(fieldnames, values)) try: write_record(record) except Exception as error: self.fail(error) for message_type, message_text in test_data['messages']: writer.write_message(message_type, '{}', message_text) for name, metric in six.iteritems(test_data['metrics']): writer.write_metric(name, metric) writer.flush(finished=True) # Read expected data expected_path = os.path.splitext(os.path.splitext(input_file)[0])[0] + '.output' with io.open(expected_path, 'rb') as ifile: expected = ifile.read() expected = self._load_chunks(StringIO(expected)) # Read observed data ifile = writer._ofile ifile.seek(0) observed = self._load_chunks(ifile) # Write observed data (as an aid to diagnostics) observed_path = expected_path + '.observed' observed_value = ifile.getvalue() with io.open(observed_path, 'wb') as ifile: ifile.write(observed_value) self._compare_chunks(observed, expected) return
def _compare_chunks(self, chunks_1, chunks_2): self.assertEqual(len(chunks_1), len(chunks_2)) n = 0 for chunk_1, chunk_2 in izip(chunks_1, chunks_2): self.assertDictEqual( chunk_1.metadata, chunk_2.metadata, 'Chunk {0}: metadata error: "{1}" != "{2}"'.format(n, chunk_1.metadata, chunk_2.metadata)) self.assertMultiLineEqual(chunk_1.body, chunk_2.body, 'Chunk {0}: data error'.format(n)) n += 1 return
def _run(self): writer = RecordWriterV2(self.recorder.output, maxresultrows=10) write_record = writer.write_record names = self.fieldnames for self._serial_number in range(0, 31): record = OrderedDict(izip(names, self.row)) write_record(record) return
def _compare_chunks(self, chunks_1, chunks_2): self.assertEqual(len(chunks_1), len(chunks_2)) n = 0 for chunk_1, chunk_2 in izip(chunks_1, chunks_2): self.assertDictEqual( chunk_1.metadata, chunk_2.metadata, 'Chunk {0}: metadata error: "{1}" != "{2}"'.format( n, chunk_1.metadata, chunk_2.metadata)) self.assertMultiLineEqual(chunk_1.body, chunk_2.body, 'Chunk {0}: data error'.format(n)) n += 1 return
def _compare_chunks(self, expected, output, time_sensitive=True): expected = expected.strip() output = output.strip() if time_sensitive: compare_csv_files = self._compare_csv_files_time_sensitive else: compare_csv_files = self._compare_csv_files_time_insensitive chunks_1 = self._load_chunks(StringIO(expected)) chunks_2 = self._load_chunks(StringIO(output)) self.assertEqual(len(chunks_1), len(chunks_2)) n = 0 for chunk_1, chunk_2 in izip(chunks_1, chunks_2): self.assertDictEqual( chunk_1.metadata, chunk_2.metadata, 'Chunk {0}: metadata error: "{1}" != "{2}"'.format( n, chunk_1.metadata, chunk_2.metadata)) compare_csv_files(chunk_1.body, chunk_2.body) n += 1 return
def search_results_info(self): """ Returns the search results info for this command invocation. The search results info object is created from the search results info file associated with the command invocation. :return: Search results info:const:`None`, if the search results info file associated with the command invocation is inaccessible. :rtype: SearchResultsInfo or NoneType """ if self._search_results_info is not None: return self._search_results_info if self._protocol_version == 1: try: path = self._input_header['infoPath'] except KeyError: return None else: assert self._protocol_version == 2 try: dispatch_dir = self._metadata.searchinfo.dispatch_dir except AttributeError: return None path = os.path.join(dispatch_dir, 'info.csv') try: with io.open(path, 'r') as f: reader = csv.reader(f, dialect=CsvDialect) fields = next(reader) values = next(reader) except IOError as error: if error.errno == 2: self.logger.error( 'Search results info file {} does not exist.'.format( json_encode_string(path))) return raise def convert_field(field): return (field[1:] if field[0] == '_' else field).replace('.', '_') decode = MetadataDecoder().decode def convert_value(value): try: return decode(value) if len(value) > 0 else value except ValueError: return value info = ObjectView( dict( imap( lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values)))) try: count_map = info.countMap except AttributeError: pass else: count_map = count_map.split(';') n = len(count_map) info.countMap = dict( izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2))) try: msg_type = info.msgType msg_text = info.msg except AttributeError: pass else: messages = ifilter( lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n'))) info.msg = [Message(message) for message in messages] del info.msgType try: info.vix_families = ElementTree.fromstring(info.vix_families) except AttributeError: pass self._search_results_info = info return info
def test_process_scpv1(self): # TestCommand.process should complain if supports_getinfo == False # We support dynamic configuration, not static # The exception line number may change, so we're using a regex match instead of a string match expected = re.compile( r'error_message=RuntimeError at ".+search_command\.py", line \d\d\d : Command test appears to be ' r'statically configured for search command protocol version 1 and static configuration is unsupported by ' r'splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza:\n' r'\[test\]\n' r'filename = test.py\n' r'enableheader = true\n' r'outputheader = true\n' r'requires_srinfo = true\n' r'supports_getinfo = true\n' r'supports_multivalues = true\n' r'supports_rawargs = true') argv = ['test.py', 'not__GETINFO__or__EXECUTE__', 'option=value', 'fieldname'] command = TestCommand() result = StringIO() self.assertRaises(SystemExit, command.process, argv, ofile=result) self.assertRegexpMatches(result.getvalue(), expected) # TestCommand.process should return configuration settings on Getinfo probe argv = ['test.py', '__GETINFO__', 'required_option_1=value', 'required_option_2=value'] command = TestCommand() ifile = StringIO('\n') result = StringIO() self.assertEqual(str(command.configuration), '') if six.PY2: expected = ("[(u'clear_required_fields', None, [1]), (u'distributed', None, [2]), (u'generates_timeorder', None, [1]), " "(u'generating', None, [1, 2]), (u'maxinputs', None, [2]), (u'overrides_timeorder', None, [1]), " "(u'required_fields', None, [1, 2]), (u'requires_preop', None, [1]), (u'retainsevents', None, [1]), " "(u'run_in_preview', None, [2]), (u'streaming', None, [1]), (u'streaming_preop', None, [1, 2]), " "(u'type', None, [2])]") else: expected = ("[('clear_required_fields', None, [1]), ('distributed', None, [2]), ('generates_timeorder', None, [1]), " "('generating', None, [1, 2]), ('maxinputs', None, [2]), ('overrides_timeorder', None, [1]), " "('required_fields', None, [1, 2]), ('requires_preop', None, [1]), ('retainsevents', None, [1]), " "('run_in_preview', None, [2]), ('streaming', None, [1]), ('streaming_preop', None, [1, 2]), " "('type', None, [2])]") self.assertEqual( repr(command.configuration), expected) try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except BaseException as error: self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue())) self.assertEqual('\r\n\r\n\r\n', result.getvalue()) # No message header and no configuration settings ifile = StringIO('\n') result = StringIO() # We might also put this sort of code into our SearchCommand.prepare override ... configuration = command.configuration # SCP v1/v2 configuration settings configuration.generating = True configuration.required_fields = ['foo', 'bar'] configuration.streaming_preop = 'some streaming command' # SCP v1 configuration settings configuration.clear_required_fields = True configuration.generates_timeorder = True configuration.overrides_timeorder = True configuration.requires_preop = True configuration.retainsevents = True configuration.streaming = True # SCP v2 configuration settings (SCP v1 requires that maxinputs and run_in_preview are set in commands.conf) configuration.distributed = True configuration.maxinputs = 50000 configuration.run_in_preview = True configuration.type = 'streaming' if six.PY2: expected = ('clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", ' 'required_fields="[u\'foo\', u\'bar\']", requires_preop="True", retainsevents="True", streaming="True", ' 'streaming_preop="some streaming command"') else: expected = ('clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", ' 'required_fields="[\'foo\', \'bar\']", requires_preop="True", retainsevents="True", streaming="True", ' 'streaming_preop="some streaming command"') self.assertEqual(str(command.configuration), expected) if six.PY2: expected = ("[(u'clear_required_fields', True, [1]), (u'distributed', True, [2]), (u'generates_timeorder', True, [1]), " "(u'generating', True, [1, 2]), (u'maxinputs', 50000, [2]), (u'overrides_timeorder', True, [1]), " "(u'required_fields', [u'foo', u'bar'], [1, 2]), (u'requires_preop', True, [1]), " "(u'retainsevents', True, [1]), (u'run_in_preview', True, [2]), (u'streaming', True, [1]), " "(u'streaming_preop', u'some streaming command', [1, 2]), (u'type', u'streaming', [2])]") else: expected = ("[('clear_required_fields', True, [1]), ('distributed', True, [2]), ('generates_timeorder', True, [1]), " "('generating', True, [1, 2]), ('maxinputs', 50000, [2]), ('overrides_timeorder', True, [1]), " "('required_fields', ['foo', 'bar'], [1, 2]), ('requires_preop', True, [1]), " "('retainsevents', True, [1]), ('run_in_preview', True, [2]), ('streaming', True, [1]), " "('streaming_preop', 'some streaming command', [1, 2]), ('type', 'streaming', [2])]") self.assertEqual( repr(command.configuration), expected) try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except BaseException as error: self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue())) result.seek(0) reader = csv.reader(result) self.assertEqual([], next(reader)) observed = dict(izip(next(reader), next(reader))) self.assertRaises(StopIteration, lambda: next(reader)) expected = { 'clear_required_fields': '1', '__mv_clear_required_fields': '', 'generating': '1', '__mv_generating': '', 'generates_timeorder': '1', '__mv_generates_timeorder': '', 'overrides_timeorder': '1', '__mv_overrides_timeorder': '', 'requires_preop': '1', '__mv_requires_preop': '', 'required_fields': 'foo,bar', '__mv_required_fields': '', 'retainsevents': '1', '__mv_retainsevents': '', 'streaming': '1', '__mv_streaming': '', 'streaming_preop': 'some streaming command', '__mv_streaming_preop': '', } self.assertDictEqual(expected, observed) # No message header and no configuration settings for action in '__GETINFO__', '__EXECUTE__': # TestCommand.process should produce an error record on parser errors argv = [ 'test.py', action, 'required_option_1=value', 'required_option_2=value', 'undefined_option=value', 'fieldname_1', 'fieldname_2'] command = TestCommand() ifile = StringIO('\n') result = StringIO() self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result) self.assertTrue( 'error_message=Unrecognized test command option: undefined_option="value"\r\n\r\n', result.getvalue()) # TestCommand.process should produce an error record when required options are missing argv = ['test.py', action, 'required_option_2=value', 'fieldname_1'] command = TestCommand() ifile = StringIO('\n') result = StringIO() self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result) self.assertTrue( 'error_message=A value for test command option required_option_1 is required\r\n\r\n', result.getvalue()) argv = ['test.py', action, 'fieldname_1'] command = TestCommand() ifile = StringIO('\n') result = StringIO() self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result) self.assertTrue( 'error_message=Values for these test command options are required: required_option_1, required_option_2' '\r\n\r\n', result.getvalue()) # TestStreamingCommand.process should exit on processing exceptions ifile = StringIO('\naction\r\nraise_error\r\n') argv = ['test.py', '__EXECUTE__'] command = TestStreamingCommand() result = StringIO() try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except SystemExit as error: self.assertNotEqual(error.code, 0) self.assertRegexpMatches( result.getvalue(), r'^error_message=RuntimeError at ".+", line \d+ : Testing\r\n\r\n$') except BaseException as error: self.fail('Expected SystemExit, but caught {}: {}'.format(type(error).__name__, error)) else: self.fail('Expected SystemExit, but no exception was raised') # Command.process should provide access to search results info info_path = os.path.join( self._package_directory, 'recordings', 'scpv1', 'Splunk-6.3', 'countmatches.execute.dispatch_dir', 'externSearchResultsInfo.csv') ifile = StringIO('infoPath:' + info_path + '\n\naction\r\nget_search_results_info\r\n') argv = ['test.py', '__EXECUTE__'] command = TestStreamingCommand() result = StringIO() try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except BaseException as error: self.fail('Expected no exception, but caught {}: {}'.format(type(error).__name__, error)) else: self.assertRegexpMatches( result.getvalue(), r'^\r\n' r'(' r'data,__mv_data,_serial,__mv__serial\r\n' r'"\{.*u\'is_summary_index\': 0, .+\}",,0,' r'|' r'_serial,__mv__serial,data,__mv_data\r\n' r'0,,"\{.*u\'is_summary_index\': 0, .+\}",' r')' r'\r\n$' ) # TestStreamingCommand.process should provide access to a service object when search results info is available self.assertIsInstance(command.service, Service) self.assertEqual(command.service.authority, command.search_results_info.splunkd_uri) self.assertEqual(command.service.scheme, command.search_results_info.splunkd_protocol) self.assertEqual(command.service.port, command.search_results_info.splunkd_port) self.assertEqual(command.service.token, command.search_results_info.auth_token) self.assertEqual(command.service.namespace.app, command.search_results_info.ppc_app) self.assertEqual(command.service.namespace.owner, None) self.assertEqual(command.service.namespace.sharing, None) # Command.process should not provide access to search results info or a service object when the 'infoPath' # input header is unavailable ifile = StringIO('\naction\r\nget_search_results_info') argv = ['teststreaming.py', '__EXECUTE__'] command = TestStreamingCommand() # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) self.assertIsNone(command.search_results_info) self.assertIsNone(command.service) return
def test_process_scpv1(self): # TestCommand.process should complain if supports_getinfo == False # We support dynamic configuration, not static # The exception line number may change, so we're using a regex match instead of a string match expected = re.compile( r'error_message=RuntimeError at ".+search_command\.py", line \d\d\d : Command test appears to be ' r'statically configured for search command protocol version 1 and static configuration is unsupported by ' r'splunklib.searchcommands. Please ensure that default/commands.conf contains this stanza:\n' r'\[test\]\n' r'filename = test.py\n' r'enableheader = true\n' r'outputheader = true\n' r'requires_srinfo = true\n' r'supports_getinfo = true\n' r'supports_multivalues = true\n' r'supports_rawargs = true') argv = [ 'test.py', 'not__GETINFO__or__EXECUTE__', 'option=value', 'fieldname' ] command = TestCommand() result = BytesIO() self.assertRaises(SystemExit, command.process, argv, ofile=result) self.assertRegexpMatches(result.getvalue().decode('UTF-8'), expected) # TestCommand.process should return configuration settings on Getinfo probe argv = [ 'test.py', '__GETINFO__', 'required_option_1=value', 'required_option_2=value' ] command = TestCommand() ifile = StringIO('\n') result = BytesIO() self.assertEqual(str(command.configuration), '') if six.PY2: expected = ( "[(u'clear_required_fields', None, [1]), (u'distributed', None, [2]), (u'generates_timeorder', None, [1]), " "(u'generating', None, [1, 2]), (u'maxinputs', None, [2]), (u'overrides_timeorder', None, [1]), " "(u'required_fields', None, [1, 2]), (u'requires_preop', None, [1]), (u'retainsevents', None, [1]), " "(u'run_in_preview', None, [2]), (u'streaming', None, [1]), (u'streaming_preop', None, [1, 2]), " "(u'type', None, [2])]") else: expected = ( "[('clear_required_fields', None, [1]), ('distributed', None, [2]), ('generates_timeorder', None, [1]), " "('generating', None, [1, 2]), ('maxinputs', None, [2]), ('overrides_timeorder', None, [1]), " "('required_fields', None, [1, 2]), ('requires_preop', None, [1]), ('retainsevents', None, [1]), " "('run_in_preview', None, [2]), ('streaming', None, [1]), ('streaming_preop', None, [1, 2]), " "('type', None, [2])]") self.assertEqual(repr(command.configuration), expected) try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except BaseException as error: self.fail('{0}: {1}: {2}\n'.format( type(error).__name__, error, result.getvalue().decode('UTF-8'))) self.assertEqual( '\r\n\r\n\r\n', result.getvalue().decode( 'UTF-8')) # No message header and no configuration settings ifile = StringIO('\n') result = BytesIO() # We might also put this sort of code into our SearchCommand.prepare override ... configuration = command.configuration # SCP v1/v2 configuration settings configuration.generating = True configuration.required_fields = ['foo', 'bar'] configuration.streaming_preop = 'some streaming command' # SCP v1 configuration settings configuration.clear_required_fields = True configuration.generates_timeorder = True configuration.overrides_timeorder = True configuration.requires_preop = True configuration.retainsevents = True configuration.streaming = True # SCP v2 configuration settings (SCP v1 requires that maxinputs and run_in_preview are set in commands.conf) configuration.distributed = True configuration.maxinputs = 50000 configuration.run_in_preview = True configuration.type = 'streaming' if six.PY2: expected = ( 'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", ' 'required_fields="[u\'foo\', u\'bar\']", requires_preop="True", retainsevents="True", streaming="True", ' 'streaming_preop="some streaming command"') else: expected = ( 'clear_required_fields="True", generates_timeorder="True", generating="True", overrides_timeorder="True", ' 'required_fields="[\'foo\', \'bar\']", requires_preop="True", retainsevents="True", streaming="True", ' 'streaming_preop="some streaming command"') self.assertEqual(str(command.configuration), expected) if six.PY2: expected = ( "[(u'clear_required_fields', True, [1]), (u'distributed', True, [2]), (u'generates_timeorder', True, [1]), " "(u'generating', True, [1, 2]), (u'maxinputs', 50000, [2]), (u'overrides_timeorder', True, [1]), " "(u'required_fields', [u'foo', u'bar'], [1, 2]), (u'requires_preop', True, [1]), " "(u'retainsevents', True, [1]), (u'run_in_preview', True, [2]), (u'streaming', True, [1]), " "(u'streaming_preop', u'some streaming command', [1, 2]), (u'type', u'streaming', [2])]" ) else: expected = ( "[('clear_required_fields', True, [1]), ('distributed', True, [2]), ('generates_timeorder', True, [1]), " "('generating', True, [1, 2]), ('maxinputs', 50000, [2]), ('overrides_timeorder', True, [1]), " "('required_fields', ['foo', 'bar'], [1, 2]), ('requires_preop', True, [1]), " "('retainsevents', True, [1]), ('run_in_preview', True, [2]), ('streaming', True, [1]), " "('streaming_preop', 'some streaming command', [1, 2]), ('type', 'streaming', [2])]" ) self.assertEqual(repr(command.configuration), expected) try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except BaseException as error: self.fail('{0}: {1}: {2}\n'.format( type(error).__name__, error, result.getvalue().decode('UTF-8'))) result.seek(0) reader = csv.reader(codecs.iterdecode(result, 'UTF-8')) self.assertEqual([], next(reader)) observed = dict(izip(next(reader), next(reader))) self.assertRaises(StopIteration, lambda: next(reader)) expected = { 'clear_required_fields': '1', '__mv_clear_required_fields': '', 'generating': '1', '__mv_generating': '', 'generates_timeorder': '1', '__mv_generates_timeorder': '', 'overrides_timeorder': '1', '__mv_overrides_timeorder': '', 'requires_preop': '1', '__mv_requires_preop': '', 'required_fields': 'foo,bar', '__mv_required_fields': '', 'retainsevents': '1', '__mv_retainsevents': '', 'streaming': '1', '__mv_streaming': '', 'streaming_preop': 'some streaming command', '__mv_streaming_preop': '', } self.assertDictEqual( expected, observed) # No message header and no configuration settings for action in '__GETINFO__', '__EXECUTE__': # TestCommand.process should produce an error record on parser errors argv = [ 'test.py', action, 'required_option_1=value', 'required_option_2=value', 'undefined_option=value', 'fieldname_1', 'fieldname_2' ] command = TestCommand() ifile = StringIO('\n') result = BytesIO() self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result) self.assertTrue( 'error_message=Unrecognized test command option: undefined_option="value"\r\n\r\n', result.getvalue().decode('UTF-8')) # TestCommand.process should produce an error record when required options are missing argv = [ 'test.py', action, 'required_option_2=value', 'fieldname_1' ] command = TestCommand() ifile = StringIO('\n') result = BytesIO() self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result) self.assertTrue( 'error_message=A value for test command option required_option_1 is required\r\n\r\n', result.getvalue().decode('UTF-8')) argv = ['test.py', action, 'fieldname_1'] command = TestCommand() ifile = StringIO('\n') result = BytesIO() self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result) self.assertTrue( 'error_message=Values for these test command options are required: required_option_1, required_option_2' '\r\n\r\n', result.getvalue().decode('UTF-8')) # TestStreamingCommand.process should exit on processing exceptions ifile = StringIO('\naction\r\nraise_error\r\n') argv = ['test.py', '__EXECUTE__'] command = TestStreamingCommand() result = BytesIO() try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except SystemExit as error: self.assertNotEqual(error.code, 0) self.assertRegexpMatches( result.getvalue().decode('UTF-8'), r'^error_message=RuntimeError at ".+", line \d+ : Testing\r\n\r\n$' ) except BaseException as error: self.fail('Expected SystemExit, but caught {}: {}'.format( type(error).__name__, error)) else: self.fail('Expected SystemExit, but no exception was raised') # Command.process should provide access to search results info info_path = os.path.join(self._package_directory, 'recordings', 'scpv1', 'Splunk-6.3', 'countmatches.execute.dispatch_dir', 'externSearchResultsInfo.csv') ifile = StringIO('infoPath:' + info_path + '\n\naction\r\nget_search_results_info\r\n') argv = ['test.py', '__EXECUTE__'] command = TestStreamingCommand() result = BytesIO() try: # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) except BaseException as error: self.fail('Expected no exception, but caught {}: {}'.format( type(error).__name__, error)) else: self.assertRegexpMatches( result.getvalue().decode('UTF-8'), r'^\r\n' r'(' r'data,__mv_data,_serial,__mv__serial\r\n' r'\"\{.*u\'is_summary_index\': 0, .+\}\",,0,' r'|' r'_serial,__mv__serial,data,__mv_data\r\n' r'0,,\"\{.*\'is_summary_index\': 0, .+\}\",' r')' r'\r\n$') # TestStreamingCommand.process should provide access to a service object when search results info is available self.assertIsInstance(command.service, Service) self.assertEqual(command.service.authority, command.search_results_info.splunkd_uri) self.assertEqual(command.service.scheme, command.search_results_info.splunkd_protocol) self.assertEqual(command.service.port, command.search_results_info.splunkd_port) self.assertEqual(command.service.token, command.search_results_info.auth_token) self.assertEqual(command.service.namespace.app, command.search_results_info.ppc_app) self.assertEqual(command.service.namespace.owner, None) self.assertEqual(command.service.namespace.sharing, None) # Command.process should not provide access to search results info or a service object when the 'infoPath' # input header is unavailable ifile = StringIO('\naction\r\nget_search_results_info') argv = ['teststreaming.py', '__EXECUTE__'] command = TestStreamingCommand() # noinspection PyTypeChecker command.process(argv, ifile, ofile=result) self.assertIsNone(command.search_results_info) self.assertIsNone(command.service) return
def test_record_writer_with_recordings(self): cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join(self._recordings_path, '.'.join( (cls.__name__, method.__name__))) for input_file in iglob(base_path + '*.input.gz'): with gzip.open(input_file, 'rb') as ifile: test_data = pickle.load(ifile) writer = RecordWriterV2( StringIO(), maxresultrows=10) # small for the purposes of this unit test write_record = writer.write_record fieldnames = test_data['fieldnames'] for values in test_data['values']: record = OrderedDict(izip(fieldnames, values)) try: write_record(record) except Exception as error: self.fail(error) for message_type, message_text in test_data['messages']: writer.write_message(message_type, '{}', message_text) for name, metric in six.iteritems(test_data['metrics']): writer.write_metric(name, metric) writer.flush(finished=True) # Read expected data expected_path = os.path.splitext( os.path.splitext(input_file)[0])[0] + '.output' with io.open(expected_path, 'rb') as ifile: expected = ifile.read() expected = self._load_chunks(StringIO(expected)) # Read observed data ifile = writer._ofile ifile.seek(0) observed = self._load_chunks(ifile) # Write observed data (as an aid to diagnostics) observed_path = expected_path + '.observed' observed_value = ifile.getvalue() with io.open(observed_path, 'wb') as ifile: ifile.write(observed_value) self._compare_chunks(observed, expected) return
def test_record_writer_with_random_data(self, save_recording=False): # Confirmed: [minint, maxint) covers the full range of values that xrange allows # RecordWriter writes apps in units of maxresultrows records. Default: 50,0000. # Partial results are written when the record count reaches maxresultrows. writer = RecordWriterV2( StringIO(), maxresultrows=10) # small for the purposes of this unit test test_data = OrderedDict() fieldnames = [ '_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode' ] test_data['fieldnames'] = fieldnames test_data['values'] = [] write_record = writer.write_record for serial_number in range(0, 31): values = [ serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode() ] record = OrderedDict(izip(fieldnames, values)) #try: write_record(record) #except Exception as error: # self.fail(error) test_data['values'].append(values) # RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result # is produced or we're finished messages = [('debug', random_unicode()), ('error', random_unicode()), ('fatal', random_unicode()), ('info', random_unicode()), ('warn', random_unicode())] test_data['messages'] = messages for message_type, message_text in messages: writer.write_message(message_type, '{}', message_text) metrics = { 'metric-1': SearchMetric(1, 2, 3, 4), 'metric-2': SearchMetric(5, 6, 7, 8) } test_data['metrics'] = metrics for name, metric in six.iteritems(metrics): writer.write_metric(name, metric) self.assertEqual(writer._chunk_count, 3) self.assertEqual(writer._record_count, 1) self.assertGreater(writer._buffer.tell(), 0) self.assertEqual(writer._total_record_count, 30) self.assertListEqual(writer._fieldnames, fieldnames) self.assertListEqual(writer._inspector['messages'], messages) self.assertDictEqual( dict( ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))), dict( imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics)))) writer.flush(finished=True) self.assertEqual(writer._chunk_count, 4) self.assertEqual(writer._record_count, 0) self.assertEqual(writer._buffer.tell(), 0) self.assertEqual(writer._buffer.getvalue(), '') self.assertEqual(writer._total_record_count, 31) self.assertRaises(AssertionError, writer.flush, finished=True, partial=True) self.assertRaises(AssertionError, writer.flush, finished='non-boolean') self.assertRaises(AssertionError, writer.flush, partial='non-boolean') self.assertRaises(AssertionError, writer.flush) self.assertRaises(RuntimeError, writer.write_record, {}) self.assertFalse(writer._ofile.closed) self.assertIsNone(writer._fieldnames) self.assertDictEqual(writer._inspector, OrderedDict()) # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling # RecordWriter.flush(partial=True). # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling # RecordWriter.flush(finish=True). if save_recording: cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join( self._recordings_path, '.'.join( (cls.__name__, method.__name__, six.text_type(time())))) with gzip.open(base_path + '.input.gz', 'wb') as f: pickle.dump(test_data, f) with open(base_path + '.output', 'wb') as f: f.write(writer._ofile.getvalue()) return
def test_command_line_parser(self): @Configuration() class TestCommandLineParserCommand(SearchCommand): required_option = Option(validate=Boolean(), require=True) unnecessary_option = Option(validate=Boolean(), default=True, require=False) class ConfigurationSettings(SearchCommand.ConfigurationSettings): @classmethod def fix_up(cls, command_class): pass # Command line without fieldnames options = ['required_option=true', 'unnecessary_option=false'] command = TestCommandLineParserCommand() CommandLineParser.parse(command, options) for option in six.itervalues(command.options): if option.name in [ 'logging_configuration', 'logging_level', 'record', 'show_configuration' ]: self.assertFalse(option.is_set) continue self.assertTrue(option.is_set) expected = 'testcommandlineparser required_option="t" unnecessary_option="f"' self.assertEqual(expected, str(command)) self.assertEqual(command.fieldnames, []) # Command line with fieldnames fieldnames = ['field_1', 'field_2', 'field_3'] command = TestCommandLineParserCommand() CommandLineParser.parse(command, options + fieldnames) for option in six.itervalues(command.options): if option.name in [ 'logging_configuration', 'logging_level', 'record', 'show_configuration' ]: self.assertFalse(option.is_set) continue self.assertTrue(option.is_set) expected = 'testcommandlineparser required_option="t" unnecessary_option="f" field_1 field_2 field_3' self.assertEqual(expected, str(command)) self.assertEqual(command.fieldnames, fieldnames) # Command line without any unnecessary options command = TestCommandLineParserCommand() CommandLineParser.parse(command, ['required_option=true'] + fieldnames) for option in six.itervalues(command.options): if option.name in [ 'unnecessary_option', 'logging_configuration', 'logging_level', 'record', 'show_configuration' ]: self.assertFalse(option.is_set) continue self.assertTrue(option.is_set) expected = 'testcommandlineparser required_option="t" field_1 field_2 field_3' self.assertEqual(expected, str(command)) self.assertEqual(command.fieldnames, fieldnames) # Command line with missing required options, with or without fieldnames or unnecessary options options = ['unnecessary_option=true'] self.assertRaises(ValueError, CommandLineParser.parse, command, options + fieldnames) self.assertRaises(ValueError, CommandLineParser.parse, command, options) self.assertRaises(ValueError, CommandLineParser.parse, command, []) # Command line with unrecognized options self.assertRaises( ValueError, CommandLineParser.parse, command, ['unrecognized_option_1=foo', 'unrecognized_option_2=bar']) # Command line with a variety of quoted/escaped text options @Configuration() class TestCommandLineParserCommand(SearchCommand): text = Option() class ConfigurationSettings(SearchCommand.ConfigurationSettings): @classmethod def fix_up(cls, command_class): pass strings = [ r'"foo bar"', r'"foo/bar"', r'"foo\\bar"', r'"""foo bar"""', r'"\"foo bar\""', r'Hello\ World!', r'\"Hello\ World!\"' ] expected_values = [ r'foo bar', r'foo/bar', r'foo\bar', r'"foo bar"', r'"foo bar"', r'Hello World!', r'"Hello World!"' ] for string, expected_value in izip(strings, expected_values): command = TestCommandLineParserCommand() argv = ['text', '=', string] CommandLineParser.parse(command, argv) self.assertEqual(command.text, expected_value) for string, expected_value in izip(strings, expected_values): command = TestCommandLineParserCommand() argv = [string] CommandLineParser.parse(command, argv) self.assertEqual(command.fieldnames[0], expected_value) for string, expected_value in izip(strings, expected_values): command = TestCommandLineParserCommand() argv = ['text', '=', string] + strings CommandLineParser.parse(command, argv) self.assertEqual(command.text, expected_value) self.assertEqual(command.fieldnames, expected_values) strings = [ 'some\\ string\\', r'some\ string"', r'"some string', r'some"string' ] for string in strings: command = TestCommandLineParserCommand() argv = [string] self.assertRaises(SyntaxError, CommandLineParser.parse, command, argv) return
def test_record_writer_with_random_data(self, save_recording=False): # Confirmed: [minint, maxint) covers the full range of values that xrange allows # RecordWriter writes apps in units of maxresultrows records. Default: 50,0000. # Partial results are written when the record count reaches maxresultrows. writer = RecordWriterV2(StringIO(), maxresultrows=10) # small for the purposes of this unit test test_data = OrderedDict() fieldnames = ['_serial', '_time', 'random_bytes', 'random_dict', 'random_integers', 'random_unicode'] test_data['fieldnames'] = fieldnames test_data['values'] = [] write_record = writer.write_record for serial_number in range(0, 31): values = [serial_number, time(), random_bytes(), random_dict(), random_integers(), random_unicode()] record = OrderedDict(izip(fieldnames, values)) #try: write_record(record) #except Exception as error: # self.fail(error) test_data['values'].append(values) # RecordWriter accumulates inspector messages and metrics until maxresultrows are written, a partial result # is produced or we're finished messages = [ ('debug', random_unicode()), ('error', random_unicode()), ('fatal', random_unicode()), ('info', random_unicode()), ('warn', random_unicode())] test_data['messages'] = messages for message_type, message_text in messages: writer.write_message(message_type, '{}', message_text) metrics = { 'metric-1': SearchMetric(1, 2, 3, 4), 'metric-2': SearchMetric(5, 6, 7, 8) } test_data['metrics'] = metrics for name, metric in six.iteritems(metrics): writer.write_metric(name, metric) self.assertEqual(writer._chunk_count, 3) self.assertEqual(writer._record_count, 1) self.assertGreater(writer._buffer.tell(), 0) self.assertEqual(writer._total_record_count, 30) self.assertListEqual(writer._fieldnames, fieldnames) self.assertListEqual(writer._inspector['messages'], messages) self.assertDictEqual( dict(ifilter(lambda k_v: k_v[0].startswith('metric.'), six.iteritems(writer._inspector))), dict(imap(lambda k_v1: ('metric.' + k_v1[0], k_v1[1]), six.iteritems(metrics)))) writer.flush(finished=True) self.assertEqual(writer._chunk_count, 4) self.assertEqual(writer._record_count, 0) self.assertEqual(writer._buffer.tell(), 0) self.assertEqual(writer._buffer.getvalue(), '') self.assertEqual(writer._total_record_count, 31) self.assertRaises(AssertionError, writer.flush, finished=True, partial=True) self.assertRaises(AssertionError, writer.flush, finished='non-boolean') self.assertRaises(AssertionError, writer.flush, partial='non-boolean') self.assertRaises(AssertionError, writer.flush) self.assertRaises(RuntimeError, writer.write_record, {}) self.assertFalse(writer._ofile.closed) self.assertIsNone(writer._fieldnames) self.assertDictEqual(writer._inspector, OrderedDict()) # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to write partial results by calling # RecordWriter.flush(partial=True). # P2 [ ] TODO: Verify that RecordWriter gives consumers the ability to finish early by calling # RecordWriter.flush(finish=True). if save_recording: cls = self.__class__ method = cls.test_record_writer_with_recordings base_path = os.path.join(self._recordings_path, '.'.join((cls.__name__, method.__name__, six.text_type(time())))) with gzip.open(base_path + '.input.gz', 'wb') as f: pickle.dump(test_data, f) with open(base_path + '.output', 'wb') as f: f.write(writer._ofile.getvalue()) return
def search_results_info(self): """ Returns the search results info for this command invocation. The search results info object is created from the search results info file associated with the command invocation. :return: Search results info:const:`None`, if the search results info file associated with the command invocation is inaccessible. :rtype: SearchResultsInfo or NoneType """ if self._search_results_info is not None: return self._search_results_info if self._protocol_version == 1: try: path = self._input_header['infoPath'] except KeyError: return None else: assert self._protocol_version == 2 try: dispatch_dir = self._metadata.searchinfo.dispatch_dir except AttributeError: return None path = os.path.join(dispatch_dir, 'info.csv') try: with io.open(path, 'r') as f: reader = csv.reader(f, dialect=CsvDialect) fields = next(reader) values = next(reader) except IOError as error: if error.errno == 2: self.logger.error('Search results info file {} does not exist.'.format(json_encode_string(path))) return raise def convert_field(field): return (field[1:] if field[0] == '_' else field).replace('.', '_') decode = MetadataDecoder().decode def convert_value(value): try: return decode(value) if len(value) > 0 else value except ValueError: return value info = ObjectView(dict(imap(lambda f_v: (convert_field(f_v[0]), convert_value(f_v[1])), izip(fields, values)))) try: count_map = info.countMap except AttributeError: pass else: count_map = count_map.split(';') n = len(count_map) info.countMap = dict(izip(islice(count_map, 0, n, 2), islice(count_map, 1, n, 2))) try: msg_type = info.msgType msg_text = info.msg except AttributeError: pass else: messages = ifilter(lambda t_m: t_m[0] or t_m[1], izip(msg_type.split('\n'), msg_text.split('\n'))) info.msg = [Message(message) for message in messages] del info.msgType try: info.vix_families = ElementTree.fromstring(info.vix_families) except AttributeError: pass self._search_results_info = info return info