def test_process_scpv2(self):
# SearchCommand.process should
# 1. Recognize all standard options:
metadata = (
'{{'
'"action": "getinfo", "preview": false, "searchinfo": {{'
'"latest_time": "0",'
'"splunk_version": "20150522",'
'"username": "******",'
'"app": "searchcommands_app",'
'"args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"search": "Ａ%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",'
'"earliest_time": "0",'
'"session_key": "0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",'
'"owner": "admin",'
'"sid": "1433261372.158",'
'"splunkd_uri": "https://127.0.0.1:8089",'
'"dispatch_dir": {dispatch_dir},'
'"raw_args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"maxresultrows": 10,'
'"command": "countmatches"'
'}}'
'}}')
basedir = self._package_directory
default_logging_configuration = os.path.join(
basedir, 'apps', 'app_with_logging_configuration', 'default',
'logging.conf')
dispatch_dir = os.path.join(basedir, 'recordings', 'scpv2',
'Splunk-6.3', 'countmatches.dispatch_dir')
logging_configuration = os.path.join(basedir, 'apps',
'app_with_logging_configuration',
'logging.conf')
logging_level = 'ERROR'
record = False
show_configuration = True
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true'
if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = BytesIO()
argv = ['some-external-search-command.py']
self.assertEqual(command.logging_level, 'WARNING')
self.assertIs(command.record, None)
self.assertIs(command.show_configuration, None)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.fail('Unexpected exception: {}: {}'.format(
type(error).__name__, error))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,68,0\n'
'{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
'chunked 1.0,17,23\n'
'{"finished":true}test,__mv_test\r\n'
'data,\r\n',
result.getvalue().decode('utf-8'))
self.assertEqual(command.protocol_version, 2)
# 2. Provide access to these properties:
# fieldnames
# input_header
# metadata
# search_results_info
# service
self.assertEqual([], command.fieldnames)
command_metadata = command.metadata
input_header = command.input_header
self.assertIsNone(input_header['allowStream'])
self.assertEqual(
input_header['infoPath'],
os.path.join(command_metadata.searchinfo.dispatch_dir, 'info.csv'))
self.assertIsNone(input_header['keywords'])
self.assertEqual(input_header['preview'], command_metadata.preview)
self.assertIs(input_header['realtime'], False)
self.assertEqual(input_header['search'],
command_metadata.searchinfo.search)
self.assertEqual(input_header['sid'], command_metadata.searchinfo.sid)
self.assertEqual(input_header['splunkVersion'],
command_metadata.searchinfo.splunk_version)
self.assertIsNone(input_header['truncated'])
self.assertEqual(command_metadata.preview, input_header['preview'])
self.assertEqual(command_metadata.searchinfo.app, 'searchcommands_app')
self.assertEqual(command_metadata.searchinfo.args, [
'logging_configuration=' + logging_configuration,
'logging_level=ERROR', 'record=false', 'show_configuration=true',
'required_option_1=value_1', 'required_option_2=value_2'
])
self.assertEqual(command_metadata.searchinfo.dispatch_dir,
os.path.dirname(input_header['infoPath']))
self.assertEqual(command_metadata.searchinfo.earliest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.latest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.owner, 'admin')
self.assertEqual(command_metadata.searchinfo.raw_args,
command_metadata.searchinfo.args)
self.assertEqual(
command_metadata.searchinfo.search,
'Ａ| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw'
)
self.assertEqual(
command_metadata.searchinfo.session_key,
'0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^'
)
self.assertEqual(command_metadata.searchinfo.sid, '1433261372.158')
self.assertEqual(command_metadata.searchinfo.splunk_version,
'20150522')
self.assertEqual(command_metadata.searchinfo.splunkd_uri,
'https://127.0.0.1:8089')
self.assertEqual(command_metadata.searchinfo.username, 'admin')
self.assertEqual(command_metadata.searchinfo.maxresultrows, 10)
self.assertEqual(command_metadata.searchinfo.command, 'countmatches')
command.search_results_info.search_metrics = command.search_results_info.search_metrics.__dict__
command.search_results_info.optional_fields_json = command.search_results_info.optional_fields_json.__dict__
self.maxDiff = None
self.assertDictEqual(
command.search_results_info.__dict__, {
u'is_summary_index': 0,
u'bs_thread_count': 1,
u'rt_backfill': 0,
u'rtspan': '',
u'search_StartTime': 1433261392.934936,
u'read_raw': 1,
u'root_sid': '',
u'field_rendering': '',
u'query_finished': 1,
u'optional_fields_json': {},
u'group_list': '',
u'remoteServers': '',
u'rt_latest': '',
u'remote_log_download_mode': 'disabled',
u'reduce_search': '',
u'request_finalization': 0,
u'auth_token':
'UQZSgWwE2f9oIKrj1QG^kVhW^T_cR4H5Z65bPtMhwlHytS5jFrFYyH^dGzjTusDjVTgoBNeR7bvIzctHF7DrLJ1ANevgDOWEWRvABNj6d_k0koqxw9Io',
u'indexed_realtime': 0,
u'ppc_bs': '$SPLUNK_HOME/etc',
u'drop_count': 0,
u'datamodel_map': '',
u'search_can_be_event_type': 0,
u'search_StartUp_Spent': 0,
u'realtime': 0,
u'splunkd_uri': 'https://127.0.0.1:8089',
u'columnOrder': '',
u'kv_store_settings':
'hosts;127.0.0.1:8191\\;;local;127.0.0.1:8191;read_preference;958513E3-8716-4ABF-9559-DA0C9678437F;replica_set_name;958513E3-8716-4ABF-9559-DA0C9678437F;status;ready;',
u'label': '',
u'summary_maxtimespan': '',
u'indexed_realtime_offset': 0,
u'sid': 1433261392.159,
u'msg': [],
u'internal_only': 0,
u'summary_id': '',
u'orig_search_head': '',
u'ppc_app': 'chunked_searchcommands',
u'countMap': {
u'invocations.dispatch.writeStatus': u'1',
u'duration.dispatch.writeStatus': u'2',
u'duration.startup.handoff': u'79',
u'duration.startup.configuration': u'34',
u'invocations.startup.handoff': u'1',
u'invocations.startup.configuration': u'1'
},
u'is_shc_mode': 0,
u'shp_id': '958513E3-8716-4ABF-9559-DA0C9678437F',
u'timestamp': 1433261392.936374,
u'is_remote_sorted': 0,
u'remote_search': '',
u'splunkd_protocol': 'https',
u'site': '',
u'maxevents': 0,
u'keySet': '',
u'summary_stopped': 0,
u'search_metrics': {
u'ConsideredEvents': 0,
u'ConsideredBuckets': 0,
u'TotalSlicesInBuckets': 0,
u'EliminatedBuckets': 0,
u'DecompressedSlices': 0
},
u'summary_mode': 'all',
u'now': 1433261392.0,
u'splunkd_port': 8089,
u'is_saved_search': 0,
u'rtoptions': '',
u'search':
'| inputlookup random_data max=50000 | sum total=total value1 record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw',
u'bundle_version': 0,
u'generation_id': 0,
u'bs_thread_id': 0,
u'is_batch_mode': 0,
u'scan_count': 0,
u'rt_earliest': '',
u'default_group': '*',
u'tstats_reduce': '',
u'kv_store_additional_settings':
'hosts_guids;958513E3-8716-4ABF-9559-DA0C9678437F\\;;',
u'enable_event_stream': 0,
u'is_remote': 0,
u'is_scheduled': 0,
u'sample_ratio': 1,
u'ppc_user': '******',
u'sample_seed': 0
})
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority,
command_metadata.searchinfo.splunkd_uri)
self.assertEqual(command.service.scheme,
command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port,
command.search_results_info.splunkd_port)
self.assertEqual(command.service.token,
command_metadata.searchinfo.session_key)
self.assertEqual(command.service.namespace.app,
command.metadata.searchinfo.app)
self.assertIsNone(command.service.namespace.owner)
self.assertIsNone(command.service.namespace.sharing)
self.assertEqual(command.protocol_version, 2)
# 3. Produce an error message, log a debug message, and exit when invalid standard option values are encountered
# Note on loggers
# Loggers are global and can't be removed once they're created. We create loggers that are keyed by class name
# Each instance of a class thus created gets access to the same logger. We created one in the prior test and
# set it's level to ERROR. That level is retained in this test.
logging_configuration = 'non-existent-logging.conf'
logging_level = 'NON-EXISTENT-LOGGING-LEVEL'
record = 'Non-boolean value'
show_configuration = 'Non-boolean value'
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=record,
show_configuration=show_configuration)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = BytesIO()
argv = ['test.py']
# noinspection PyTypeChecker
self.assertRaises(SystemExit,
command.process,
argv,
ifile,
ofile=result)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, False)
self.assertEqual(command.show_configuration, False)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,287,0\n'
'{"inspector":{"messages":[["ERROR","Illegal value: logging_configuration=non-existent-logging.conf"],'
'["ERROR","Illegal value: logging_level=NON-EXISTENT-LOGGING-LEVEL"],'
'["ERROR","Illegal value: record=Non-boolean value"],'
'["ERROR","Illegal value: show_configuration=Non-boolean value"]]}}\n'
'chunked 1.0,17,0\n'
'{"finished":true}',
result.getvalue().decode('utf-8'))
self.assertEqual(command.protocol_version, 2)
# 4. Produce an error message, log an error message that includes a traceback, and exit when an exception is
# raised during command execution.
logging_configuration = os.path.join(basedir, 'apps',
'app_with_logging_configuration',
'logging.conf')
logging_level = 'WARNING'
record = False
show_configuration = False
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true'
if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'action\r\nraise_exception\r\n'
ifile = StringIO('chunked 1.0,{},0\n{}'.format(len(getinfo_metadata),
getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(
len(execute_metadata), len(execute_body),
execute_metadata, execute_body))
command = TestCommand()
result = BytesIO()
argv = ['test.py']
try:
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(0, error.code)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(
type(error).__name__, error,
result.getvalue().decode('utf-8')))
else:
self.fail(
'Expected SystemExit, not a return from TestCommand.process: {}\n'
.format(result.getvalue().decode('utf-8')))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, logging_level)
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
finished = r'\"finished\":true'
if six.PY2:
inspector = \
r'\"inspector\":\{\"messages\":\[\[\"ERROR\",\"StandardError at \\\".+\\\", line \d+ : test ' \
r'logging_configuration=\\\".+\\\" logging_level=\\\"WARNING\\\" record=\\\"f\\\" ' \
r'required_option_1=\\\"value_1\\\" required_option_2=\\\"value_2\\\" show_configuration=\\\"f\\\"\"\]\]\}'
else:
inspector = \
r'\"inspector\":\{\"messages\":\[\[\"ERROR\",\"Exception at \\\".+\\\", line \d+ : test ' \
r'logging_configuration=\\\".+\\\" logging_level=\\\"WARNING\\\" record=\\\"f\\\" ' \
r'required_option_1=\\\"value_1\\\" required_option_2=\\\"value_2\\\" show_configuration=\\\"f\\\"\"\]\]\}'
self.assertRegexpMatches(
result.getvalue().decode('utf-8'), r'^chunked 1.0,2,0\n'
r'\{\}\n'
r'chunked 1.0,\d+,0\n'
r'\{(' + inspector + r',' + finished + r'|' + finished + r',' +
inspector + r')\}')
self.assertEqual(command.protocol_version, 2)
return
def test_process_scpv2(self):
# SearchCommand.process should
# 1. Recognize all standard options:
metadata = (
'{{'
'"action": "getinfo", "preview": false, "searchinfo": {{'
'"latest_time": "0",'
'"splunk_version": "20150522",'
'"username": "******",'
'"app": "searchcommands_app",'
'"args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"search": "Ａ%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",'
'"earliest_time": "0",'
'"session_key": "0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",'
'"owner": "admin",'
'"sid": "1433261372.158",'
'"splunkd_uri": "https://127.0.0.1:8089",'
'"dispatch_dir": {dispatch_dir},'
'"raw_args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
'],'
'"maxresultrows": 10,'
'"command": "countmatches"'
'}}'
'}}')
basedir = self._package_directory
default_logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'default', 'logging.conf')
dispatch_dir = os.path.join(basedir, 'recordings', 'scpv2', 'Splunk-6.3', 'countmatches.dispatch_dir')
logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'logging.conf')
logging_level = 'ERROR'
record = False
show_configuration = True
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true' if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO(
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['some-external-search-command.py']
self.assertEqual(command.logging_level, 'WARNING')
self.assertIs(command.record, None)
self.assertIs(command.show_configuration, None)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.fail('Unexpected exception: {}: {}'.format(type(error).__name__, error))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,68,0\n'
'{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
'chunked 1.0,17,23\n'
'{"finished":true}test,__mv_test\r\n'
'data,\r\n',
result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 2. Provide access to these properties:
# fieldnames
# input_header
# metadata
# search_results_info
# service
self.assertEqual([], command.fieldnames)
command_metadata = command.metadata
input_header = command.input_header
self.assertIsNone(input_header['allowStream'])
self.assertEqual(input_header['infoPath'], os.path.join(command_metadata.searchinfo.dispatch_dir, 'info.csv'))
self.assertIsNone(input_header['keywords'])
self.assertEqual(input_header['preview'], command_metadata.preview)
self.assertIs(input_header['realtime'], False)
self.assertEqual(input_header['search'], command_metadata.searchinfo.search)
self.assertEqual(input_header['sid'], command_metadata.searchinfo.sid)
self.assertEqual(input_header['splunkVersion'], command_metadata.searchinfo.splunk_version)
self.assertIsNone(input_header['truncated'])
self.assertEqual(command_metadata.preview, input_header['preview'])
self.assertEqual(command_metadata.searchinfo.app, 'searchcommands_app')
self.assertEqual(command_metadata.searchinfo.args, ['logging_configuration=' + logging_configuration, 'logging_level=ERROR', 'record=false', 'show_configuration=true', 'required_option_1=value_1', 'required_option_2=value_2'])
self.assertEqual(command_metadata.searchinfo.dispatch_dir, os.path.dirname(input_header['infoPath']))
self.assertEqual(command_metadata.searchinfo.earliest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.latest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.owner, 'admin')
self.assertEqual(command_metadata.searchinfo.raw_args, command_metadata.searchinfo.args)
self.assertEqual(command_metadata.searchinfo.search, 'Ａ| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw')
self.assertEqual(command_metadata.searchinfo.session_key, '0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^')
self.assertEqual(command_metadata.searchinfo.sid, '1433261372.158')
self.assertEqual(command_metadata.searchinfo.splunk_version, '20150522')
self.assertEqual(command_metadata.searchinfo.splunkd_uri, 'https://127.0.0.1:8089')
self.assertEqual(command_metadata.searchinfo.username, 'admin')
self.assertEqual(command_metadata.searchinfo.maxresultrows, 10)
self.assertEqual(command_metadata.searchinfo.command, 'countmatches')
command.search_results_info.search_metrics = command.search_results_info.search_metrics.__dict__
command.search_results_info.optional_fields_json = command.search_results_info.optional_fields_json.__dict__
self.maxDiff = None
self.assertDictEqual(command.search_results_info.__dict__, {
u'is_summary_index': 0,
u'bs_thread_count': 1,
u'rt_backfill': 0,
u'rtspan': '',
u'search_StartTime': 1433261392.934936,
u'read_raw': 1,
u'root_sid': '',
u'field_rendering': '',
u'query_finished': 1,
u'optional_fields_json': {},
u'group_list': '',
u'remoteServers': '',
u'rt_latest': '',
u'remote_log_download_mode': 'disabled',
u'reduce_search': '',
u'request_finalization': 0,
u'auth_token': 'UQZSgWwE2f9oIKrj1QG^kVhW^T_cR4H5Z65bPtMhwlHytS5jFrFYyH^dGzjTusDjVTgoBNeR7bvIzctHF7DrLJ1ANevgDOWEWRvABNj6d_k0koqxw9Io',
u'indexed_realtime': 0,
u'ppc_bs': '$SPLUNK_HOME/etc',
u'drop_count': 0,
u'datamodel_map': '',
u'search_can_be_event_type': 0,
u'search_StartUp_Spent': 0,
u'realtime': 0,
u'splunkd_uri': 'https://127.0.0.1:8089',
u'columnOrder': '',
u'kv_store_settings': 'hosts;127.0.0.1:8191\\;;local;127.0.0.1:8191;read_preference;958513E3-8716-4ABF-9559-DA0C9678437F;replica_set_name;958513E3-8716-4ABF-9559-DA0C9678437F;status;ready;',
u'label': '',
u'summary_maxtimespan': '',
u'indexed_realtime_offset': 0,
u'sid': 1433261392.159,
u'msg': [],
u'internal_only': 0,
u'summary_id': '',
u'orig_search_head': '',
u'ppc_app': 'chunked_searchcommands',
u'countMap': {
u'invocations.dispatch.writeStatus': u'1',
u'duration.dispatch.writeStatus': u'2',
u'duration.startup.handoff': u'79',
u'duration.startup.configuration': u'34',
u'invocations.startup.handoff': u'1',
u'invocations.startup.configuration': u'1'},
u'is_shc_mode': 0,
u'shp_id': '958513E3-8716-4ABF-9559-DA0C9678437F',
u'timestamp': 1433261392.936374, u'is_remote_sorted': 0,
u'remote_search': '',
u'splunkd_protocol': 'https',
u'site': '',
u'maxevents': 0,
u'keySet': '',
u'summary_stopped': 0,
u'search_metrics': {
u'ConsideredEvents': 0,
u'ConsideredBuckets': 0,
u'TotalSlicesInBuckets': 0,
u'EliminatedBuckets': 0,
u'DecompressedSlices': 0},
u'summary_mode': 'all', u'now': 1433261392.0,
u'splunkd_port': 8089, u'is_saved_search': 0,
u'rtoptions': '',
u'search': '| inputlookup random_data max=50000 | sum total=total value1 record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw',
u'bundle_version': 0,
u'generation_id': 0,
u'bs_thread_id': 0,
u'is_batch_mode': 0,
u'scan_count': 0,
u'rt_earliest': '',
u'default_group': '*',
u'tstats_reduce': '',
u'kv_store_additional_settings': 'hosts_guids;958513E3-8716-4ABF-9559-DA0C9678437F\\;;',
u'enable_event_stream': 0,
u'is_remote': 0,
u'is_scheduled': 0,
u'sample_ratio': 1,
u'ppc_user': '******',
u'sample_seed': 0})
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority, command_metadata.searchinfo.splunkd_uri)
self.assertEqual(command.service.scheme, command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port, command.search_results_info.splunkd_port)
self.assertEqual(command.service.token, command_metadata.searchinfo.session_key)
self.assertEqual(command.service.namespace.app, command.metadata.searchinfo.app)
self.assertIsNone(command.service.namespace.owner)
self.assertIsNone(command.service.namespace.sharing)
self.assertEqual(command.protocol_version, 2)
# 3. Produce an error message, log a debug message, and exit when invalid standard option values are encountered
# Note on loggers
# Loggers are global and can't be removed once they're created. We create loggers that are keyed by class name
# Each instance of a class thus created gets access to the same logger. We created one in the prior test and
# set it's level to ERROR. That level is retained in this test.
logging_configuration = 'non-existent-logging.conf'
logging_level = 'NON-EXISTENT-LOGGING-LEVEL'
record = 'Non-boolean value'
show_configuration = 'Non-boolean value'
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=record,
show_configuration=show_configuration)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'test\r\ndata\r\n'
ifile = StringIO(
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
# noinspection PyTypeChecker
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertEqual(command.logging_level, 'ERROR')
self.assertEqual(command.record, False)
self.assertEqual(command.show_configuration, False)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
self.assertEqual(
'chunked 1.0,287,0\n'
'{"inspector":{"messages":[["ERROR","Illegal value: logging_configuration=non-existent-logging.conf"],'
'["ERROR","Illegal value: logging_level=NON-EXISTENT-LOGGING-LEVEL"],'
'["ERROR","Illegal value: record=Non-boolean value"],'
'["ERROR","Illegal value: show_configuration=Non-boolean value"]]}}\n'
'chunked 1.0,17,0\n'
'{"finished":true}',
result.getvalue())
self.assertEqual(command.protocol_version, 2)
# 4. Produce an error message, log an error message that includes a traceback, and exit when an exception is
# raised during command execution.
logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'logging.conf')
logging_level = 'WARNING'
record = False
show_configuration = False
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=('true' if record is True else 'false'),
show_configuration=('true' if show_configuration is True else 'false'))
execute_metadata = '{"action":"execute","finished":true}'
execute_body = 'action\r\nraise_exception\r\n'
ifile = StringIO(
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
command = TestCommand()
result = StringIO()
argv = ['test.py']
try:
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(0, error.code)
except BaseException as error:
self.fail('{0}: {1}: {2}\n'.format(type(error).__name__, error, result.getvalue()))
else:
self.fail('Expected SystemExit, not a return from TestCommand.process: {}\n'.format(result.getvalue()))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, logging_level)
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, 'value_1')
self.assertEqual(command.required_option_2, 'value_2')
finished = r'"finished":true'
inspector = \
r'"inspector":\{"messages":\[\["ERROR","StandardError at \\".+\\", line \d+ : test ' \
r'logging_configuration=\\".+\\" logging_level=\\"WARNING\\" record=\\"f\\" ' \
r'required_option_1=\\"value_1\\" required_option_2=\\"value_2\\" show_configuration=\\"f\\""\]\]\}'
self.assertRegexpMatches(
result.getvalue(),
r'^chunked 1.0,2,0\n'
r'\{\}\n'
r'chunked 1.0,\d+,0\n'
r'\{(' + inspector + r',' + finished + r'|' + finished + r',' + inspector + r')\}')
self.assertEqual(command.protocol_version, 2)
return
def test_process_scpv2(self):
# SearchCommand.process should
# 1. Recognize all standard options:
metadata = (
"{{"
'"action": "getinfo", "preview": false, "searchinfo": {{'
'"latest_time": "0",'
'"splunk_version": "20150522",'
'"username": "******",'
'"app": "searchcommands_app",'
'"args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
"],"
'"search": "%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",'
'"earliest_time": "0",'
'"session_key": "0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",'
'"owner": "admin",'
'"sid": "1433261372.158",'
'"splunkd_uri": "https://127.0.0.1:8089",'
'"dispatch_dir": {dispatch_dir},'
'"raw_args": ['
'"logging_configuration={logging_configuration}",'
'"logging_level={logging_level}",'
'"record={record}",'
'"show_configuration={show_configuration}",'
'"required_option_1=value_1",'
'"required_option_2=value_2"'
"]"
"}}"
"}}"
)
basedir = self._package_directory
default_logging_configuration = os.path.join(
basedir, "apps", "app_with_logging_configuration", "default", "logging.conf"
)
dispatch_dir = os.path.join(basedir, "recordings", "scpv2", "Splunk-6.3", "countmatches.dispatch_dir")
logging_configuration = os.path.join(basedir, "apps", "app_with_logging_configuration", "logging.conf")
logging_level = "ERROR"
record = False
show_configuration = True
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=("true" if record is True else "false"),
show_configuration=("true" if show_configuration is True else "false"),
)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = "test\r\ndata\r\n"
ifile = StringIO(
"chunked 1.0,{},0\n{}".format(len(getinfo_metadata), getinfo_metadata)
+ "chunked 1.0,{},{}\n{}{}".format(len(execute_metadata), len(execute_body), execute_metadata, execute_body)
)
command = TestCommand()
result = StringIO()
argv = ["some-external-search-command.py"]
self.assertEqual(command.logging_level, "WARNING")
self.assertIs(command.record, None)
self.assertIs(command.show_configuration, None)
try:
# noinspection PyTypeChecker
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.fail("Unexpected exception: {}: {}".format(type(error).__name__, error))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, "ERROR")
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, "value_1")
self.assertEqual(command.required_option_2, "value_2")
self.assertEqual(
"chunked 1.0,68,0\n"
'{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
"chunked 1.0,17,23\n"
'{"finished":true}test,__mv_test\r\n'
"data,\r\n",
result.getvalue(),
)
self.assertEqual(command.protocol_version, 2)
# 2. Provide access to these properties:
# fieldnames
# input_header
# metadata
# search_results_info
# service
self.assertEqual([], command.fieldnames)
command_metadata = command.metadata
input_header = command.input_header
self.assertIsNone(input_header["allowStream"])
self.assertEqual(input_header["infoPath"], os.path.join(command_metadata.searchinfo.dispatch_dir, "info.csv"))
self.assertIsNone(input_header["keywords"])
self.assertEqual(input_header["preview"], command_metadata.preview)
self.assertIs(input_header["realtime"], False)
self.assertEqual(input_header["search"], command_metadata.searchinfo.search)
self.assertEqual(input_header["sid"], command_metadata.searchinfo.sid)
self.assertEqual(input_header["splunkVersion"], command_metadata.searchinfo.splunk_version)
self.assertIsNone(input_header["truncated"])
self.assertEqual(command_metadata.preview, input_header["preview"])
self.assertEqual(command_metadata.searchinfo.app, "searchcommands_app")
self.assertEqual(
command_metadata.searchinfo.args,
[
"logging_configuration=" + logging_configuration,
"logging_level=ERROR",
"record=false",
"show_configuration=true",
"required_option_1=value_1",
"required_option_2=value_2",
],
)
self.assertEqual(command_metadata.searchinfo.dispatch_dir, os.path.dirname(input_header["infoPath"]))
self.assertEqual(command_metadata.searchinfo.earliest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.latest_time, 0.0)
self.assertEqual(command_metadata.searchinfo.owner, "admin")
self.assertEqual(command_metadata.searchinfo.raw_args, command_metadata.searchinfo.args)
self.assertEqual(
command_metadata.searchinfo.search,
'| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw',
)
self.assertEqual(
command_metadata.searchinfo.session_key,
"0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",
)
self.assertEqual(command_metadata.searchinfo.sid, "1433261372.158")
self.assertEqual(command_metadata.searchinfo.splunk_version, "20150522")
self.assertEqual(command_metadata.searchinfo.splunkd_uri, "https://127.0.0.1:8089")
self.assertEqual(command_metadata.searchinfo.username, "admin")
command.search_results_info.search_metrics = command.search_results_info.search_metrics.__dict__
command.search_results_info.optional_fields_json = command.search_results_info.optional_fields_json.__dict__
self.maxDiff = None
self.assertDictEqual(
command.search_results_info.__dict__,
{
"is_summary_index": 0,
"bs_thread_count": 1,
"rt_backfill": 0,
"rtspan": "",
"search_StartTime": 1433261392.934936,
"read_raw": 1,
"root_sid": "",
"field_rendering": "",
"query_finished": 1,
"optional_fields_json": {},
"group_list": "",
"remoteServers": "",
"rt_latest": "",
"remote_log_download_mode": "disabled",
"reduce_search": "",
"request_finalization": 0,
"auth_token": "UQZSgWwE2f9oIKrj1QG^kVhW^T_cR4H5Z65bPtMhwlHytS5jFrFYyH^dGzjTusDjVTgoBNeR7bvIzctHF7DrLJ1ANevgDOWEWRvABNj6d_k0koqxw9Io",
"indexed_realtime": 0,
"ppc_bs": "$SPLUNK_HOME/etc",
"drop_count": 0,
"datamodel_map": "",
"search_can_be_event_type": 0,
"search_StartUp_Spent": 0,
"realtime": 0,
"splunkd_uri": "https://127.0.0.1:8089",
"columnOrder": "",
"kv_store_settings": "hosts;127.0.0.1:8191\\;;local;127.0.0.1:8191;read_preference;958513E3-8716-4ABF-9559-DA0C9678437F;replica_set_name;958513E3-8716-4ABF-9559-DA0C9678437F;status;ready;",
"label": "",
"summary_maxtimespan": "",
"indexed_realtime_offset": 0,
"sid": 1433261392.159,
"msg": [],
"internal_only": 0,
"summary_id": "",
"orig_search_head": "",
"ppc_app": "chunked_searchcommands",
"countMap": {
"invocations.dispatch.writeStatus": "1",
"duration.dispatch.writeStatus": "2",
"duration.startup.handoff": "79",
"duration.startup.configuration": "34",
"invocations.startup.handoff": "1",
"invocations.startup.configuration": "1",
},
"is_shc_mode": 0,
"shp_id": "958513E3-8716-4ABF-9559-DA0C9678437F",
"timestamp": 1433261392.936374,
"is_remote_sorted": 0,
"remote_search": "",
"splunkd_protocol": "https",
"site": "",
"maxevents": 0,
"keySet": "",
"summary_stopped": 0,
"search_metrics": {
"ConsideredEvents": 0,
"ConsideredBuckets": 0,
"TotalSlicesInBuckets": 0,
"EliminatedBuckets": 0,
"DecompressedSlices": 0,
},
"summary_mode": "all",
"now": 1433261392.0,
"splunkd_port": 8089,
"is_saved_search": 0,
"rtoptions": "",
"search": "| inputlookup random_data max=50000 | sum total=total value1 record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw",
"bundle_version": 0,
"generation_id": 0,
"bs_thread_id": 0,
"is_batch_mode": 0,
"scan_count": 0,
"rt_earliest": "",
"default_group": "*",
"tstats_reduce": "",
"kv_store_additional_settings": "hosts_guids;958513E3-8716-4ABF-9559-DA0C9678437F\\;;",
"enable_event_stream": 0,
"is_remote": 0,
"is_scheduled": 0,
"sample_ratio": 1,
"ppc_user": "******",
"sample_seed": 0,
},
)
self.assertIsInstance(command.service, Service)
self.assertEqual(command.service.authority, command_metadata.searchinfo.splunkd_uri)
self.assertEqual(command.service.scheme, command.search_results_info.splunkd_protocol)
self.assertEqual(command.service.port, command.search_results_info.splunkd_port)
self.assertEqual(command.service.token, command_metadata.searchinfo.session_key)
self.assertEqual(command.service.namespace.app, command.metadata.searchinfo.app)
self.assertIsNone(command.service.namespace.owner)
self.assertIsNone(command.service.namespace.sharing)
self.assertEqual(command.protocol_version, 2)
# 3. Produce an error message, log a debug message, and exit when invalid standard option values are encountered
# Note on loggers
# Loggers are global and can't be removed once they're created. We create loggers that are keyed by class name
# Each instance of a class thus created gets access to the same logger. We created one in the prior test and
# set it's level to ERROR. That level is retained in this test.
logging_configuration = "non-existent-logging.conf"
logging_level = "NON-EXISTENT-LOGGING-LEVEL"
record = "Non-boolean value"
show_configuration = "Non-boolean value"
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=record,
show_configuration=show_configuration,
)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = "test\r\ndata\r\n"
ifile = StringIO(
"chunked 1.0,{},0\n{}".format(len(getinfo_metadata), getinfo_metadata)
+ "chunked 1.0,{},{}\n{}{}".format(len(execute_metadata), len(execute_body), execute_metadata, execute_body)
)
command = TestCommand()
result = StringIO()
argv = ["test.py"]
# noinspection PyTypeChecker
self.assertRaises(SystemExit, command.process, argv, ifile, ofile=result)
self.assertEqual(command.logging_level, "ERROR")
self.assertEqual(command.record, False)
self.assertEqual(command.show_configuration, False)
self.assertEqual(command.required_option_1, "value_1")
self.assertEqual(command.required_option_2, "value_2")
self.assertEqual(
"chunked 1.0,287,0\n"
'{"inspector":{"messages":[["ERROR","Illegal value: logging_configuration=non-existent-logging.conf"],'
'["ERROR","Illegal value: logging_level=NON-EXISTENT-LOGGING-LEVEL"],'
'["ERROR","Illegal value: record=Non-boolean value"],'
'["ERROR","Illegal value: show_configuration=Non-boolean value"]]}}\n'
"chunked 1.0,17,0\n"
'{"finished":true}',
result.getvalue(),
)
self.assertEqual(command.protocol_version, 2)
# 4. Produce an error message, log an error message that includes a traceback, and exit when an exception is
# raised during command execution.
logging_configuration = os.path.join(basedir, "apps", "app_with_logging_configuration", "logging.conf")
logging_level = "WARNING"
record = False
show_configuration = False
getinfo_metadata = metadata.format(
dispatch_dir=encode_string(dispatch_dir),
logging_configuration=encode_string(logging_configuration)[1:-1],
logging_level=logging_level,
record=("true" if record is True else "false"),
show_configuration=("true" if show_configuration is True else "false"),
)
execute_metadata = '{"action":"execute","finished":true}'
execute_body = "action\r\nraise_exception\r\n"
ifile = StringIO(
"chunked 1.0,{},0\n{}".format(len(getinfo_metadata), getinfo_metadata)
+ "chunked 1.0,{},{}\n{}{}".format(len(execute_metadata), len(execute_body), execute_metadata, execute_body)
)
command = TestCommand()
result = StringIO()
argv = ["test.py"]
try:
command.process(argv, ifile, ofile=result)
except SystemExit as error:
self.assertNotEqual(0, error.code)
except BaseException as error:
self.fail("{0}: {1}: {2}\n".format(type(error).__name__, error, result.getvalue()))
else:
self.fail("Expected SystemExit, not a return from TestCommand.process: {}\n".format(result.getvalue()))
self.assertEqual(command.logging_configuration, logging_configuration)
self.assertEqual(command.logging_level, logging_level)
self.assertEqual(command.record, record)
self.assertEqual(command.show_configuration, show_configuration)
self.assertEqual(command.required_option_1, "value_1")
self.assertEqual(command.required_option_2, "value_2")
finished = r'"finished":true'
inspector = (
r'"inspector":\{"messages":\[\["ERROR","StandardError at \\".+\\", line \d+ : test '
r'logging_configuration=\\".+\\" logging_level=\\"WARNING\\" record=\\"f\\" '
r'required_option_1=\\"value_1\\" required_option_2=\\"value_2\\" show_configuration=\\"f\\""\]\]\}'
)
self.assertRegexpMatches(
result.getvalue(),
r"^chunked 1.0,2,0\n"
r"\{\}\n"
r"chunked 1.0,\d+,0\n"
r"\{(" + inspector + r"," + finished + r"|" + finished + r"," + inspector + r")\}",
)
self.assertEqual(command.protocol_version, 2)
return