def test_recursive_multi_path(self, os_walk_mock, isdir_mock,
update_jobs_mock):
"""
Run test mode and pass multiple paths with recursive path option.
"""
os_walk_mock.side_effect = os_walk_side_effects
isdir_mock.return_value = True
path_list = os_walk_return_values.keys()
paths = []
for path in path_list:
paths.extend([p for p, _ in os_walk_return_values[path]])
multipath = os.pathsep.join(path_list)
args = self.parser.parse_args(['test', '-r', multipath])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config)
update_jobs_mock.assert_called_with(paths, [], output=args.output_dir,
n_workers=mock.ANY)
args = self.parser.parse_args(['test', multipath])
args.output_dir = mock.MagicMock()
self.config.set('job_builder', 'recursive', 'True')
cmd.execute(args, self.config)
update_jobs_mock.assert_called_with(paths, [], output=args.output_dir,
n_workers=mock.ANY)
def _factor_query_builder(self, builder_class):
mocked_handler = mock.MagicMock()
mocked_logger = mock.MagicMock()
query_builder = builder_class(config_index_name='foo',
kibana_index_name='bar',
saved_search_title='foo',
custom_search='bar',
http_handler=mocked_handler,
logger=mocked_logger)
return query_builder
def test_custom_search_v6_without_saved_search_and_no_search_query(self, mock_es_detection):
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_6
mocked_handler = mock.MagicMock()
mocked_logger = mock.MagicMock()
custom_search = None
saved_search = None
# test
test_query = self._factor_query(mocked_handler, mocked_logger, saved_search, custom_search)
self._evaluate_custom_search_query_6(test_query, '*')
def test_custom_search_v4_with_saved_search_and_search_query(self, mock_es_detection):
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_2
test_response_kibana4 = self._get_test_data('saved_searches_kibana4')
mocked_handler = mock.MagicMock()
mocked_handler.request.return_value = test_response_kibana4
mocked_logger = mock.MagicMock()
custom_search = 'host: foobar'
saved_search = 'Parse Errors'
# test
test_query = self._factor_query(mocked_handler, mocked_logger, saved_search, custom_search)
self._evaluate_custom_search_query_4(test_query, custom_search)
def test_request_http_error_retry(self):
mocked_response = mock.MagicMock(status=200, spec=HTTPResponse)
mocked_response.read.return_value = b'{ "foo": "bar" }'
mocked_response.headers = mock.Mock()
mocked_response.headers.get_charset.return_value = 'utf-8'
# raise error on first call, then a valid response
mock_side_effect = [URLError('test error'), mocked_response]
test_servers = deque(TEST_SERVERS)
http_client = ElasticsearchRequestController(test_servers,
TEST_TIMEOUT, None, False,
self._mocked_logger)
# pre-flight check, basically cannot fail but won't hurt
self.assertEqual(http_client._servers[0], TEST_SERVER_1)
# test
with mock.patch.object(http_client, '_url_opener') as mock_url_opener:
mock_url_opener.open.side_effect = mock_side_effect
result = http_client.request('/', data=None)
# http_client._servers[0] is a deque and has been rotated after the first error,
# so now we expect the second server item to be at the first index
self.assertEqual(http_client._servers[0], TEST_SERVER_2)
# the response should match in any way, so check it
self.assertEqual(result, dict(foo='bar'))
def test_recursive_multi_path_with_excludes(self, os_walk_mock, isdir_mock,
update_job_mock):
"""
Run test mode and pass multiple paths with recursive path option.
"""
os_walk_mock.side_effect = os_walk_side_effects
isdir_mock.return_value = True
path_list = os_walk_return_values.keys()
paths = []
for path in path_list:
paths.extend([
p for p, __ in os_walk_return_values[path]
if 'dir1' not in p and 'dir2' not in p
])
multipath = os.pathsep.join(path_list)
args = self.parser.parse_args(
['test', '-r', multipath, '-x', 'dir1:dir2'])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config)
update_job_mock.assert_called_with(paths, [], output=args.output_dir)
def test_timestamp_query_v2(self, mock_es_detection):
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_2
http_handler = mock.MagicMock()
factory = QueryBuilderFactory(http_handler, self._mocked_logger)
query_builder = factory.factor(
'foo', 'bar', 'foo', 'foobar', http_handler, self._mocked_logger)
timestamp_from = datetime.now()
# fake filter and query
base_query = deepcopy(BASE_QUERY_ES2)
base_query['query']['filtered']['filter']['bool']['must'].append({'something': 'else'})
query = Query('foo-index', base_query, time_field_name='@timestamp')
# test with timestamp added
query_timestamp_added = query_builder.build_query_for_time_range(query, timestamp_from)
self._evaluate_timestamped_query(
query_timestamp_added.query['query']['filtered']['filter']['bool']['must'],
timestamp_from)
# test with timestamp replaced
query_timestamp_replaced = query_builder.build_query_for_time_range(
query_timestamp_added,
timestamp_from)
self._evaluate_timestamped_query(
query_timestamp_replaced.query['query']['filtered']['filter']['bool']['must'],
timestamp_from)
def test_non_existing_config_file(self):
"""
Run test mode and pass a non-existing configuration file
"""
args = self.parser.parse_args(['test', 'non-existing.yaml'])
args.output_dir = mock.MagicMock()
self.assertRaises(IOError, cmd.execute, args, self.config)
def test_log_request_error(self):
http_client = ElasticsearchRequestController(
TEST_SERVERS, TEST_TIMEOUT,
verify_ssl_certificates=True,
debug=False,
logger=self._mocked_logger)
# simple exception
self._mocked_logger.reset_mock()
http_client._log_request_error('/foo', TEST_SERVER_1, ValueError('foobar exc'))
expected_log_message = 'Server "{}" failed: {}, trying next server'.format(
TEST_SERVER_1.name, 'foobar exc')
self._mocked_logger.warning.assert_called_once_with(expected_log_message)
# HTTPError exception
self._mocked_logger.reset_mock()
exc_text = 'foobar exc read'
exc = mock.MagicMock()
exc.__str__.return_value = exc_text
exc.read.return_value = exc_text
http_client._log_request_error('/foo', TEST_SERVER_1, exc)
expected_log_message = 'Server "{}" failed: {}, trying next server'.format(
TEST_SERVER_1.name, exc_text)
self._mocked_logger.warning.assert_called_once_with(expected_log_message)
self._mocked_logger.debug.assert_called_once()
def test_detect_elasticsearch_version_v7(self, mock_es_detection):
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_7
http_handler = mock.MagicMock()
factory = QueryBuilderFactory(http_handler, self._mocked_logger)
query_builder = factory.factor(
'foo', 'bar', 'foo', 'foobar', http_handler, self._mocked_logger)
self.assertTrue(isinstance(query_builder, ElasticSearch7QueryBuilder))
def test_detect_elasticsearch_version_negative_invalid_data_wrong_version(
self):
http_handler = mock.MagicMock()
# unexpected JSON input - wrong version (ES 3 has never been existed)
http_handler.request.return_value = dict(version=dict(number='3.1.2'))
es_major_version = detect_elasticsearch_version(
http_handler, self._mocked_logger)
self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION)
self._mocked_logger.reset_mock()
def test_detect_elasticsearch_version_negative_invalid_data_string(self):
http_handler = mock.MagicMock()
# string input
http_handler.request.return_value = 'invalid cluster state, json expected'
es_major_version = detect_elasticsearch_version(
http_handler, self._mocked_logger)
self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION)
self._mocked_logger.info.assert_called_once() # expect a log entry
self._mocked_logger.reset_mock()
def test_detect_elasticsearch_version(self, es_major_version, test_data):
test_data = self._get_test_data(test_data)
http_handler = mock.MagicMock()
http_handler.request.return_value = test_data
es_major_version = detect_elasticsearch_version(
http_handler, self._mocked_logger)
self.assertEqual(es_major_version, es_major_version)
def test_detect_elasticsearch_version_negative_invalid_data_no_version(
self):
http_handler = mock.MagicMock()
# unexpected JSON input - empty version
http_handler.request.return_value = dict(version=dict())
es_major_version = detect_elasticsearch_version(
http_handler, self._mocked_logger)
self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION)
self._mocked_logger.info.assert_called_once() # expect a log entry
self._mocked_logger.reset_mock()
def test_valid_job(self):
"""
Run test mode and pass a valid job name
"""
args = self.parser.parse_args([
'test',
os.path.join(self.fixtures_path, 'cmd-001.yaml'), 'foo-job'
])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config) # probably better to fail here
def test_non_existing_job(self):
"""
Run test mode and pass a non-existing job name
(probably better to fail here)
"""
args = self.parser.parse_args([
'test',
os.path.join(self.fixtures_path, 'cmd-001.yaml'), 'invalid'
])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config) # probably better to fail here
def test_list_kibana_saved_searches_negative_no_hits(self, mock_es_detection):
test_response = self._get_test_data('saved_searches_empty')
# empty search response is the same for ES 2.x and 6.x, so this test should suffice
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_6
mock_handler = mock.MagicMock()
mock_handler.request.return_value = test_response
lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger)
result = lister.list()
# check
self.assertEqual(result, [])
def test_multi_path(self, update_job_mock):
"""
Run test mode and pass multiple paths.
"""
path_list = list(os_walk_return_values.keys())
multipath = os.pathsep.join(path_list)
args = self.parser.parse_args(['test', multipath])
args.output_dir = mock.MagicMock()
cmd.execute(args, self.config)
self.assertEqual(args.path, path_list)
update_job_mock.assert_called_with(path_list, [],
output=args.output_dir)
def test_detect_elasticsearch_version_negative_http_error(self):
http_handler = mock.MagicMock()
http_handler.request.side_effect = HTTPError(
url=None,
code='666',
msg='Faked test case error',
hdrs=None,
fp=None)
es_major_version = detect_elasticsearch_version(
http_handler, self._mocked_logger)
self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION)
# expect a log entry
self._mocked_logger.info.assert_called_once()
self._mocked_logger.reset_mock()
def test_list_kibana_saved_searches_positive_v6(self, mock_es_detection):
test_response_kibana6 = self._get_test_data('saved_searches_kibana6')
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_6
mock_handler = mock.MagicMock()
mock_handler.request.return_value = test_response_kibana6
lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger)
result = lister.list()
# check
expected_output = [
KibanaSavedSearch(
'Dummy saved search 1',
'host, program, log_level, _id, log_level_no_orig, message'),
KibanaSavedSearch('Parse Errors', 'tags, logsource, program, message'),
KibanaSavedSearch('Syslog', 'host, program, message')]
self.assertCountEqual(result, expected_output)
def test_list_kibana_saved_searches_positive_v4(self, mock_es_detection):
test_response_kibana4 = self._get_test_data('saved_searches_kibana4')
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_2
mock_handler = mock.MagicMock()
mock_handler.request.return_value = test_response_kibana4
lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger)
result = lister.list()
# check
expected_output = [
KibanaSavedSearch('Kibana Saved Search 2', 'fromhost, programname'),
KibanaSavedSearch(
'Kibana saved search 1',
'fromhost, requestHost, vHostPort, requestUrl'),
KibanaSavedSearch('Süslogging', 'fromhost, programname, severity, message')]
self.assertCountEqual(result, expected_output)
def test_list_kibana_saved_searches_positive_v7(self, mock_es_detection):
test_response_kibana6 = self._get_test_data('saved_searches_kibana7')
mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_7
mock_handler = mock.MagicMock()
mock_handler.request.return_value = test_response_kibana6
lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger)
result = lister.list()
# check
expected_output = [
KibanaSavedSearch('Parse Failures', 'tags, logsource, program, message'),
KibanaSavedSearch(
'Webserver',
'http_host, http_clientip, http_verb, message, http_code'),
KibanaSavedSearch('Webserver 404', 'http_clientip, http_verb, message, http_code')]
self.assertCountEqual(result, expected_output)
