def test_recursive_multi_path(self, os_walk_mock, isdir_mock, update_jobs_mock): """ Run test mode and pass multiple paths with recursive path option. """ os_walk_mock.side_effect = os_walk_side_effects isdir_mock.return_value = True path_list = os_walk_return_values.keys() paths = [] for path in path_list: paths.extend([p for p, _ in os_walk_return_values[path]]) multipath = os.pathsep.join(path_list) args = self.parser.parse_args(['test', '-r', multipath]) args.output_dir = mock.MagicMock() cmd.execute(args, self.config) update_jobs_mock.assert_called_with(paths, [], output=args.output_dir, n_workers=mock.ANY) args = self.parser.parse_args(['test', multipath]) args.output_dir = mock.MagicMock() self.config.set('job_builder', 'recursive', 'True') cmd.execute(args, self.config) update_jobs_mock.assert_called_with(paths, [], output=args.output_dir, n_workers=mock.ANY)
def _factor_query_builder(self, builder_class): mocked_handler = mock.MagicMock() mocked_logger = mock.MagicMock() query_builder = builder_class(config_index_name='foo', kibana_index_name='bar', saved_search_title='foo', custom_search='bar', http_handler=mocked_handler, logger=mocked_logger) return query_builder
def test_custom_search_v6_without_saved_search_and_no_search_query(self, mock_es_detection): mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_6 mocked_handler = mock.MagicMock() mocked_logger = mock.MagicMock() custom_search = None saved_search = None # test test_query = self._factor_query(mocked_handler, mocked_logger, saved_search, custom_search) self._evaluate_custom_search_query_6(test_query, '*')
def test_custom_search_v4_with_saved_search_and_search_query(self, mock_es_detection): mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_2 test_response_kibana4 = self._get_test_data('saved_searches_kibana4') mocked_handler = mock.MagicMock() mocked_handler.request.return_value = test_response_kibana4 mocked_logger = mock.MagicMock() custom_search = 'host: foobar' saved_search = 'Parse Errors' # test test_query = self._factor_query(mocked_handler, mocked_logger, saved_search, custom_search) self._evaluate_custom_search_query_4(test_query, custom_search)
def test_request_http_error_retry(self): mocked_response = mock.MagicMock(status=200, spec=HTTPResponse) mocked_response.read.return_value = b'{ "foo": "bar" }' mocked_response.headers = mock.Mock() mocked_response.headers.get_charset.return_value = 'utf-8' # raise error on first call, then a valid response mock_side_effect = [URLError('test error'), mocked_response] test_servers = deque(TEST_SERVERS) http_client = ElasticsearchRequestController(test_servers, TEST_TIMEOUT, None, False, self._mocked_logger) # pre-flight check, basically cannot fail but won't hurt self.assertEqual(http_client._servers[0], TEST_SERVER_1) # test with mock.patch.object(http_client, '_url_opener') as mock_url_opener: mock_url_opener.open.side_effect = mock_side_effect result = http_client.request('/', data=None) # http_client._servers[0] is a deque and has been rotated after the first error, # so now we expect the second server item to be at the first index self.assertEqual(http_client._servers[0], TEST_SERVER_2) # the response should match in any way, so check it self.assertEqual(result, dict(foo='bar'))
def test_recursive_multi_path_with_excludes(self, os_walk_mock, isdir_mock, update_job_mock): """ Run test mode and pass multiple paths with recursive path option. """ os_walk_mock.side_effect = os_walk_side_effects isdir_mock.return_value = True path_list = os_walk_return_values.keys() paths = [] for path in path_list: paths.extend([ p for p, __ in os_walk_return_values[path] if 'dir1' not in p and 'dir2' not in p ]) multipath = os.pathsep.join(path_list) args = self.parser.parse_args( ['test', '-r', multipath, '-x', 'dir1:dir2']) args.output_dir = mock.MagicMock() cmd.execute(args, self.config) update_job_mock.assert_called_with(paths, [], output=args.output_dir)
def test_timestamp_query_v2(self, mock_es_detection): mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_2 http_handler = mock.MagicMock() factory = QueryBuilderFactory(http_handler, self._mocked_logger) query_builder = factory.factor( 'foo', 'bar', 'foo', 'foobar', http_handler, self._mocked_logger) timestamp_from = datetime.now() # fake filter and query base_query = deepcopy(BASE_QUERY_ES2) base_query['query']['filtered']['filter']['bool']['must'].append({'something': 'else'}) query = Query('foo-index', base_query, time_field_name='@timestamp') # test with timestamp added query_timestamp_added = query_builder.build_query_for_time_range(query, timestamp_from) self._evaluate_timestamped_query( query_timestamp_added.query['query']['filtered']['filter']['bool']['must'], timestamp_from) # test with timestamp replaced query_timestamp_replaced = query_builder.build_query_for_time_range( query_timestamp_added, timestamp_from) self._evaluate_timestamped_query( query_timestamp_replaced.query['query']['filtered']['filter']['bool']['must'], timestamp_from)
def test_non_existing_config_file(self): """ Run test mode and pass a non-existing configuration file """ args = self.parser.parse_args(['test', 'non-existing.yaml']) args.output_dir = mock.MagicMock() self.assertRaises(IOError, cmd.execute, args, self.config)
def test_log_request_error(self): http_client = ElasticsearchRequestController( TEST_SERVERS, TEST_TIMEOUT, verify_ssl_certificates=True, debug=False, logger=self._mocked_logger) # simple exception self._mocked_logger.reset_mock() http_client._log_request_error('/foo', TEST_SERVER_1, ValueError('foobar exc')) expected_log_message = 'Server "{}" failed: {}, trying next server'.format( TEST_SERVER_1.name, 'foobar exc') self._mocked_logger.warning.assert_called_once_with(expected_log_message) # HTTPError exception self._mocked_logger.reset_mock() exc_text = 'foobar exc read' exc = mock.MagicMock() exc.__str__.return_value = exc_text exc.read.return_value = exc_text http_client._log_request_error('/foo', TEST_SERVER_1, exc) expected_log_message = 'Server "{}" failed: {}, trying next server'.format( TEST_SERVER_1.name, exc_text) self._mocked_logger.warning.assert_called_once_with(expected_log_message) self._mocked_logger.debug.assert_called_once()
def test_detect_elasticsearch_version_v7(self, mock_es_detection): mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_7 http_handler = mock.MagicMock() factory = QueryBuilderFactory(http_handler, self._mocked_logger) query_builder = factory.factor( 'foo', 'bar', 'foo', 'foobar', http_handler, self._mocked_logger) self.assertTrue(isinstance(query_builder, ElasticSearch7QueryBuilder))
def test_detect_elasticsearch_version_negative_invalid_data_wrong_version( self): http_handler = mock.MagicMock() # unexpected JSON input - wrong version (ES 3 has never been existed) http_handler.request.return_value = dict(version=dict(number='3.1.2')) es_major_version = detect_elasticsearch_version( http_handler, self._mocked_logger) self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION) self._mocked_logger.reset_mock()
def test_detect_elasticsearch_version_negative_invalid_data_string(self): http_handler = mock.MagicMock() # string input http_handler.request.return_value = 'invalid cluster state, json expected' es_major_version = detect_elasticsearch_version( http_handler, self._mocked_logger) self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION) self._mocked_logger.info.assert_called_once() # expect a log entry self._mocked_logger.reset_mock()
def test_detect_elasticsearch_version(self, es_major_version, test_data): test_data = self._get_test_data(test_data) http_handler = mock.MagicMock() http_handler.request.return_value = test_data es_major_version = detect_elasticsearch_version( http_handler, self._mocked_logger) self.assertEqual(es_major_version, es_major_version)
def test_detect_elasticsearch_version_negative_invalid_data_no_version( self): http_handler = mock.MagicMock() # unexpected JSON input - empty version http_handler.request.return_value = dict(version=dict()) es_major_version = detect_elasticsearch_version( http_handler, self._mocked_logger) self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION) self._mocked_logger.info.assert_called_once() # expect a log entry self._mocked_logger.reset_mock()
def test_valid_job(self): """ Run test mode and pass a valid job name """ args = self.parser.parse_args([ 'test', os.path.join(self.fixtures_path, 'cmd-001.yaml'), 'foo-job' ]) args.output_dir = mock.MagicMock() cmd.execute(args, self.config) # probably better to fail here
def test_non_existing_job(self): """ Run test mode and pass a non-existing job name (probably better to fail here) """ args = self.parser.parse_args([ 'test', os.path.join(self.fixtures_path, 'cmd-001.yaml'), 'invalid' ]) args.output_dir = mock.MagicMock() cmd.execute(args, self.config) # probably better to fail here
def test_list_kibana_saved_searches_negative_no_hits(self, mock_es_detection): test_response = self._get_test_data('saved_searches_empty') # empty search response is the same for ES 2.x and 6.x, so this test should suffice mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_6 mock_handler = mock.MagicMock() mock_handler.request.return_value = test_response lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger) result = lister.list() # check self.assertEqual(result, [])
def test_multi_path(self, update_job_mock): """ Run test mode and pass multiple paths. """ path_list = list(os_walk_return_values.keys()) multipath = os.pathsep.join(path_list) args = self.parser.parse_args(['test', multipath]) args.output_dir = mock.MagicMock() cmd.execute(args, self.config) self.assertEqual(args.path, path_list) update_job_mock.assert_called_with(path_list, [], output=args.output_dir)
def test_detect_elasticsearch_version_negative_http_error(self): http_handler = mock.MagicMock() http_handler.request.side_effect = HTTPError( url=None, code='666', msg='Faked test case error', hdrs=None, fp=None) es_major_version = detect_elasticsearch_version( http_handler, self._mocked_logger) self.assertEqual(es_major_version, DEFAULT_ELASTICSEARCH_MAJOR_VERSION) # expect a log entry self._mocked_logger.info.assert_called_once() self._mocked_logger.reset_mock()
def test_list_kibana_saved_searches_positive_v6(self, mock_es_detection): test_response_kibana6 = self._get_test_data('saved_searches_kibana6') mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_6 mock_handler = mock.MagicMock() mock_handler.request.return_value = test_response_kibana6 lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger) result = lister.list() # check expected_output = [ KibanaSavedSearch( 'Dummy saved search 1', 'host, program, log_level, _id, log_level_no_orig, message'), KibanaSavedSearch('Parse Errors', 'tags, logsource, program, message'), KibanaSavedSearch('Syslog', 'host, program, message')] self.assertCountEqual(result, expected_output)
def test_list_kibana_saved_searches_positive_v4(self, mock_es_detection): test_response_kibana4 = self._get_test_data('saved_searches_kibana4') mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_2 mock_handler = mock.MagicMock() mock_handler.request.return_value = test_response_kibana4 lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger) result = lister.list() # check expected_output = [ KibanaSavedSearch('Kibana Saved Search 2', 'fromhost, programname'), KibanaSavedSearch( 'Kibana saved search 1', 'fromhost, requestHost, vHostPort, requestUrl'), KibanaSavedSearch('Süslogging', 'fromhost, programname, severity, message')] self.assertCountEqual(result, expected_output)
def test_list_kibana_saved_searches_positive_v7(self, mock_es_detection): test_response_kibana6 = self._get_test_data('saved_searches_kibana7') mock_es_detection.return_value = ELASTICSEARCH_MAJOR_VERSION_7 mock_handler = mock.MagicMock() mock_handler.request.return_value = test_response_kibana6 lister = ListKibanaSavedSearchesController(TEST_CONFIG, mock_handler, self._mocked_logger) result = lister.list() # check expected_output = [ KibanaSavedSearch('Parse Failures', 'tags, logsource, program, message'), KibanaSavedSearch( 'Webserver', 'http_host, http_clientip, http_verb, message, http_code'), KibanaSavedSearch('Webserver 404', 'http_clientip, http_verb, message, http_code')] self.assertCountEqual(result, expected_output)