def test_calculate_call_duration(self): http_client = ElasticsearchRequestController( TEST_SERVERS, TEST_TIMEOUT, verify_ssl_certificates=True, debug=False, logger=self._mocked_logger) begin_date_time = datetime(2018, 2, 22, 22, 22, 22) end_date_time = datetime(2018, 2, 22, 22, 22, 42) with freeze_time(end_date_time): duration = http_client._calculate_call_duration(begin_date_time) self.assertEqual(duration, 20) end_date_time = begin_date_time with freeze_time(end_date_time): duration = http_client._calculate_call_duration(begin_date_time) self.assertEqual(duration, 0)
def test_valid_server_url(self): http_client = ElasticsearchRequestController( TEST_SERVERS, TEST_TIMEOUT, verify_ssl_certificates=True, debug=False, logger=self._mocked_logger) # normal server = mock.Mock(url='http://127.0.0.1:9200') path = 'foo/bar' url = http_client._factor_url(server, path) self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar') # trailing slashes on host for i in range(1, 5): server = mock.Mock(url='http://127.0.0.1:9200{}'.format('/' * i)) path = 'foo/bar' url = http_client._factor_url(server, path) self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar') # trailing slashes on host for i in range(1, 5): server = mock.Mock(url='http://127.0.0.1:9200') path = '{}foo/bar'.format('/' * i) url = http_client._factor_url(server, path) self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar') # both for i in range(1, 5): server = mock.Mock(url='http://127.0.0.1:9200{}'.format('/' * i)) path = '{}foo/bar'.format('/' * i) url = http_client._factor_url(server, path) self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar')
def test_log_request_error(self): http_client = ElasticsearchRequestController( TEST_SERVERS, TEST_TIMEOUT, verify_ssl_certificates=True, debug=False, logger=self._mocked_logger) # simple exception self._mocked_logger.reset_mock() http_client._log_request_error('/foo', TEST_SERVER_1, ValueError('foobar exc')) expected_log_message = 'Server "{}" failed: {}, trying next server'.format( TEST_SERVER_1.name, 'foobar exc') self._mocked_logger.warning.assert_called_once_with(expected_log_message) # HTTPError exception self._mocked_logger.reset_mock() exc_text = 'foobar exc read' exc = mock.MagicMock() exc.__str__.return_value = exc_text exc.read.return_value = exc_text http_client._log_request_error('/foo', TEST_SERVER_1, exc) expected_log_message = 'Server "{}" failed: {}, trying next server'.format( TEST_SERVER_1.name, exc_text) self._mocked_logger.warning.assert_called_once_with(expected_log_message) self._mocked_logger.debug.assert_called_once()
def test_request_http_error_retry(self): mocked_response = mock.MagicMock(status=200, spec=HTTPResponse) mocked_response.read.return_value = b'{ "foo": "bar" }' mocked_response.headers = mock.Mock() mocked_response.headers.get_charset.return_value = 'utf-8' # raise error on first call, then a valid response mock_side_effect = [URLError('test error'), mocked_response] test_servers = deque(TEST_SERVERS) http_client = ElasticsearchRequestController( test_servers, TEST_TIMEOUT, None, False, self._mocked_logger) # pre-flight check, basically cannot fail but won't hurt self.assertEqual(http_client._servers[0], TEST_SERVER_1) # test with mock.patch.object(http_client, '_url_opener') as mock_url_opener: mock_url_opener.open.side_effect = mock_side_effect result = http_client.request('/', data=None) # http_client._servers[0] is a deque and has been rotated after the first error, # so now we expect the second server item to be at the first index self.assertEqual(http_client._servers[0], TEST_SERVER_2) # the response should match in any way, so check it self.assertEqual(result, dict(foo='bar'))
def test_assert_response_not_timed_out(self): http_client = ElasticsearchRequestController( TEST_SERVERS, TEST_TIMEOUT, verify_ssl_certificates=True, debug=False, logger=self._mocked_logger) # simulate timeout self._mocked_logger.reset_mock() response = dict(timed_out=True) with self.assertRaises(HttpRetryError): http_client._assert_response_not_timed_out(response) self._mocked_logger.warning.assert_called_once() # response is not a dict response = None http_client._assert_response_not_timed_out(response) # response is dict but without timedout key response = dict(foo='bar') http_client._assert_response_not_timed_out(response)
def test_request_http_error(self): test_servers = deque(TEST_SERVERS) http_client = ElasticsearchRequestController( test_servers, TEST_TIMEOUT, None, False, self._mocked_logger) # expect HttpRetryError on URLError with mock.patch.object(http_client, '_url_opener') as mock_url_opener: mock_url_opener.open.side_effect = URLError('test error') with self.assertRaises(HttpRetryError): http_client._request_inner('/', data=None) # expect real error on non-URLError with mock.patch.object(http_client, '_url_opener') as mock_url_opener2: mock_url_opener2.open.side_effect = KeyboardInterrupt('test error') with self.assertRaises(KeyboardInterrupt): http_client._request_inner('/', data=None)
def test_factor_request(self): http_client = ElasticsearchRequestController( TEST_SERVERS, TEST_TIMEOUT, verify_ssl_certificates=True, debug=False, logger=self._mocked_logger) # GET request = http_client._factor_request('http://127.0.0.1:9200/foo', None, 'GET') self.assertEqual(request.get_method(), 'GET') # POST (even if the argument says GET) request = http_client._factor_request('http://127.0.0.1:9200/foo', 'data', 'GET') self.assertEqual(request.get_method(), 'POST') # HEAD request = http_client._factor_request('http://127.0.0.1:9200/foo', None, 'HEAD') self.assertEqual(request.get_method(), 'HEAD') # HEAD with data => exception with self.assertRaises(ValueError): http_client._factor_request('http://127.0.0.1:9200/foo', 'data', 'HEAD')
def test_get_encoding_from_response(self): http_client = ElasticsearchRequestController(None, None, None, False, None) response_raw = mock.Mock() # headers.get_charset() returns something, so we expect something response_raw.headers.get_charset.return_value = 'foo-enc' result = http_client._get_encoding_from_response(response_raw) self.assertEqual(result, 'foo-enc') # headers.get_charset() returns None, so we fall back to headers.get_content_charset() response_raw.headers.get_charset.return_value = None response_raw.headers.get_content_charset.return_value = 'foo-content-enc' result = http_client._get_encoding_from_response(response_raw) self.assertEqual(result, 'foo-content-enc') # headers.get_charset() returns None and headers.get_content_charset() returns None, # so we expect LOG_ENCODING as fallback with mock.patch('lstail.http.LOG_ENCODING', 'some-fallback-enc'): response_raw.headers.get_charset.return_value = None response_raw.headers.get_content_charset.return_value = None result = http_client._get_encoding_from_response(response_raw) self.assertEqual(result, 'some-fallback-enc') # headers.get_charset() returns something and headers.get_content_charset() returns # something else so we expect something response_raw.headers.get_charset.return_value = 'some-enc' response_raw.headers.get_content_charset.return_value = 'some-content-enc' result = http_client._get_encoding_from_response(response_raw) self.assertEqual(result, 'some-enc') # headers.get_charset() returns None and headers.get_content_charset() returns # something so we expect something but not LOG_ENCODING with mock.patch('lstail.http.LOG_ENCODING', 'some-fallback-enc'): response_raw.headers.get_charset.return_value = None response_raw.headers.get_content_charset.return_value = 'some-content-enc' result = http_client._get_encoding_from_response(response_raw) self.assertEqual(result, 'some-content-enc')
def test_parse_response(self): http_client = ElasticsearchRequestController(None, None, None, False, None) # simple text response_bytes_raw = mock.Mock() response_bytes_raw.read.return_value = b'test' # disable charset in response to force use of lstail.http.LOG_ENCODING response_bytes_raw.headers.get_charset.return_value = None response_bytes_raw.headers.get_content_charset.return_value = None decode_as_json = False result = http_client._parse_response(response_bytes_raw, decode_as_json) self.assertEqual(result, 'test') response_raw = mock.Mock() response_raw.headers.get_charset.return_value = None response_raw.headers.get_content_charset.return_value = None # None response with mock.patch('lstail.http.LOG_ENCODING', 'utf-8'): response_raw.read.return_value = None decode_as_json = True result = http_client._parse_response(response_raw, decode_as_json) self.assertEqual(result, None) # empty string response with mock.patch('lstail.http.LOG_ENCODING', 'utf-8'): response_raw.read.return_value = '' decode_as_json = True result = http_client._parse_response(response_raw, decode_as_json) self.assertEqual(result, '') # JSON, ISO encoding with mock.patch('lstail.http.LOG_ENCODING', 'iso-8859-15'): response_raw.read.return_value = '{ "foo": "bär" }'.encode( 'iso-8859-15') decode_as_json = True result = http_client._parse_response(response_raw, decode_as_json) self.assertEqual(result, dict(foo='bär'))
def test_setup_request_headers_type(self): def mocked_add_header(key, value): headers[key] = value http_client = ElasticsearchRequestController(None, None, None, False, None) http_client._user_agent = 'Dummy User-Agent for testing' # simple GET request request = mock.Mock() request.add_header = mocked_add_header data = None content_type = None server = TEST_SERVER_1 headers = dict() http_client._setup_request_headers(request, data, content_type, server) expected_headers = { 'User-agent': 'Dummy User-Agent for testing', 'key1_1': 'value1_1', 'key1_2': 'value1_2', } self.assertEqual(headers, expected_headers) # simple GET request with content-type request = mock.Mock() request.add_header = mocked_add_header data = None content_type = 'text/css' server = TEST_SERVER_1 headers = dict() http_client._setup_request_headers(request, data, content_type, server) expected_headers = { 'User-agent': 'Dummy User-Agent for testing', 'key1_1': 'value1_1', 'key1_2': 'value1_2', 'Content-Type': 'text/css', } self.assertEqual(headers, expected_headers) # request with data and content-type request = mock.Mock() request.add_header = mocked_add_header data = 'dummy data' content_type = 'text/css' server = TEST_SERVER_1 headers = dict() http_client._setup_request_headers(request, data, content_type, server) expected_headers = { 'User-agent': 'Dummy User-Agent for testing', 'key1_1': 'value1_1', 'key1_2': 'value1_2', 'Content-Type': 'text/css', } self.assertEqual(headers, expected_headers) # request with data, without content-type request = mock.Mock() request.add_header = mocked_add_header data = 'dummy data' content_type = None server = TEST_SERVER_1 headers = dict() http_client._setup_request_headers(request, data, content_type, server) expected_headers = { 'User-agent': 'Dummy User-Agent for testing', 'key1_1': 'value1_1', 'key1_2': 'value1_2', 'Content-Type': 'application/json', } self.assertEqual(headers, expected_headers)
class LogstashReader: # ---------------------------------------------------------------------- def __init__(self, config): self._config = config self._user_agent = None self._http_handler = None self._query_builder = None self._kibana_search = None self._base_query = None self._documents = None self._last_timestamp = None self._logger = None self._output = sys.stdout # ---------------------------------------------------------------------- def show_version(self): print('Lstail {}'.format(VERSION), file=self._output) # ---------------------------------------------------------------------- def list_kibana_saved_searches(self): self._setup_logger() self._setup_http_handler() saved_searches = self._get_kibana_saved_searches() if not saved_searches: print('No saved searches found in Kibana', file=self._output) else: for saved_search in saved_searches: print(u'{} ({})'.format(saved_search.title, saved_search.columns), file=self._output) # ---------------------------------------------------------------------- def _get_kibana_saved_searches(self): controller = ListKibanaSavedSearchesController(self._config, self._http_handler, self._logger) return controller.list() # ---------------------------------------------------------------------- def read(self): self._setup_logger() self._setup_http_handler() self._setup_timezone() self._setup_initial_time_range() self._prompt_for_kibana_saved_search_selection_if_necessary() self._factor_query_builder() self._build_base_query() self._print_header() while True: try: self._fetch_latest_documents() self._fetch_latest_timestamp() self._print_latest_documents() self._stop_reader_loop_if_necessary() self._wait_for_next_refresh_interval() except (StopReaderLoop, KeyboardInterrupt): return except Exception as exc: # pylint: disable=broad-except if self._config.debug: traceback = format_exc() traceback = '\n{}'.format(traceback) else: traceback = '' self._logger.error('Unexpected error occurred: {}{}', exc, traceback) self._wait_for_next_refresh_interval() # ---------------------------------------------------------------------- def _setup_logger(self): self._logger = LstailLogger(config=self._config, output=self._output, verbose=self._config.verbose) # ---------------------------------------------------------------------- def _setup_http_handler(self): self._http_handler = ElasticsearchRequestController( self._config.servers, timeout=self._config.timeout, verify_ssl_certificates=self._config.verify_ssl_certificates, debug=self._config.debug, logger=self._logger) # ---------------------------------------------------------------------- def _setup_timezone(self): environ['TZ'] = 'UTC' tzset() # ---------------------------------------------------------------------- def _setup_initial_time_range(self): if not self._config.initial_time_range: self._config.initial_time_range = '1d' # fallback to one day self._last_timestamp = parse_and_convert_time_range_to_start_date_time( self._config.initial_time_range) # ---------------------------------------------------------------------- def _prompt_for_kibana_saved_search_selection_if_necessary(self): if self._config.kibana.saved_search == '-' or self._config.select_kibana_saved_search: self._prompt_for_kibana_saved_search_selection() # ---------------------------------------------------------------------- def _prompt_for_kibana_saved_search_selection(self): saved_searches = self._get_kibana_saved_searches() kibana_saved_search_select_prompt = KibanaSavedSearchSelectPrompt( saved_searches) selected_saved_search = kibana_saved_search_select_prompt.prompt() # overwrite previously set saved search title self._config.kibana.saved_search = selected_saved_search # ---------------------------------------------------------------------- def _factor_query_builder(self): query_builder_factory = QueryBuilderFactory(self._http_handler, self._logger) self._query_builder = query_builder_factory.factor( self._config.default_index, self._config.kibana.index_name, self._config.kibana.saved_search, self._config.kibana.custom_search, self._http_handler, self._logger) # ---------------------------------------------------------------------- def _build_base_query(self): self._base_query = self._query_builder.build() # ---------------------------------------------------------------------- def _print_header(self): self._logger.print_header() # ---------------------------------------------------------------------- def _fetch_latest_documents(self): path = '%s/_search' % self._base_query.index timestamp_from = self._last_timestamp.strftime( ELASTICSEARCH_TIMESTAMP_FORMAT) query = self._query_builder.build_query_for_time_range( self._base_query, timestamp_from) query.query['size'] = self._config.initial_query_size query_json = dumps(query.query) response = self._http_handler.request(path, query_json) self._documents = response['hits']['hits'] self._documents.reverse() # ---------------------------------------------------------------------- def _fetch_latest_timestamp(self): if self._documents: latest_document = self._documents[-1] timestamp = latest_document['_source'][ self._base_query.time_field_name] last_timestamp = parse_timestamp_from_elasticsearch(timestamp) self._last_timestamp = min(last_timestamp, datetime.now()) # ---------------------------------------------------------------------- def _print_latest_documents(self): for document in self._documents: self._logger.log_document(document) # ---------------------------------------------------------------------- def _stop_reader_loop_if_necessary(self): if not self._config.follow: # raise a dedicated exception to break the while(true) loop in self.read() # if "--follow" CLI option was not specified raise StopReaderLoop() # ---------------------------------------------------------------------- def _wait_for_next_refresh_interval(self): sleep(self._config.refresh_interval)