def test_calculate_call_duration(self):
http_client = ElasticsearchRequestController(
TEST_SERVERS, TEST_TIMEOUT,
verify_ssl_certificates=True,
debug=False,
logger=self._mocked_logger)
begin_date_time = datetime(2018, 2, 22, 22, 22, 22)
end_date_time = datetime(2018, 2, 22, 22, 22, 42)
with freeze_time(end_date_time):
duration = http_client._calculate_call_duration(begin_date_time)
self.assertEqual(duration, 20)
end_date_time = begin_date_time
with freeze_time(end_date_time):
duration = http_client._calculate_call_duration(begin_date_time)
self.assertEqual(duration, 0)
def test_valid_server_url(self):
http_client = ElasticsearchRequestController(
TEST_SERVERS, TEST_TIMEOUT,
verify_ssl_certificates=True,
debug=False,
logger=self._mocked_logger)
# normal
server = mock.Mock(url='http://127.0.0.1:9200')
path = 'foo/bar'
url = http_client._factor_url(server, path)
self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar')
# trailing slashes on host
for i in range(1, 5):
server = mock.Mock(url='http://127.0.0.1:9200{}'.format('/' * i))
path = 'foo/bar'
url = http_client._factor_url(server, path)
self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar')
# trailing slashes on host
for i in range(1, 5):
server = mock.Mock(url='http://127.0.0.1:9200')
path = '{}foo/bar'.format('/' * i)
url = http_client._factor_url(server, path)
self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar')
# both
for i in range(1, 5):
server = mock.Mock(url='http://127.0.0.1:9200{}'.format('/' * i))
path = '{}foo/bar'.format('/' * i)
url = http_client._factor_url(server, path)
self.assertEqual(url, 'http://127.0.0.1:9200/foo/bar')
def test_log_request_error(self):
http_client = ElasticsearchRequestController(
TEST_SERVERS, TEST_TIMEOUT,
verify_ssl_certificates=True,
debug=False,
logger=self._mocked_logger)
# simple exception
self._mocked_logger.reset_mock()
http_client._log_request_error('/foo', TEST_SERVER_1, ValueError('foobar exc'))
expected_log_message = 'Server "{}" failed: {}, trying next server'.format(
TEST_SERVER_1.name, 'foobar exc')
self._mocked_logger.warning.assert_called_once_with(expected_log_message)
# HTTPError exception
self._mocked_logger.reset_mock()
exc_text = 'foobar exc read'
exc = mock.MagicMock()
exc.__str__.return_value = exc_text
exc.read.return_value = exc_text
http_client._log_request_error('/foo', TEST_SERVER_1, exc)
expected_log_message = 'Server "{}" failed: {}, trying next server'.format(
TEST_SERVER_1.name, exc_text)
self._mocked_logger.warning.assert_called_once_with(expected_log_message)
self._mocked_logger.debug.assert_called_once()
def test_request_http_error_retry(self):
mocked_response = mock.MagicMock(status=200, spec=HTTPResponse)
mocked_response.read.return_value = b'{ "foo": "bar" }'
mocked_response.headers = mock.Mock()
mocked_response.headers.get_charset.return_value = 'utf-8'
# raise error on first call, then a valid response
mock_side_effect = [URLError('test error'), mocked_response]
test_servers = deque(TEST_SERVERS)
http_client = ElasticsearchRequestController(
test_servers, TEST_TIMEOUT, None, False, self._mocked_logger)
# pre-flight check, basically cannot fail but won't hurt
self.assertEqual(http_client._servers[0], TEST_SERVER_1)
# test
with mock.patch.object(http_client, '_url_opener') as mock_url_opener:
mock_url_opener.open.side_effect = mock_side_effect
result = http_client.request('/', data=None)
# http_client._servers[0] is a deque and has been rotated after the first error,
# so now we expect the second server item to be at the first index
self.assertEqual(http_client._servers[0], TEST_SERVER_2)
# the response should match in any way, so check it
self.assertEqual(result, dict(foo='bar'))
def test_assert_response_not_timed_out(self):
http_client = ElasticsearchRequestController(
TEST_SERVERS, TEST_TIMEOUT,
verify_ssl_certificates=True,
debug=False,
logger=self._mocked_logger)
# simulate timeout
self._mocked_logger.reset_mock()
response = dict(timed_out=True)
with self.assertRaises(HttpRetryError):
http_client._assert_response_not_timed_out(response)
self._mocked_logger.warning.assert_called_once()
# response is not a dict
response = None
http_client._assert_response_not_timed_out(response)
# response is dict but without timedout key
response = dict(foo='bar')
http_client._assert_response_not_timed_out(response)
def test_request_http_error(self):
test_servers = deque(TEST_SERVERS)
http_client = ElasticsearchRequestController(
test_servers, TEST_TIMEOUT, None, False, self._mocked_logger)
# expect HttpRetryError on URLError
with mock.patch.object(http_client, '_url_opener') as mock_url_opener:
mock_url_opener.open.side_effect = URLError('test error')
with self.assertRaises(HttpRetryError):
http_client._request_inner('/', data=None)
# expect real error on non-URLError
with mock.patch.object(http_client, '_url_opener') as mock_url_opener2:
mock_url_opener2.open.side_effect = KeyboardInterrupt('test error')
with self.assertRaises(KeyboardInterrupt):
http_client._request_inner('/', data=None)
def test_factor_request(self):
http_client = ElasticsearchRequestController(
TEST_SERVERS, TEST_TIMEOUT,
verify_ssl_certificates=True,
debug=False,
logger=self._mocked_logger)
# GET
request = http_client._factor_request('http://127.0.0.1:9200/foo', None, 'GET')
self.assertEqual(request.get_method(), 'GET')
# POST (even if the argument says GET)
request = http_client._factor_request('http://127.0.0.1:9200/foo', 'data', 'GET')
self.assertEqual(request.get_method(), 'POST')
# HEAD
request = http_client._factor_request('http://127.0.0.1:9200/foo', None, 'HEAD')
self.assertEqual(request.get_method(), 'HEAD')
# HEAD with data => exception
with self.assertRaises(ValueError):
http_client._factor_request('http://127.0.0.1:9200/foo', 'data', 'HEAD')
def test_get_encoding_from_response(self):
http_client = ElasticsearchRequestController(None, None, None, False,
None)
response_raw = mock.Mock()
# headers.get_charset() returns something, so we expect something
response_raw.headers.get_charset.return_value = 'foo-enc'
result = http_client._get_encoding_from_response(response_raw)
self.assertEqual(result, 'foo-enc')
# headers.get_charset() returns None, so we fall back to headers.get_content_charset()
response_raw.headers.get_charset.return_value = None
response_raw.headers.get_content_charset.return_value = 'foo-content-enc'
result = http_client._get_encoding_from_response(response_raw)
self.assertEqual(result, 'foo-content-enc')
# headers.get_charset() returns None and headers.get_content_charset() returns None,
# so we expect LOG_ENCODING as fallback
with mock.patch('lstail.http.LOG_ENCODING', 'some-fallback-enc'):
response_raw.headers.get_charset.return_value = None
response_raw.headers.get_content_charset.return_value = None
result = http_client._get_encoding_from_response(response_raw)
self.assertEqual(result, 'some-fallback-enc')
# headers.get_charset() returns something and headers.get_content_charset() returns
# something else so we expect something
response_raw.headers.get_charset.return_value = 'some-enc'
response_raw.headers.get_content_charset.return_value = 'some-content-enc'
result = http_client._get_encoding_from_response(response_raw)
self.assertEqual(result, 'some-enc')
# headers.get_charset() returns None and headers.get_content_charset() returns
# something so we expect something but not LOG_ENCODING
with mock.patch('lstail.http.LOG_ENCODING', 'some-fallback-enc'):
response_raw.headers.get_charset.return_value = None
response_raw.headers.get_content_charset.return_value = 'some-content-enc'
result = http_client._get_encoding_from_response(response_raw)
self.assertEqual(result, 'some-content-enc')
def test_parse_response(self):
http_client = ElasticsearchRequestController(None, None, None, False,
None)
# simple text
response_bytes_raw = mock.Mock()
response_bytes_raw.read.return_value = b'test'
# disable charset in response to force use of lstail.http.LOG_ENCODING
response_bytes_raw.headers.get_charset.return_value = None
response_bytes_raw.headers.get_content_charset.return_value = None
decode_as_json = False
result = http_client._parse_response(response_bytes_raw,
decode_as_json)
self.assertEqual(result, 'test')
response_raw = mock.Mock()
response_raw.headers.get_charset.return_value = None
response_raw.headers.get_content_charset.return_value = None
# None response
with mock.patch('lstail.http.LOG_ENCODING', 'utf-8'):
response_raw.read.return_value = None
decode_as_json = True
result = http_client._parse_response(response_raw, decode_as_json)
self.assertEqual(result, None)
# empty string response
with mock.patch('lstail.http.LOG_ENCODING', 'utf-8'):
response_raw.read.return_value = ''
decode_as_json = True
result = http_client._parse_response(response_raw, decode_as_json)
self.assertEqual(result, '')
# JSON, ISO encoding
with mock.patch('lstail.http.LOG_ENCODING', 'iso-8859-15'):
response_raw.read.return_value = '{ "foo": "bär" }'.encode(
'iso-8859-15')
decode_as_json = True
result = http_client._parse_response(response_raw, decode_as_json)
self.assertEqual(result, dict(foo='bär'))
def test_setup_request_headers_type(self):
def mocked_add_header(key, value):
headers[key] = value
http_client = ElasticsearchRequestController(None, None, None, False, None)
http_client._user_agent = 'Dummy User-Agent for testing'
# simple GET request
request = mock.Mock()
request.add_header = mocked_add_header
data = None
content_type = None
server = TEST_SERVER_1
headers = dict()
http_client._setup_request_headers(request, data, content_type, server)
expected_headers = {
'User-agent': 'Dummy User-Agent for testing',
'key1_1': 'value1_1',
'key1_2': 'value1_2',
}
self.assertEqual(headers, expected_headers)
# simple GET request with content-type
request = mock.Mock()
request.add_header = mocked_add_header
data = None
content_type = 'text/css'
server = TEST_SERVER_1
headers = dict()
http_client._setup_request_headers(request, data, content_type, server)
expected_headers = {
'User-agent': 'Dummy User-Agent for testing',
'key1_1': 'value1_1',
'key1_2': 'value1_2',
'Content-Type': 'text/css',
}
self.assertEqual(headers, expected_headers)
# request with data and content-type
request = mock.Mock()
request.add_header = mocked_add_header
data = 'dummy data'
content_type = 'text/css'
server = TEST_SERVER_1
headers = dict()
http_client._setup_request_headers(request, data, content_type, server)
expected_headers = {
'User-agent': 'Dummy User-Agent for testing',
'key1_1': 'value1_1',
'key1_2': 'value1_2',
'Content-Type': 'text/css',
}
self.assertEqual(headers, expected_headers)
# request with data, without content-type
request = mock.Mock()
request.add_header = mocked_add_header
data = 'dummy data'
content_type = None
server = TEST_SERVER_1
headers = dict()
http_client._setup_request_headers(request, data, content_type, server)
expected_headers = {
'User-agent': 'Dummy User-Agent for testing',
'key1_1': 'value1_1',
'key1_2': 'value1_2',
'Content-Type': 'application/json',
}
self.assertEqual(headers, expected_headers)
class LogstashReader:
# ----------------------------------------------------------------------
def __init__(self, config):
self._config = config
self._user_agent = None
self._http_handler = None
self._query_builder = None
self._kibana_search = None
self._base_query = None
self._documents = None
self._last_timestamp = None
self._logger = None
self._output = sys.stdout
# ----------------------------------------------------------------------
def show_version(self):
print('Lstail {}'.format(VERSION), file=self._output)
# ----------------------------------------------------------------------
def list_kibana_saved_searches(self):
self._setup_logger()
self._setup_http_handler()
saved_searches = self._get_kibana_saved_searches()
if not saved_searches:
print('No saved searches found in Kibana', file=self._output)
else:
for saved_search in saved_searches:
print(u'{} ({})'.format(saved_search.title,
saved_search.columns),
file=self._output)
# ----------------------------------------------------------------------
def _get_kibana_saved_searches(self):
controller = ListKibanaSavedSearchesController(self._config,
self._http_handler,
self._logger)
return controller.list()
# ----------------------------------------------------------------------
def read(self):
self._setup_logger()
self._setup_http_handler()
self._setup_timezone()
self._setup_initial_time_range()
self._prompt_for_kibana_saved_search_selection_if_necessary()
self._factor_query_builder()
self._build_base_query()
self._print_header()
while True:
try:
self._fetch_latest_documents()
self._fetch_latest_timestamp()
self._print_latest_documents()
self._stop_reader_loop_if_necessary()
self._wait_for_next_refresh_interval()
except (StopReaderLoop, KeyboardInterrupt):
return
except Exception as exc: # pylint: disable=broad-except
if self._config.debug:
traceback = format_exc()
traceback = '\n{}'.format(traceback)
else:
traceback = ''
self._logger.error('Unexpected error occurred: {}{}', exc,
traceback)
self._wait_for_next_refresh_interval()
# ----------------------------------------------------------------------
def _setup_logger(self):
self._logger = LstailLogger(config=self._config,
output=self._output,
verbose=self._config.verbose)
# ----------------------------------------------------------------------
def _setup_http_handler(self):
self._http_handler = ElasticsearchRequestController(
self._config.servers,
timeout=self._config.timeout,
verify_ssl_certificates=self._config.verify_ssl_certificates,
debug=self._config.debug,
logger=self._logger)
# ----------------------------------------------------------------------
def _setup_timezone(self):
environ['TZ'] = 'UTC'
tzset()
# ----------------------------------------------------------------------
def _setup_initial_time_range(self):
if not self._config.initial_time_range:
self._config.initial_time_range = '1d' # fallback to one day
self._last_timestamp = parse_and_convert_time_range_to_start_date_time(
self._config.initial_time_range)
# ----------------------------------------------------------------------
def _prompt_for_kibana_saved_search_selection_if_necessary(self):
if self._config.kibana.saved_search == '-' or self._config.select_kibana_saved_search:
self._prompt_for_kibana_saved_search_selection()
# ----------------------------------------------------------------------
def _prompt_for_kibana_saved_search_selection(self):
saved_searches = self._get_kibana_saved_searches()
kibana_saved_search_select_prompt = KibanaSavedSearchSelectPrompt(
saved_searches)
selected_saved_search = kibana_saved_search_select_prompt.prompt()
# overwrite previously set saved search title
self._config.kibana.saved_search = selected_saved_search
# ----------------------------------------------------------------------
def _factor_query_builder(self):
query_builder_factory = QueryBuilderFactory(self._http_handler,
self._logger)
self._query_builder = query_builder_factory.factor(
self._config.default_index, self._config.kibana.index_name,
self._config.kibana.saved_search,
self._config.kibana.custom_search, self._http_handler,
self._logger)
# ----------------------------------------------------------------------
def _build_base_query(self):
self._base_query = self._query_builder.build()
# ----------------------------------------------------------------------
def _print_header(self):
self._logger.print_header()
# ----------------------------------------------------------------------
def _fetch_latest_documents(self):
path = '%s/_search' % self._base_query.index
timestamp_from = self._last_timestamp.strftime(
ELASTICSEARCH_TIMESTAMP_FORMAT)
query = self._query_builder.build_query_for_time_range(
self._base_query, timestamp_from)
query.query['size'] = self._config.initial_query_size
query_json = dumps(query.query)
response = self._http_handler.request(path, query_json)
self._documents = response['hits']['hits']
self._documents.reverse()
# ----------------------------------------------------------------------
def _fetch_latest_timestamp(self):
if self._documents:
latest_document = self._documents[-1]
timestamp = latest_document['_source'][
self._base_query.time_field_name]
last_timestamp = parse_timestamp_from_elasticsearch(timestamp)
self._last_timestamp = min(last_timestamp, datetime.now())
# ----------------------------------------------------------------------
def _print_latest_documents(self):
for document in self._documents:
self._logger.log_document(document)
# ----------------------------------------------------------------------
def _stop_reader_loop_if_necessary(self):
if not self._config.follow:
# raise a dedicated exception to break the while(true) loop in self.read()
# if "--follow" CLI option was not specified
raise StopReaderLoop()
# ----------------------------------------------------------------------
def _wait_for_next_refresh_interval(self):
sleep(self._config.refresh_interval)