def test_wait_for_green_retry_test(self, mocked_sleep):
"""Test that the retry is called again when cluster health request throws an exception."""
self.elasticsearch1.cluster.health = mock.Mock(
side_effect=TransportError(500, "test"))
self.elasticsearch2.cluster.health = mock.Mock(
side_effect=TransportError(500, "test"))
elasticsearch_clusters = self.default_elasticsearch_clusters()
with pytest.raises(ec.ElasticsearchClusterCheckError):
elasticsearch_clusters.wait_for_green(timedelta(seconds=20))
assert mocked_sleep.called
assert self.elasticsearch1.cluster.health.call_count == 2
assert self.elasticsearch2.cluster.health.call_count == 2
def test_rejected_documents_are_retried_at_most_max_retries_times(self):
failing_client = FailingBulkClient(
self.client, fail_at=(1, 2), fail_with=TransportError(429, "Rejected!", {})
)
docs = [
{"_index": "i", "_id": 47, "f": "v"},
{"_index": "i", "_id": 45, "f": "v"},
{"_index": "i", "_id": 42, "f": "v"},
]
results = list(
helpers.streaming_bulk(
failing_client,
docs,
raise_on_exception=False,
raise_on_error=False,
chunk_size=1,
max_retries=1,
initial_backoff=0,
)
)
self.assertEqual(3, len(results))
self.assertEqual([False, True, True], [r[0] for r in results])
self.client.indices.refresh(index="i")
res = self.client.search(index="i")
self.assertEqual({"value": 2, "relation": "eq"}, res["hits"]["total"])
self.assertEqual(4, failing_client._called)
def reject_first_3_batches(*_args, **_kwargs):
"""Reject the first 3 batches that are indexed, and then accept the rest."""
if self._rejection_counter < 3:
self._rejection_counter += 1
raise TransportError(429, 'Rejected bulk request', 'Queue is full')
else:
return self.get_bulk_api_response(1)
def test_transport_error_is_raised_with_max_retries(self):
failing_client = FailingBulkClient(self.client,
fail_at=(
1,
2,
3,
4,
),
fail_with=TransportError(
429, 'Rejected!', {}))
def streaming_bulk():
results = list(
helpers.streaming_bulk(failing_client, [{
"a": 42
}, {
"a": 39
}],
raise_on_exception=True,
max_retries=3,
initial_backoff=0))
return results
self.assertRaises(TransportError, streaming_bulk)
self.assertEquals(4, failing_client._called)
def sniff_hosts(self, initial=False):
"""
Obtain a list of nodes from the cluster and create a new connection
pool using the information retrieved.
To extract the node connection parameters use the ``nodes_to_host_callback``.
:arg initial: flag indicating if this is during startup
(``sniff_on_start``), ignore the ``sniff_timeout`` if ``True``
"""
node_info = yield from self._get_sniff_data(initial)
hosts = list(filter(None, (self._get_host_info(n) for n in node_info)))
# we weren't able to get any nodes, maybe using an incompatible
# transport_schema or host_info_callback blocked all - raise error.
if not hosts:
raise TransportError(
"N/A", "Unable to sniff hosts - no viable hosts found.")
# remember current live connections
orig_connections = self.connection_pool.connections[:]
self.set_connections(hosts)
# close those connections that are not in use any more
for c in orig_connections:
if c not in self.connection_pool.connections:
yield from c.close()
async def execute(self, ignore_cache=False, raise_on_error=True):
"""
Execute the multi search request and return a list of search results.
"""
if ignore_cache or not hasattr(self, '_response'):
es = get_connection(self._using)
responses = await es.msearch(index=self._index,
body=self.to_dict(),
**self._params)
out = []
for s, r in zip(self._searches, responses['responses']):
if r.get('error', False):
if raise_on_error:
raise TransportError('N/A', r['error']['type'],
r['error'])
r = None
else:
r = Response(s, r)
out.append(r)
self._response = out
return self._response
def test_ready_es_failure(self):
""" Verify Elasticsearch errors are raised if the app fails to create the index. """
with mock.patch.object(IndicesClient, 'create') as mock_create:
mock_create.side_effect = TransportError(500)
with self.assertRaises(TransportError):
call_command('install_es_indexes')
def test_ready_index_exists(self):
""" Verify the app does not setup a new Elasticsearch index if one exists already. """
host = settings.ELASTICSEARCH['host']
index = settings.ELASTICSEARCH['index']
# Verify the index exists
self.assertTrue(self.es.indices.exists(index=index))
with mock.patch.object(IndicesClient, 'create') as mock_create:
mock_create.side_effect = TransportError(400)
with LogCapture(LOGGER_NAME) as l:
# This call should NOT raise an exception.
call_command('install_es_indexes')
# Verify the index still exists
self.assertTrue(self.es.indices.exists(index=index))
l.check(
(LOGGER_NAME, 'INFO',
'Attempting to establish initial connection to Elasticsearch host [{}]...'.format(host)),
(LOGGER_NAME, 'INFO', '...success!'),
(LOGGER_NAME, 'INFO', 'Making sure index [{}] exists...'.format(index)),
(LOGGER_NAME, 'INFO', '...index already exists.')
)
def test_search__transport_error_without_status_code(self, mock_essearch):
mock_essearch.side_effect = TransportError('N/A', "Error")
url = reverse('complaint_search:search')
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual({"error": "Elasticsearch error: Error"},
response.data)
async def test_transport_error_is_raised_with_max_retries(
self, async_client):
failing_client = FailingBulkClient(
async_client,
fail_at=(1, 2, 3, 4),
fail_with=TransportError(429, "Rejected!", {}),
)
async def streaming_bulk():
results = [
x async for x in helpers.async_streaming_bulk(
failing_client,
[{
"a": 42
}, {
"a": 39
}],
raise_on_exception=True,
max_retries=3,
initial_backoff=0,
)
]
return results
with pytest.raises(TransportError):
await streaming_bulk()
assert 4 == failing_client._called
def listing(cls, system, path, offset, limit):
list_search = PublicSearchManager(PublicObject,
PublicObjectIndexed.search(),
page_size=limit)
base_path, name = os.path.split(path.strip('/'))
base_path = base_path or '/'
search = PublicObjectIndexed.search()
query = Q('bool',
must=[Q({'term': {'name._exact': name}}),
Q({'term': {'path._exact': base_path}}),
Q({'term': {'systemId': system}})])
search.query = query
res = search.execute()
if res.hits.total:
listing = cls(doc=res[0])
list_path = path.strip('/')
elif not path or path == '/':
listing = cls(PublicObjectIndexed(systemId=PublicElasticFileManager.DEFAULT_SYSTEM_ID,
path='/',
name=''))
listing.system = system
list_path = '/'
else:
raise TransportError()
list_search._search.query = Q('bool',
must=[Q({'term': {'path._exact': list_path}}),
Q({'term': {'systemId': system}})])
list_search.sort({'project._exact': 'asc'})
listing.children = list_search.results(offset)
return listing
async def test_rejected_documents_are_retried_at_most_max_retries_times(
self, async_client
):
failing_client = FailingBulkClient(
async_client, fail_at=(1, 2), fail_with=TransportError(429, "Rejected!", {})
)
docs = [
{"_index": "i", "_id": 47, "f": "v"},
{"_index": "i", "_id": 45, "f": "v"},
{"_index": "i", "_id": 42, "f": "v"},
]
results = [
x
async for x in helpers.async_streaming_bulk(
failing_client,
docs,
raise_on_exception=False,
raise_on_error=False,
chunk_size=1,
max_retries=1,
initial_backoff=0,
)
]
assert 3 == len(results)
assert [False, True, True] == [r[0] for r in results]
await async_client.indices.refresh(index="i")
res = await async_client.search(index="i")
assert {"value": 2, "relation": "eq"} == res["hits"]["total"]
assert 4 == failing_client._called
def test_search__transport_error(self, mock_essearch):
mock_essearch.side_effect = TransportError('N/A', "Error")
url = reverse('complaint_search:search')
response = self.client.get(url)
self.assertEqual(response.status_code, 424)
self.assertDictEqual(
{"error": "There was an error calling Elasticsearch"},
response.data)
def test_document__transport_error(self, mock_esdocument):
mock_esdocument.side_effect = TransportError('N/A', "Error")
url = reverse('complaint_search:complaint', kwargs={"id": "123456"})
response = self.client.get(url)
self.assertEqual(response.status_code, 424)
self.assertDictEqual(
{"error": "There was an error calling Elasticsearch"},
response.data)
def __init__(
self, client, fail_at=(2,), fail_with=TransportError(599, "Error!", {})
):
self.client = client
self._called = 0
self._fail_at = fail_at
self.transport = client.transport
self._fail_with = fail_with
def test_correct_exception_is_logged_without_info_attribute(self, es_mock):
es_mock.search.side_effect = TransportError(500, 'NewConnectionError', None)
response = self.client.get('/test-index/services/search')
data = json.loads(response.get_data())
assert response.status_code == 500
assert data['error'] is None
def test_document__transport_error_with_status_code(self, mock_esdocument):
mock_esdocument.side_effect = TransportError(status.HTTP_404_NOT_FOUND,
"Error")
url = reverse('complaint_search:document', kwargs={"id": "123456"})
response = self.client.get(url)
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
self.assertDictEqual({"error": "Elasticsearch error: Error"},
response.data)
def test_too_many_rejected_batches(self):
self.create_task(max_attempts=3)
self.mock_es.bulk.side_effect = TransportError(429, 'Rejected bulk request', 'Queue is full')
with self.assertRaisesRegexp(IndexingError, 'Batch of records rejected too many times. Aborting.'):
self._get_reducer_output(['a'])
self.assertEqual(len(self.mock_es.bulk.mock_calls), 3)
def test_suggest__transport_error(self, mock_essuggest):
mock_essuggest.side_effect = TransportError('N/A', "Error")
url = reverse('complaint_search:suggest_zip')
param = {"text": "test"}
response = self.client.get(url, param)
self.assertEqual(response.status_code, 424)
self.assertDictEqual(
{"error": "There was an error calling Elasticsearch"},
response.data)
def test_suggest__transport_error_without_status_code(
self, mock_essuggest):
mock_essuggest.side_effect = TransportError('N/A', "Error")
url = reverse('complaint_search:suggest_company')
param = {"text": "test"}
response = self.client.get(url, param)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertDictEqual({"error": "Elasticsearch error: Error"},
response.data)
def test_status_when_elasticsearch_is_down(self, indices):
with self.app.app_context():
indices.status.side_effect = TransportError(500, "BARR", "FOO")
response = self.client.get('/_status')
assert_equal(response.status_code, 500)
data = json.loads(response.data.decode('utf-8'))
assert_equal(data['es_status']['message'], "FOO")
def test_fallback_list_view(self):
with mock.patch('django_elasticsearch.query.EsQueryset.do_search'
) as mock_search:
mock_search.side_effect = TransportError()
response = self.client.get('/tests/')
content = json.loads(response.content.decode('utf-8'))
self.assertEqual(len(content), 1)
self.assertEqual(content[0]['fields']['first_name'], u"woot")
self.assertEqual(content[0]['fields']['last_name'], u"foo")
def test_constructor_with_existing_index(self, mocked_elastic):
elastic_client = mock.MagicMock()
elastic_client.indices.create.side_effect = [
TransportError(234234, 'index_already_exists_exception', 'bla')
]
mocked_elastic.return_value = elastic_client
ElasticSearchDriver(index, doc_type)
elastic_client.indices.create.assert_called_once_with(index=index)
def test_reset_read_only_wraps_exceptions(self):
"""Exceptions from underlying elasticsearch client should be wrapped."""
elasticsearch_clusters = self.default_elasticsearch_clusters()
for client in [self.elasticsearch1, self.elasticsearch2]:
client.indices.put_settings = mock.Mock(
side_effect=TransportError(500, "test"))
with pytest.raises(ec.ElasticsearchClusterError):
elasticsearch_clusters.reset_indices_to_read_write()
assert not self.elasticsearch2.indices.put_settings.called
def test_get_nodes_wraps_exceptions():
"""Get nodes should wrap exceptions from elasticsearch client."""
elasticsearch = mock.Mock()
elasticsearch.nodes.info = mock.Mock(
side_effect=TransportError(500, "test"))
remote = mock.Mock()
cluster = ec.ElasticsearchCluster(elasticsearch, remote)
with pytest.raises(ec.ElasticsearchClusterError):
cluster.get_nodes()
def test_status_when_elasticsearch_is_down(self, indices):
with self.app.app_context():
indices.stats.side_effect = TransportError(500, "BARR", "FOO")
response = self.client.get('/_status')
assert response.status_code == 500
data = json.loads(response.data.decode('utf-8'))
assert data['message'] == [
'Error connecting to elasticsearch (status_code: 500, message: FOO)'
]
def test_fallback_list_view(self):
# Note: in this case views responses don't match because i'm lazy
with mock.patch(
'django_elasticsearch.client.es_client.search') as mock_search:
mock_search.side_effect = TransportError()
response = self.client.get('/tests/')
content = json.loads(response.content)
self.assertEqual(len(content), 1)
self.assertEqual(content[0]['fields']['first_name'], u"woot")
self.assertEqual(content[0]['fields']['last_name'], u"foo")
def test_failing_index_creation(self, mocked_elastic):
elastic_client = mock.MagicMock()
elastic_client.indices.create.side_effect = [
TransportError(500, 'unknown', 'bla')
]
mocked_elastic.return_value = elastic_client
with pytest.raises(TransportError):
ElasticSearchDriver(index, doc_type)
elastic_client.indices.create.assert_called_once_with(index=index)
def test_macro_usage_2nd_es_exception(mock_sources, mock_page_count):
"""When follow-on ElasticSearch call raises, reraise exception."""
mock_sources.return_value = {
'A11yRoleQuicklinks': 'A11yRoleQuicklinks.ejs'
}
mock_page_count.side_effect = [
{'a11yrolequicklinks': 200, 'othermacro': 50},
TransportError("Can't reach ElasticSearch")
]
with pytest.raises(TransportError):
kumascript.macro_usage()
def test_fallback_gracefully(self):
# Note: can't use override settings because of how restframework handle settings :(
# from django_elasticsearch.tests.urls import TestViewSet
from rest_framework.filters import DjangoFilterBackend, OrderingFilter
from rest_framework.settings import api_settings
api_settings.DEFAULT_FILTER_BACKENDS = (DjangoFilterBackend, OrderingFilter)
# TODO: better way to fake es cluster's death ?
with mock.patch.object(es_client, 'search') as mock_search:
mock_search.side_effect = TransportError()
with mock.patch.object(es_client, 'count') as mock_count:
mock_count.side_effect = TransportError()
with mock.patch.object(es_client, 'get') as mock_get:
mock_get.side_effect = TransportError()
# should fallback to a regular django queryset / filtering
r = self.client.get('/rf/tests/')
self.assertEqual(r.status_code, 200)
self.assertEqual(r.data['filter_status'], 'Failed')
self.assertEqual(r.data['count'], 3)
self._test_filter_backend_filters()
self._test_pagination()