def test_empty(self, es_client):
indexer = indexing.ESIndexer(es_client, 'share_v1')
indexer.index()
indexer = indexing.ESIndexer(es_client, 'share_v1',
self.message_factory(ids=[]),
self.message_factory('Agent', ids=[]))
indexer.index()
def test_indexes(self, elastic):
messages = [
self.message_factory(ids=[
x.id
for x in factories.AbstractCreativeWorkFactory.create_batch(5)
]),
self.message_factory(ids=[
x.id
for x in factories.AbstractCreativeWorkFactory.create_batch(3)
]),
self.message_factory(ids=[
x.id
for x in factories.AbstractCreativeWorkFactory.create_batch(8)
]),
]
indexer = indexing.ESIndexer(elastic.es_client,
settings.ELASTICSEARCH_INDEX, *messages)
indexer.index()
elastic.es_client.indices.refresh(elastic.es_index)
assert elastic.es_client.count(index=elastic.es_index,
doc_type='creativeworks')['count'] == 16
assert set(doc['_id'] for doc in elastic.es_client.search(
doc_type='creativeworks',
index=elastic.es_index,
_source=['_id'],
size=20)['hits']['hits']) == set(
util.IDObfuscator.encode(work)
for work in models.AbstractCreativeWork.objects.all())
def test_gentle_mode(self, es_client):
es_client.cluster.health.return_value = {'status': 'yellow'}
indexer = indexing.ESIndexer(es_client, settings.ELASTICSEARCH_INDEX,
self.message_factory())
indexer.bulk_stream = mock.MagicMock()
indexer._index()
assert indexer.bulk_stream.assert_called_once_with(
models.CreativeWork, mock.ANY, mock.ANY, gentle=True) is None
def test_red_fails(self, es_client):
es_client.cluster.health.return_value = {'status': 'red'}
indexer = indexing.ESIndexer(es_client, settings.ELASTICSEARCH_INDEX)
with pytest.raises(ValueError) as e:
indexer._index()
assert e.value.args == (
'ES cluster health is red, Refusing to index', )
def test_retries(self, sleep, es_client):
indexer = indexing.ESIndexer(es_client, settings.ELASTICSEARCH_INDEX)
indexer._index = mock.Mock()
indexer._index.side_effect = Exception('Testing')
with pytest.raises(SystemExit):
indexer.index()
assert len(indexer._index.call_args_list) == indexer.MAX_RETRIES
assert sleep.call_args_list == [
mock.call(2**(i + 1)) for i in range(indexer.MAX_RETRIES - 1)
]
def test_acks(self, elastic):
messages = [
self.message_factory(ids=[
x.id
for x in factories.AbstractCreativeWorkFactory.create_batch(6)
]),
self.message_factory(ids=[
x.id
for x in factories.AbstractCreativeWorkFactory.create_batch(4)
]),
self.message_factory(ids=[
x.id
for x in factories.AbstractCreativeWorkFactory.create_batch(8)
]),
]
indexer = indexing.ESIndexer(elastic.es_client,
settings.ELASTICSEARCH_INDEX, *messages)
indexer.index()
for message in messages:
assert message.ack.called is True
def test_checks_health(self, es_client):
indexer = indexing.ESIndexer(es_client, settings.ELASTICSEARCH_INDEX)
indexer.index()
assert es_client.cluster.health.called is True