def test_get_indices(self):
# Not reindexing.
assert not Reindexing.objects.filter(alias='foo').exists()
assert Reindexing.get_indices('foo') == ['foo']
# Reindexing on 'foo'.
Reindexing.objects.create(alias='foo', new_index='bar',
old_index='baz')
self.assertSetEqual(Reindexing.get_indices('foo'), ['bar', 'baz'])
# Doesn't clash on other aliases.
self.assertSetEqual(Reindexing.get_indices('other'), ['other'])
def unindexer(cls, ids=None, _all=False, index=None):
"""
Empties an index, but doesn't delete it. Useful for tearDowns.
ids -- list of IDs to unindex.
_all -- unindex all objects.
"""
if _all:
# Mostly used for test tearDowns.
qs = cls.get_model()
if hasattr(qs, 'with_deleted'):
qs = qs.with_deleted
else:
qs = qs.objects
ids = list(qs.order_by('id').values_list('id', flat=True))
if not ids:
return
log.info('Unindexing %s %s-%s. [%s]' %
(cls.get_model()._meta.model_name, ids[0], ids[-1], len(ids)))
index = index or cls.get_index()
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = Reindexing.get_indices(index)
es = cls.get_es(urls=settings.ES_URLS)
for id_ in ids:
for idx in indices:
try:
cls.unindex(id_=id_, es=es, index=idx)
except elasticsearch.exceptions.NotFoundError:
# Ignore if it's not there.
log.info(u'[%s:%s] object not found in index' %
(cls.get_model()._meta.model_name, id_))
def index(ids, indexer, **kw):
"""
Given a list of IDs and an indexer, index into ES.
If an reindexation is currently occurring, index on both the old and new.
"""
task_log.info("Indexing {0} {1}-{2}. [{3}]".format(indexer.get_model()._meta.model_name, ids[0], ids[-1], len(ids)))
# If reindexing is currently occurring, index on both old and new indexes.
indices = Reindexing.get_indices(indexer.get_index())
es = indexer.get_es(urls=settings.ES_URLS)
for obj in indexer.get_indexable().filter(id__in=ids):
doc = indexer.extract_document(obj.id, obj)
for idx in indices:
indexer.index(doc, id_=obj.id, es=es, index=idx)
def index_webapps(ids, **kw):
"""TODO: use search/indexers.py:index."""
task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))
index = kw.pop('index', WebappIndexer.get_index())
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = Reindexing.get_indices(index)
es = WebappIndexer.get_es(urls=settings.ES_URLS)
qs = Webapp.indexing_transformer(Webapp.with_deleted.no_cache().filter(
id__in=ids))
for obj in qs:
doc = WebappIndexer.extract_document(obj.id, obj)
for idx in indices:
WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)
def index(ids, indexer, **kw):
"""
Given a list of IDs and an indexer, index into ES.
If an reindexation is currently occurring, index on both the old and new.
"""
log.info('Indexing {0} {1}-{2}. [{3}]'.format(
indexer.get_model()._meta.model_name, ids[0], ids[-1], len(ids)))
# If reindexing is currently occurring, index on both old and new indexes.
indices = Reindexing.get_indices(indexer.get_index())
es = indexer.get_es(urls=settings.ES_URLS)
for obj in indexer.get_indexable().filter(id__in=ids):
doc = indexer.extract_document(obj.id, obj)
for idx in indices:
indexer.index(doc, id_=obj.id, es=es, index=idx)
def unindex_webapps(ids, **kw):
if not ids:
return
task_log.info('Un-indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))
index = kw.pop('index', WebappIndexer.get_index())
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = Reindexing.get_indices(index)
es = WebappIndexer.get_es(urls=settings.ES_URLS)
for id_ in ids:
for idx in indices:
try:
WebappIndexer.unindex(id_=id_, es=es, index=idx)
except ElasticHttpNotFoundError:
# Ignore if it's not there.
task_log.info(
u'[Webapp:%s] Unindexing app but not found in index' % id_)
def unindexer(cls, ids=None, _all=False, index=None):
"""
Empties an index, but doesn't delete it. Useful for tearDowns.
ids -- list of IDs to unindex.
_all -- unindex all objects.
"""
if _all:
# Mostly used for test tearDowns.
qs = cls.get_model()
if hasattr(qs, 'with_deleted'):
qs = qs.with_deleted
else:
qs = qs.objects
ids = list(qs.order_by('id').values_list('id', flat=True))
if not ids:
return
task_log.info('Unindexing %s %s-%s. [%s]' %
(cls.get_model()._meta.model_name, ids[0], ids[-1],
len(ids)))
index = index or cls.get_index()
# Note: If reindexing is currently occurring, `get_indices` will return
# more than one index.
indices = Reindexing.get_indices(index)
es = cls.get_es(urls=settings.ES_URLS)
for id_ in ids:
for idx in indices:
try:
cls.unindex(id_=id_, es=es, index=idx)
except elasticsearch.exceptions.NotFoundError:
# Ignore if it's not there.
task_log.info(u'[%s:%s] object not found in index' %
(cls.get_model()._meta.model_name, id_))
