def test_get_indices(self): # Not reindexing. assert not Reindexing.objects.filter(alias='foo').exists() assert Reindexing.get_indices('foo') == ['foo'] # Reindexing on 'foo'. Reindexing.objects.create(alias='foo', new_index='bar', old_index='baz') self.assertSetEqual(Reindexing.get_indices('foo'), ['bar', 'baz']) # Doesn't clash on other aliases. self.assertSetEqual(Reindexing.get_indices('other'), ['other'])
def unindexer(cls, ids=None, _all=False, index=None): """ Empties an index, but doesn't delete it. Useful for tearDowns. ids -- list of IDs to unindex. _all -- unindex all objects. """ if _all: # Mostly used for test tearDowns. qs = cls.get_model() if hasattr(qs, 'with_deleted'): qs = qs.with_deleted else: qs = qs.objects ids = list(qs.order_by('id').values_list('id', flat=True)) if not ids: return log.info('Unindexing %s %s-%s. [%s]' % (cls.get_model()._meta.model_name, ids[0], ids[-1], len(ids))) index = index or cls.get_index() # Note: If reindexing is currently occurring, `get_indices` will return # more than one index. indices = Reindexing.get_indices(index) es = cls.get_es(urls=settings.ES_URLS) for id_ in ids: for idx in indices: try: cls.unindex(id_=id_, es=es, index=idx) except elasticsearch.exceptions.NotFoundError: # Ignore if it's not there. log.info(u'[%s:%s] object not found in index' % (cls.get_model()._meta.model_name, id_))
def index(ids, indexer, **kw): """ Given a list of IDs and an indexer, index into ES. If an reindexation is currently occurring, index on both the old and new. """ task_log.info("Indexing {0} {1}-{2}. [{3}]".format(indexer.get_model()._meta.model_name, ids[0], ids[-1], len(ids))) # If reindexing is currently occurring, index on both old and new indexes. indices = Reindexing.get_indices(indexer.get_index()) es = indexer.get_es(urls=settings.ES_URLS) for obj in indexer.get_indexable().filter(id__in=ids): doc = indexer.extract_document(obj.id, obj) for idx in indices: indexer.index(doc, id_=obj.id, es=es, index=idx)
def index_webapps(ids, **kw): """TODO: use search/indexers.py:index.""" task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids))) index = kw.pop('index', WebappIndexer.get_index()) # Note: If reindexing is currently occurring, `get_indices` will return # more than one index. indices = Reindexing.get_indices(index) es = WebappIndexer.get_es(urls=settings.ES_URLS) qs = Webapp.indexing_transformer(Webapp.with_deleted.no_cache().filter( id__in=ids)) for obj in qs: doc = WebappIndexer.extract_document(obj.id, obj) for idx in indices: WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)
def index(ids, indexer, **kw): """ Given a list of IDs and an indexer, index into ES. If an reindexation is currently occurring, index on both the old and new. """ log.info('Indexing {0} {1}-{2}. [{3}]'.format( indexer.get_model()._meta.model_name, ids[0], ids[-1], len(ids))) # If reindexing is currently occurring, index on both old and new indexes. indices = Reindexing.get_indices(indexer.get_index()) es = indexer.get_es(urls=settings.ES_URLS) for obj in indexer.get_indexable().filter(id__in=ids): doc = indexer.extract_document(obj.id, obj) for idx in indices: indexer.index(doc, id_=obj.id, es=es, index=idx)
def unindex_webapps(ids, **kw): if not ids: return task_log.info('Un-indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids))) index = kw.pop('index', WebappIndexer.get_index()) # Note: If reindexing is currently occurring, `get_indices` will return # more than one index. indices = Reindexing.get_indices(index) es = WebappIndexer.get_es(urls=settings.ES_URLS) for id_ in ids: for idx in indices: try: WebappIndexer.unindex(id_=id_, es=es, index=idx) except ElasticHttpNotFoundError: # Ignore if it's not there. task_log.info( u'[Webapp:%s] Unindexing app but not found in index' % id_)
def unindexer(cls, ids=None, _all=False, index=None): """ Empties an index, but doesn't delete it. Useful for tearDowns. ids -- list of IDs to unindex. _all -- unindex all objects. """ if _all: # Mostly used for test tearDowns. qs = cls.get_model() if hasattr(qs, 'with_deleted'): qs = qs.with_deleted else: qs = qs.objects ids = list(qs.order_by('id').values_list('id', flat=True)) if not ids: return task_log.info('Unindexing %s %s-%s. [%s]' % (cls.get_model()._meta.model_name, ids[0], ids[-1], len(ids))) index = index or cls.get_index() # Note: If reindexing is currently occurring, `get_indices` will return # more than one index. indices = Reindexing.get_indices(index) es = cls.get_es(urls=settings.ES_URLS) for id_ in ids: for idx in indices: try: cls.unindex(id_=id_, es=es, index=idx) except elasticsearch.exceptions.NotFoundError: # Ignore if it's not there. task_log.info(u'[%s:%s] object not found in index' % (cls.get_model()._meta.model_name, id_))