def test_do_update(self, mock_obj): """ Tests the update_index.do_update function """ engine = ElasticsearchMultilingualSearchEngine() backend = mock.Mock() engine._backend = backend index = engine.get_unified_index() qs = Document.objects.all() start = 0 end = len(qs) total = end - start do_update(backend, index, qs, start, end, total, verbosity=1, commit=False) # The update method has been called self.assertTrue(backend.update.called) call_args_list = backend.update.call_args_list # args, the queryset cannot be testet for equality. self.assertEqual(call_args_list[0][0][0], index) # kwargs self.assertEqual(call_args_list[0][1], {'commit': False})
def handle_app(self, app, **options): from django.db.models import get_models from haystack.exceptions import NotRegistered site = get_site(self.site) for model in get_models(app): try: index = site.get_index(model) # Manually set the ``site`` on the backend to the correct one. index.backend.site = site except NotRegistered: if self.verbosity >= 2: print "Skipping '%s' - no index." % model continue qs = build_queryset(index, model, age=self.age, verbosity=self.verbosity) total = qs.count() if self.limit == 0 and self.offset == 0: pass else: if self.limit == 0: qs = qs[self.offset:] total = total - self.offset else: qs = qs[self.offset:(self.offset + self.limit)] total = self.limit if self.verbosity >= 1: print "Indexing %d %s." % ( total, smart_str(model._meta.verbose_name_plural)) pks_seen = set( [smart_str(pk) for pk in qs.values_list('pk', flat=True)]) for start in range(0, total, self.batchsize): end = min(start + self.batchsize, total) do_update(index, qs, start, end, total, self.verbosity)
def test_do_update(self, mock_obj): """ Tests the update_index.do_update function """ engine = ElasticsearchMultilingualSearchEngine() backend = mock.Mock() engine._backend = backend index = engine.get_unified_index() qs = Document.objects.all() start = 0 end = len(qs) total = end - start do_update(backend, index, qs, start, end, total, verbosity=1, commit=False) # The update method has been called self.assertTrue(backend.update.called) call_args_list = backend.update.call_args_list # args, the queryset cannot be testet for equality. self.assertEqual(call_args_list[0][0][0], index) # kwargs self.assertEqual(call_args_list[0][1], {"commit": False})
def update_backend(self, label, using): # shamelessly copy pasted from haystack.management.commands.update_index.Command backend = haystack_connections[using].get_backend() if self.workers > 0: import multiprocessing for model in self.get_models(label): try: index = self.get_index(using, model) except NotHandled: if self.verbosity >= 2: print("Skipping '%s' - no index." % model) continue if self.workers > 0: # workers resetting connections leads to references to models / connections getting # stale and having their connection disconnected from under them. Resetting before # the loop continues and it accesses the ORM makes it better. db.close_connection() qs = self.build_queryset(index, using) total = qs.count() if self.verbosity >= 1: print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural))) pks_seen = set([smart_bytes(pk) for pk in qs.values_list('pk', flat=True)]) batch_size = self.batchsize or backend.batch_size if self.workers > 0: ghetto_queue = [] for start in range(0, total, batch_size): end = min(start + batch_size, total) if self.workers == 0: do_update(backend, index, qs, start, end, total, self.verbosity) else: ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate() if self.remove: if self.start_date or self.end_date or total <= 0: # They're using a reduced set, which may not incorporate # all pks. Rebuild the list with everything. qs = index.index_queryset().values_list('pk', flat=True) pks_seen = set([smart_bytes(pk) for pk in qs]) total = len(pks_seen) if self.workers > 0: ghetto_queue = [] for start in range(0, total, batch_size): upper_bound = start + batch_size if self.workers == 0: do_remove(backend, index, model, pks_seen, start, upper_bound) else: ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate()