def test_do_update(self, mock_obj):
"""
Tests the update_index.do_update function
"""
engine = ElasticsearchMultilingualSearchEngine()
backend = mock.Mock()
engine._backend = backend
index = engine.get_unified_index()
qs = Document.objects.all()
start = 0
end = len(qs)
total = end - start
do_update(backend,
index,
qs,
start,
end,
total,
verbosity=1,
commit=False)
# The update method has been called
self.assertTrue(backend.update.called)
call_args_list = backend.update.call_args_list
# args, the queryset cannot be testet for equality.
self.assertEqual(call_args_list[0][0][0], index)
# kwargs
self.assertEqual(call_args_list[0][1], {'commit': False})
def handle_app(self, app, **options):
from django.db.models import get_models
from haystack.exceptions import NotRegistered
site = get_site(self.site)
for model in get_models(app):
try:
index = site.get_index(model)
# Manually set the ``site`` on the backend to the correct one.
index.backend.site = site
except NotRegistered:
if self.verbosity >= 2:
print "Skipping '%s' - no index." % model
continue
qs = build_queryset(index,
model,
age=self.age,
verbosity=self.verbosity)
total = qs.count()
if self.limit == 0 and self.offset == 0:
pass
else:
if self.limit == 0:
qs = qs[self.offset:]
total = total - self.offset
else:
qs = qs[self.offset:(self.offset + self.limit)]
total = self.limit
if self.verbosity >= 1:
print "Indexing %d %s." % (
total, smart_str(model._meta.verbose_name_plural))
pks_seen = set(
[smart_str(pk) for pk in qs.values_list('pk', flat=True)])
for start in range(0, total, self.batchsize):
end = min(start + self.batchsize, total)
do_update(index, qs, start, end, total, self.verbosity)
def test_do_update(self, mock_obj):
"""
Tests the update_index.do_update function
"""
engine = ElasticsearchMultilingualSearchEngine()
backend = mock.Mock()
engine._backend = backend
index = engine.get_unified_index()
qs = Document.objects.all()
start = 0
end = len(qs)
total = end - start
do_update(backend, index, qs, start, end, total, verbosity=1, commit=False)
# The update method has been called
self.assertTrue(backend.update.called)
call_args_list = backend.update.call_args_list
# args, the queryset cannot be testet for equality.
self.assertEqual(call_args_list[0][0][0], index)
# kwargs
self.assertEqual(call_args_list[0][1], {"commit": False})
def update_backend(self, label, using):
# shamelessly copy pasted from haystack.management.commands.update_index.Command
backend = haystack_connections[using].get_backend()
if self.workers > 0:
import multiprocessing
for model in self.get_models(label):
try:
index = self.get_index(using, model)
except NotHandled:
if self.verbosity >= 2:
print("Skipping '%s' - no index." % model)
continue
if self.workers > 0:
# workers resetting connections leads to references to models / connections getting
# stale and having their connection disconnected from under them. Resetting before
# the loop continues and it accesses the ORM makes it better.
db.close_connection()
qs = self.build_queryset(index, using)
total = qs.count()
if self.verbosity >= 1:
print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural)))
pks_seen = set([smart_bytes(pk) for pk in qs.values_list('pk', flat=True)])
batch_size = self.batchsize or backend.batch_size
if self.workers > 0:
ghetto_queue = []
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
if self.workers == 0:
do_update(backend, index, qs, start, end, total, self.verbosity)
else:
ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
if self.remove:
if self.start_date or self.end_date or total <= 0:
# They're using a reduced set, which may not incorporate
# all pks. Rebuild the list with everything.
qs = index.index_queryset().values_list('pk', flat=True)
pks_seen = set([smart_bytes(pk) for pk in qs])
total = len(pks_seen)
if self.workers > 0:
ghetto_queue = []
for start in range(0, total, batch_size):
upper_bound = start + batch_size
if self.workers == 0:
do_remove(backend, index, model, pks_seen, start, upper_bound)
else:
ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
