def update_backend(self, label, using): from haystack.exceptions import NotHandled backend = haystack_connections[using].get_backend() unified_index = haystack_connections[using].get_unified_index() if self.workers > 0: import multiprocessing for model in get_models(label): try: index = unified_index.get_index(model) except NotHandled: if self.verbosity >= 2: print("Skipping '%s' - no index." % model) continue if self.workers > 0: # workers resetting connections leads to references to models / connections getting # stale and having their connection disconnected from under them. Resetting before # the loop continues and it accesses the ORM makes it better. db.close_connection() qs = index.build_queryset(using=using, start_date=self.start_date, end_date=self.end_date) total = qs.count() if self.verbosity >= 1: print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural))) batch_size = self.batchsize or backend.batch_size if self.workers > 0: # multi-worker, many-queries ghetto_queue = [] for start in range(0, total, batch_size): end = min(start + batch_size, total) ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity)) pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate() else: # single-query, still-batched start = 0 with server_side_cursors(): with transaction.atomic(): items = qs.iterator() # prevents filling query-cache while True: added = do_update_batch(backend, index, items, start, batch_size, total, self.verbosity) if added > 0: start += added continue break if self.remove: if self.start_date or self.end_date or total <= 0: # They're using a reduced set, which may not incorporate # all pks. Rebuild the list with everything. qs = index.index_queryset().values_list('pk', flat=True) pks_seen = set(smart_bytes(pk) for pk in qs) total = len(pks_seen) else: pks_seen = set(smart_bytes(pk) for pk in qs.values_list('pk', flat=True)) if self.workers > 0: ghetto_queue = [] for start in range(0, total, batch_size): upper_bound = start + batch_size if self.workers == 0: do_remove(backend, index, model, pks_seen, start, upper_bound) else: ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate()
def test_simple_02(self): with self.assertNumQueries(1): with server_side_cursors(): self.assertEqual(len([x for x in TestModel.objects.all()]), 200)