def get_all_indexed_models(): backends = haystack_connections.connections_info.keys() available_models = {} for backend_key in backends: unified_index = haystack_connections[backend_key].get_unified_index() for app in load_apps(): for model in get_models(app): try: unified_index.get_index(model) except NotHandled: continue model_name = model.__module__ + '.' + model.__name__ available_models[model_name] = { 'backend_key': backend_key, 'app': app, 'model': model, } return available_models
def update_backend(self, label, using): from haystack.exceptions import NotHandled backend = haystack_connections[using].get_backend() unified_index = haystack_connections[using].get_unified_index() if self.workers > 0: import multiprocessing for model in get_models(label): try: index = unified_index.get_index(model) except NotHandled: if self.verbosity >= 2: print("Skipping '%s' - no index." % model) continue if self.workers > 0: # workers resetting connections leads to references to models / connections getting # stale and having their connection disconnected from under them. Resetting before # the loop continues and it accesses the ORM makes it better. db.close_connection() qs = index.build_queryset(using=using, start_date=self.start_date, end_date=self.end_date) total = qs.count() if self.verbosity >= 1: print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural))) batch_size = self.batchsize or backend.batch_size if self.workers > 0: ghetto_queue = [] for start in range(0, total, batch_size): end = min(start + batch_size, total) if self.workers == 0: do_update(backend, index, qs, start, end, total, self.verbosity) else: ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate() if self.remove: if self.start_date or self.end_date or total <= 0: # They're using a reduced set, which may not incorporate # all pks. Rebuild the list with everything. qs = index.index_queryset().values_list('pk', flat=True) pks_seen = set(smart_bytes(pk) for pk in qs) total = len(pks_seen) else: pks_seen = set(smart_bytes(pk) for pk in qs.values_list('pk', flat=True)) #added total = SearchQuerySet(using=backend.connection_alias).models(model).count() if self.workers > 0: ghetto_queue = [] list_to_delete = [] for start in range(0, total, batch_size): upper_bound = start + batch_size if self.workers == 0: to_delete = do_remove(backend, index, model, pks_seen, start, upper_bound) list_to_delete.extend(to_delete) else: ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity)) for result in list_to_delete: backend.remove(".".join([result[0], result[1], result[2]])) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate()
def update_backend(self, label, using): from haystack.exceptions import NotHandled backend = haystack_connections[using].get_backend() unified_index = haystack_connections[using].get_unified_index() if self.workers > 0: import multiprocessing for model in get_models(label): try: index = unified_index.get_index(model) except NotHandled: if self.verbosity >= 2: print("Skipping '%s' - no index." % model) continue if self.workers > 0: # workers resetting connections leads to references to models / connections getting # stale and having their connection disconnected from under them. Resetting before # the loop continues and it accesses the ORM makes it better. db.close_connection() qs = index.build_queryset(using=using, start_date=self.start_date, end_date=self.end_date) total = qs.count() if self.verbosity >= 1: print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural))) batch_size = self.batchsize or backend.batch_size if self.workers > 0: # multi-worker, many-queries ghetto_queue = [] for start in range(0, total, batch_size): end = min(start + batch_size, total) ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity)) pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate() else: # single-query, still-batched start = 0 with server_side_cursors(): with transaction.atomic(): items = qs.iterator() # prevents filling query-cache while True: added = do_update_batch(backend, index, items, start, batch_size, total, self.verbosity) if added > 0: start += added continue break if self.remove: if self.start_date or self.end_date or total <= 0: # They're using a reduced set, which may not incorporate # all pks. Rebuild the list with everything. qs = index.index_queryset().values_list('pk', flat=True) pks_seen = set(smart_bytes(pk) for pk in qs) total = len(pks_seen) else: pks_seen = set(smart_bytes(pk) for pk in qs.values_list('pk', flat=True)) if self.workers > 0: ghetto_queue = [] for start in range(0, total, batch_size): upper_bound = start + batch_size if self.workers == 0: do_remove(backend, index, model, pks_seen, start, upper_bound) else: ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate()
def update_backend(self, label, using): from haystack.exceptions import NotHandled backend = haystack_connections[using].get_backend() unified_index = haystack_connections[using].get_unified_index() if self.workers > 0: import multiprocessing for model in get_models(label): try: index = unified_index.get_index(model) except NotHandled: if self.verbosity >= 2: print("Skipping '%s' - no index." % model) continue if self.workers > 0: # workers resetting connections leads to references to models / connections getting # stale and having their connection disconnected from under them. Resetting before # the loop continues and it accesses the ORM makes it better. db.close_connection() qs = index.build_queryset(using=using, start_date=self.start_date, end_date=self.end_date) total = qs.count() if self.verbosity >= 1: print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural))) batch_size = self.batchsize or backend.batch_size if self.workers > 0: ghetto_queue = [] for start in range(0, total, batch_size): end = min(start + batch_size, total) if self.workers == 0: do_update(backend, index, qs, start, end, total, self.verbosity) else: ghetto_queue.append( ('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate() if self.remove: if self.start_date or self.end_date or total <= 0: # They're using a reduced set, which may not incorporate # all pks. Rebuild the list with everything. qs = index.index_queryset().values_list('pk', flat=True) pks_seen = set(smart_bytes(pk) for pk in qs) total = len(pks_seen) else: pks_seen = set( smart_bytes(pk) for pk in qs.values_list('pk', flat=True)) if self.workers > 0: ghetto_queue = [] for start in range(0, total, batch_size): upper_bound = start + batch_size if self.workers == 0: do_remove(backend, index, model, pks_seen, start, upper_bound) else: ghetto_queue.append( ('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity)) if self.workers > 0: pool = multiprocessing.Pool(self.workers) pool.map(worker, ghetto_queue) pool.terminate()
def test_get_models_specific(self): from test_haystack.core.models import MockModel models = app_loading.get_models('core.MockModel') self.assertIsInstance(models, (list, GeneratorType)) self.assertListEqual(models, [MockModel])
def test_get_models_all(self): models = app_loading.get_models('core') self.assertIsInstance(models, (list, GeneratorType))