def get_all_indexed_models():
backends = haystack_connections.connections_info.keys()
available_models = {}
for backend_key in backends:
unified_index = haystack_connections[backend_key].get_unified_index()
for app in load_apps():
for model in get_models(app):
try:
unified_index.get_index(model)
except NotHandled:
continue
model_name = model.__module__ + '.' + model.__name__
available_models[model_name] = {
'backend_key': backend_key,
'app': app,
'model': model,
}
return available_models
def update_backend(self, label, using):
from haystack.exceptions import NotHandled
backend = haystack_connections[using].get_backend()
unified_index = haystack_connections[using].get_unified_index()
if self.workers > 0:
import multiprocessing
for model in get_models(label):
try:
index = unified_index.get_index(model)
except NotHandled:
if self.verbosity >= 2:
print("Skipping '%s' - no index." % model)
continue
if self.workers > 0:
# workers resetting connections leads to references to models / connections getting
# stale and having their connection disconnected from under them. Resetting before
# the loop continues and it accesses the ORM makes it better.
db.close_connection()
qs = index.build_queryset(using=using, start_date=self.start_date,
end_date=self.end_date)
total = qs.count()
if self.verbosity >= 1:
print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural)))
batch_size = self.batchsize or backend.batch_size
if self.workers > 0:
ghetto_queue = []
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
if self.workers == 0:
do_update(backend, index, qs, start, end, total, self.verbosity)
else:
ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
if self.remove:
if self.start_date or self.end_date or total <= 0:
# They're using a reduced set, which may not incorporate
# all pks. Rebuild the list with everything.
qs = index.index_queryset().values_list('pk', flat=True)
pks_seen = set(smart_bytes(pk) for pk in qs)
total = len(pks_seen)
else:
pks_seen = set(smart_bytes(pk) for pk in qs.values_list('pk', flat=True))
#added
total = SearchQuerySet(using=backend.connection_alias).models(model).count()
if self.workers > 0:
ghetto_queue = []
list_to_delete = []
for start in range(0, total, batch_size):
upper_bound = start + batch_size
if self.workers == 0:
to_delete = do_remove(backend, index, model, pks_seen, start, upper_bound)
list_to_delete.extend(to_delete)
else:
ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity))
for result in list_to_delete:
backend.remove(".".join([result[0], result[1], result[2]]))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
def update_backend(self, label, using):
from haystack.exceptions import NotHandled
backend = haystack_connections[using].get_backend()
unified_index = haystack_connections[using].get_unified_index()
if self.workers > 0:
import multiprocessing
for model in get_models(label):
try:
index = unified_index.get_index(model)
except NotHandled:
if self.verbosity >= 2:
print("Skipping '%s' - no index." % model)
continue
if self.workers > 0:
# workers resetting connections leads to references to models / connections getting
# stale and having their connection disconnected from under them. Resetting before
# the loop continues and it accesses the ORM makes it better.
db.close_connection()
qs = index.build_queryset(using=using, start_date=self.start_date,
end_date=self.end_date)
total = qs.count()
if self.verbosity >= 1:
print(u"Indexing %d %s" % (total, force_text(model._meta.verbose_name_plural)))
batch_size = self.batchsize or backend.batch_size
if self.workers > 0:
# multi-worker, many-queries
ghetto_queue = []
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
ghetto_queue.append(('do_update', model, start, end, total, using, self.start_date, self.end_date, self.verbosity))
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
else:
# single-query, still-batched
start = 0
with server_side_cursors():
with transaction.atomic():
items = qs.iterator() # prevents filling query-cache
while True:
added = do_update_batch(backend, index, items, start, batch_size, total, self.verbosity)
if added > 0:
start += added
continue
break
if self.remove:
if self.start_date or self.end_date or total <= 0:
# They're using a reduced set, which may not incorporate
# all pks. Rebuild the list with everything.
qs = index.index_queryset().values_list('pk', flat=True)
pks_seen = set(smart_bytes(pk) for pk in qs)
total = len(pks_seen)
else:
pks_seen = set(smart_bytes(pk) for pk in qs.values_list('pk', flat=True))
if self.workers > 0:
ghetto_queue = []
for start in range(0, total, batch_size):
upper_bound = start + batch_size
if self.workers == 0:
do_remove(backend, index, model, pks_seen, start, upper_bound)
else:
ghetto_queue.append(('do_remove', model, pks_seen, start, upper_bound, using, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
def update_backend(self, label, using):
from haystack.exceptions import NotHandled
backend = haystack_connections[using].get_backend()
unified_index = haystack_connections[using].get_unified_index()
if self.workers > 0:
import multiprocessing
for model in get_models(label):
try:
index = unified_index.get_index(model)
except NotHandled:
if self.verbosity >= 2:
print("Skipping '%s' - no index." % model)
continue
if self.workers > 0:
# workers resetting connections leads to references to models / connections getting
# stale and having their connection disconnected from under them. Resetting before
# the loop continues and it accesses the ORM makes it better.
db.close_connection()
qs = index.build_queryset(using=using,
start_date=self.start_date,
end_date=self.end_date)
total = qs.count()
if self.verbosity >= 1:
print(u"Indexing %d %s" %
(total, force_text(model._meta.verbose_name_plural)))
batch_size = self.batchsize or backend.batch_size
if self.workers > 0:
ghetto_queue = []
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
if self.workers == 0:
do_update(backend, index, qs, start, end, total,
self.verbosity)
else:
ghetto_queue.append(
('do_update', model, start, end, total, using,
self.start_date, self.end_date, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
if self.remove:
if self.start_date or self.end_date or total <= 0:
# They're using a reduced set, which may not incorporate
# all pks. Rebuild the list with everything.
qs = index.index_queryset().values_list('pk', flat=True)
pks_seen = set(smart_bytes(pk) for pk in qs)
total = len(pks_seen)
else:
pks_seen = set(
smart_bytes(pk)
for pk in qs.values_list('pk', flat=True))
if self.workers > 0:
ghetto_queue = []
for start in range(0, total, batch_size):
upper_bound = start + batch_size
if self.workers == 0:
do_remove(backend, index, model, pks_seen, start,
upper_bound)
else:
ghetto_queue.append(
('do_remove', model, pks_seen, start, upper_bound,
using, self.verbosity))
if self.workers > 0:
pool = multiprocessing.Pool(self.workers)
pool.map(worker, ghetto_queue)
pool.terminate()
def test_get_models_specific(self):
from test_haystack.core.models import MockModel
models = app_loading.get_models('core.MockModel')
self.assertIsInstance(models, (list, GeneratorType))
self.assertListEqual(models, [MockModel])
def test_get_models_all(self):
models = app_loading.get_models('core')
self.assertIsInstance(models, (list, GeneratorType))
