def test_lazy_model_dict(self):
user1, user2 = User.objects.all()[:2]
users = LazyModelDict()
self.assertEqual(len(users), 0)
# Add a User instance to the dict.
# A new LazyModel instance should be returned.
user1 = users.get_or_add(user1)
self.assertEqual(len(users), 1)
self.assertTrue(LazyModel.get_identifier(user1) in users)
self.assertTrue(isinstance(user1, LazyModel))
# Add the user again.
# The previous LazyModel instance should be returned.
new_user1 = users.get_or_add(User, user1.pk)
self.assertEqual(len(users), 1)
self.assertTrue(new_user1 is user1)
# Add the other user.
# A new LazyModel instance should be returned.
users.get_or_add(user2)
self.assertEqual(len(users), 2)
self.assertTrue(LazyModel.get_identifier(user2) in users)
def model_facet_counts(self):
"""
Get facet counts, same as facet_counts(), but convert any
ForeignKeyField and ManyToManyField values into LazyModel instances.
"""
engine = connections[self.query.backend.connection_alias]
unified_index = engine.get_unified_index()
fields = unified_index.fields
facet_counts = self.facet_counts()
facet_fields = facet_counts.get('fields', {})
for field_name, values in facet_fields.items():
field = fields.get(field_name)
if isinstance(field, (ForeignKeyField, ManyToManyField)):
new_values = []
for identifier, count in values:
value = LazyModel(identifier)
new_values.append((value, count))
facet_fields[field_name] = new_values
return facet_counts
def _get_model(item):
"""
Figure out what model we're dealing with.
Accepts an identifier string, a Model, a Model instance,
or a LazyModel instance.
"""
if isinstance(item, basestring):
# An identifier string.
identifier = item
elif isinstance(item, LazyModel) and not item:
# A LazyModel instance for an object that does not exist.
identifier = LazyModel.get_identifier(item)
else:
# A valid LazyModel instance or normal Model instance/class.
# Note: don't use the type() function with LazyModel instances.
if inspect.isclass(item):
return item
else:
return item.__class__
# Figure out the model from the identifier string.
app_label, model, object_pk = identifier.split('.', 2)
content_type = ContentType.objects.get_by_natural_key(app_label, model)
return content_type.model_class()
def update_object(item, remove=False, exception_handling=True):
"""
Update or remove an object from the search index.
Accepts an identifier string, a Model instance, or a LazyModel instance.
Runs after the transaction is committed, allowing for related data
to be saved before indexing the object.
"""
try:
index = get_index(item)
if isinstance(item, basestring):
# Dealing with an identifier string.
if not remove:
# Create a lazy instance with the read only cache. This means
# that it can benefit from existing cached objects, but won't
# fill up the cache with everything that gets indexed here.
item = LazyModel(item, cache_backend=read_only_cache)
if not remove and isinstance(item, LazyModel) and not item:
# The identifier was for an object that does not exist any
# more, so change this to a remove operation.
logging.warning('Could not access %r for indexing' % LazyModel.get_identifier(item))
remove = True
if remove:
# Remove this object from the index.
identifier = LazyModel.get_identifier(item)
index.remove_object(identifier)
else:
# Update this object in the index. This can actually remove the
# object from the index, if the result of should_index is False.
index.update_object(item)
except Exception:
if exception_handling:
logging.exception('Error running update_object(%r)' % item, debug_raise=True)
else:
raise
def __init__(self, *args, **kwargs):
if len(args) == 1 and isinstance(args[0], (Manager, QuerySet)):
value = args[0]
if isinstance(value, Manager):
value = value.all()
model = value.model
pk_list = value.values_list('pk', flat=True)
self.query_string = [LazyModel.get_identifier(model, pk) for pk in pk_list]
else:
lazy_object = LazyModel(*args, **kwargs)
if lazy_object.object_pk:
self.query_string = LazyModel.get_identifier(lazy_object)
else:
model = LazyModel.get_model_class(lazy_object)
raise model.DoesNotExist('%s with lookup %s does not exist.' % (
model.__name__, kwargs,
))
self.kwargs = kwargs
def prepare(self, obj):
value = super(MultiValueField, self).prepare(obj)
if value is None:
return None
if isinstance(value, Manager):
value = value.all()
if isinstance(value, QuerySet):
model = value.model
pk_list = value.values_list("pk", flat=True)
return [LazyModel.get_identifier(model, pk) for pk in pk_list]
if isinstance(value, Model):
value = [value]
return [LazyModel.get_identifier(item) for item in value]
def prepare(self, obj):
value = super(MultiValueField, self).prepare(obj)
if value is None:
return None
if isinstance(value, Manager):
value = value.all()
if isinstance(value, QuerySet):
model = value.model
pk_list = value.values_list('pk', flat=True)
return [LazyModel.get_identifier(model, pk) for pk in pk_list]
if isinstance(value, Model):
value = [value]
return [LazyModel.get_identifier(item) for item in value]
def __init__(self, *args, **kwargs):
if len(args) == 1 and isinstance(args[0], (Manager, QuerySet)):
value = args[0]
if isinstance(value, Manager):
value = value.all()
model = value.model
pk_list = value.values_list('pk', flat=True)
self.query_string = [
LazyModel.get_identifier(model, pk) for pk in pk_list
]
else:
lazy_object = LazyModel(*args, **kwargs)
if lazy_object.object_pk:
self.query_string = LazyModel.get_identifier(lazy_object)
else:
model = LazyModel.get_model_class(lazy_object)
raise model.DoesNotExist('%s with lookup %s does not exist.' %
(
model.__name__,
kwargs,
))
self.kwargs = kwargs
def queue_update(item, remove=False):
"""Queue an update for the search index."""
message = {
'identifier': LazyModel.get_identifier(item)
}
if remove:
message['remove'] = True
message_body = pickle.dumps(message, -1)
try:
with message_queue.open(QUEUE_NAME) as queue:
queue.put(message_body)
except Exception:
logging.exception(
'Could not send async message. '
'Running search update immediately.',
debug_raise=True,
)
update_object(item, remove=remove)
def prepare(self, obj):
value = super(ForeignKeyField, self).prepare(obj)
if value:
return LazyModel.get_identifier(value)
# or consider a middleware approach
# but that doesn't work from outside of requests
# 2. from mq.backends.sqs_backend import create_backend
# message_queue = create_backend(region, queue_prefix)
# maybe leave out the mq requirement and then the user can use this library
# with any message queue library.
try:
import cPickle as pickle
except ImportError:
import pickle
QUEUE_NAME = settings.APN_SEARCH_QUEUE
post_commit_once = post_commit(key=lambda item, **kwargs: LazyModel.get_identifier(item))
@post_commit_once
def update_object(item, remove=False, exception_handling=True):
"""
Update or remove an object from the search index.
Accepts an identifier string, a Model instance, or a LazyModel instance.
Runs after the transaction is committed, allowing for related data
to be saved before indexing the object.
"""
try:
def convert(self, value):
"""Returns a lazy object which fetches the related object."""
if value:
return LazyModel(value)
def prepare(self, obj):
value = super(ForeignKeyField, self).prepare(obj)
if value:
return LazyModel.get_identifier(value)
def convert(self, value):
"""Returns a fake manager object for accessing the lazy objects."""
values = (LazyModel(item) for item in value or [])
return ManyToManyManager(*values)
def object(self):
if self._object is None:
self._object = LazyModel(self.model, self.pk)
return self._object
