def test_create_key(self):
self.assertEqual(create_key('the best of times, the blurst of Times'),
'besttimesblurst')
self.assertEqual(create_key('<?php $bling; $bling; ?>'),
'phpblingbling')
self.assertEqual(create_key(''), '')
self.assertEqual(create_key('the a an'), '')
self.assertEqual(create_key('a'), '')
def test_create_key(self):
self.assertEqual(
create_key('the best of times, the blurst of Times'),
'bestoftimes')
self.assertEqual(create_key('<?php $bling; $bling; ?>'),
'phpblingbling')
self.assertEqual(create_key(''), '')
self.assertEqual(create_key('the a an'), '')
self.assertEqual(create_key('a'), '')
def remove_object(self, obj, data):
title = data["title"]
keys = []
for partial_title in partial_complete(title):
partial_key = create_key(partial_title)
self.client.srem(partial_key, data)
key = "%s%s" % (self.prefix, partial_key)
self.client.zrem(key, "^")
def autocomplete_keys(self, title):
key = create_key(title)
current_key = key[:MIN_LENGTH]
for char in key[MIN_LENGTH:]:
yield (current_key, char, ord(char))
current_key += char
yield (current_key, self.terminator, 0)
def autocomplete_keys(self, title):
key = create_key(title)
current_key = key[:MIN_LENGTH]
for char in key[MIN_LENGTH:]:
yield (current_key, char, ord(char))
current_key += char
yield (current_key, self.terminator, 0)
def suggest(self, phrase, limit):
phrase = create_key(phrase)
if not phrase:
return []
qs = AutocompleteObject.objects.filter(
title__startswith=phrase,
sites__pk__exact=settings.SITE_ID
).values_list('data', flat=True)
if limit is not None:
qs = qs[:limit]
return qs
def remove_object(self, obj, data):
title = data['title']
keys = []
obj_data = self.get_object_data(obj)
#...how to figure out if its the final item...
for partial_title in partial_complete(title):
# get a list of all the keys that would have been set for the tries
autocomplete_keys = list(self.autocomplete_keys(partial_title))
# flag for whether ours is the last object at this lookup
is_last = False
# grab all the members of this lookup set
partial_key = create_key(partial_title)
objects_at_key = self.client.smembers(partial_key)
# check the data at this lookup set to see if ours was the only obj
# referenced at this point
if obj_data not in objects_at_key:
# something weird happened and our data isn't even here
continue
elif len(objects_at_key) == 1:
# only one object stored here, remove the terminal flag
zset_key = '%s%s' % (self.prefix, partial_key)
self.client.zrem(zset_key, '^')
# see if there are any other references to keys here
is_last = self.client.zcard(zset_key) == 0
if is_last:
for (key, value, score) in reversed(autocomplete_keys):
key = '%s%s' % (self.prefix, key)
# another lookup ends here, so bail
if '^' in self.client.zrange(key, 0, -1):
self.client.zrem(key, value)
break
else:
self.client.delete(key)
# we can just blow away the lookup key
self.client.delete(partial_key)
else:
# remove only our object's data
self.client.srem(partial_key, obj_data)
# finally, remove the data from the data key
self.client.delete('objdata:%s' % obj_data)
def suggest(self, phrase, limit):
"""
Wrap our search & results with prefixing
"""
phrase = create_key(phrase)
results = self._suggest("%s%s" % (self.prefix, phrase), limit)
prefix_len = len(self.prefix)
cleaned_keys = map(lambda x: x[prefix_len:], results)
data = []
for key in cleaned_keys:
data.extend(self.client.smembers(key))
return data
def remove_object(self, obj, data):
title = data["title"]
keys = []
obj_data = self.get_object_data(obj)
# ...how to figure out if its the final item...
for partial_title in partial_complete(title):
# get a list of all the keys that would have been set for the tries
autocomplete_keys = list(self.autocomplete_keys(partial_title))
# flag for whether ours is the last object at this lookup
is_last = False
# grab all the members of this lookup set
partial_key = create_key(partial_title)
objects_at_key = self.client.smembers(partial_key)
# check the data at this lookup set to see if ours was the only obj
# referenced at this point
if obj_data not in objects_at_key:
# something weird happened and our data isn't even here
continue
elif len(objects_at_key) == 1:
# only one object stored here, remove the terminal flag
zset_key = "%s%s" % (self.prefix, partial_key)
self.client.zrem(zset_key, "^")
# see if there are any other references to keys here
is_last = self.client.zcard(zset_key) == 0
if is_last:
for (key, value, score) in reversed(autocomplete_keys):
key = "%s%s" % (self.prefix, key)
# another lookup ends here, so bail
if "^" in self.client.zrange(key, 0, -1):
self.client.zrem(key, value)
break
else:
self.client.delete(key)
# we can just blow away the lookup key
self.client.delete(partial_key)
else:
# remove only our object's data
self.client.srem(partial_key, obj_data)
# finally, remove the data from the data key
self.client.delete("objdata:%s" % obj_data)
def store_object(self, obj, data):
"""
Given a title & some data that needs to be stored, make it available
for autocomplete via the suggest() method
"""
title = data['title']
# store actual object data
obj_data = self.get_object_data(obj)
self.client.set('objdata:%s' % obj_data, data['data'])
# create tries using sorted sets and add obj_data to the lookup set
for partial_title in partial_complete(title):
# store a reference to our object in the lookup set
self.client.sadd(create_key(partial_title), obj_data)
for (key, value, score) in self.autocomplete_keys(partial_title):
self.client.zadd('%s%s' % (self.prefix, key), value, score)
def store_object(self, obj, data):
"""
Given a title & some data that needs to be stored, make it available
for autocomplete via the suggest() method
"""
self.remove_object(obj, data)
title = data['title']
for partial_title in partial_complete(title):
key = create_key(partial_title)
autocomplete_obj = AutocompleteObject(
title=key,
object_id=obj.pk,
content_type=ContentType.objects.get_for_model(obj),
pub_date=data['pub_date'],
data=data['data'])
autocomplete_obj.save()
autocomplete_obj.sites = data['sites']
def store_object(self, obj, data):
"""
Given a title & some data that needs to be stored, make it available
for autocomplete via the suggest() method
"""
title = data["title"]
# store actual object data
obj_data = self.get_object_data(obj)
self.client.set("objdata:%s" % obj_data, data["data"])
# create tries using sorted sets and add obj_data to the lookup set
for partial_title in partial_complete(title):
# store a reference to our object in the lookup set
self.client.sadd(create_key(partial_title), obj_data)
for (key, value, score) in self.autocomplete_keys(partial_title):
self.client.zadd("%s%s" % (self.prefix, key), value, score)
def store_object(self, obj, data):
"""
Given a title & some data that needs to be stored, make it available
for autocomplete via the suggest() method
"""
self.remove_object(obj, data)
title = data['title']
for partial_title in partial_complete(title):
key = create_key(partial_title)
obj = AutocompleteObject(
title=key,
object_id=obj.pk,
content_type=ContentType.objects.get_for_model(obj),
pub_date=data['pub_date'],
data=data['data']
)
obj.save()
obj.sites = data['sites']
def suggest(self, phrase, limit, models):
"""
Wrap our search & results with prefixing
"""
phrase = create_key(phrase)
# perform the depth-first search over the sorted sets
results = self._suggest('%s%s' % (self.prefix, phrase), limit)
# strip the prefix off the keys that indicated they matched a lookup
prefix_len = len(self.prefix)
cleaned_keys = map(lambda x: x[prefix_len:], results)
# lookup the data references for each lookup set
obj_data_lookups = []
for key in cleaned_keys:
obj_data_lookups.extend(self.client.smembers(key))
seen = set()
data = []
if models:
valid_models = set([str(model_class._meta) for model_class in models])
# grab the data for each object
for lookup in obj_data_lookups:
if lookup in seen:
continue
seen.add(lookup)
if models:
model_class, obj_pk = lookup.split(':')
if model_class not in valid_models:
continue
data.append(self.client.get('objdata:%s' % lookup))
return data
def suggest(self, phrase, limit, models):
"""
Wrap our search & results with prefixing
"""
phrase = create_key(phrase)
# perform the depth-first search over the sorted sets
results = self._suggest("%s%s" % (self.prefix, phrase), limit)
# strip the prefix off the keys that indicated they matched a lookup
prefix_len = len(self.prefix)
cleaned_keys = map(lambda x: x[prefix_len:], results)
# lookup the data references for each lookup set
obj_data_lookups = []
for key in cleaned_keys:
obj_data_lookups.extend(self.client.smembers(key))
seen = set()
data = []
if models:
valid_models = set([str(model_class._meta) for model_class in models])
# grab the data for each object
for lookup in obj_data_lookups:
if lookup in seen:
continue
seen.add(lookup)
if models:
model_class, obj_pk = lookup.split(":")
if model_class not in valid_models:
continue
data.append(self.client.get("objdata:%s" % lookup))
return data
def suggest(self, phrase, limit, models):
phrase = create_key(phrase)
if not phrase:
return []
query = dict(
title__startswith=phrase,
sites__pk__exact=settings.SITE_ID,
)
if models is not None:
query.update(content_type__in=[
ContentType.objects.get_for_model(model_class) \
for model_class in models
])
qs = AutocompleteObject.objects.filter(**query).values_list(
'data', flat=True).distinct()
if limit is not None:
qs = qs[:limit]
return qs
def suggest(self, phrase, limit, models):
phrase = create_key(phrase)
if not phrase:
return []
query = dict(
title__startswith=phrase,
sites__pk__exact=settings.SITE_ID,
)
if models is not None:
query.update(content_type__in=[
ContentType.objects.get_for_model(model_class) \
for model_class in models
])
qs = AutocompleteObject.objects.filter(
**query
).values_list('data', flat=True).distinct()
if limit is not None:
qs = qs[:limit]
return qs
