def test_create_key(self): self.assertEqual(create_key('the best of times, the blurst of Times'), 'besttimesblurst') self.assertEqual(create_key('<?php $bling; $bling; ?>'), 'phpblingbling') self.assertEqual(create_key(''), '') self.assertEqual(create_key('the a an'), '') self.assertEqual(create_key('a'), '')
def test_create_key(self): self.assertEqual( create_key('the best of times, the blurst of Times'), 'bestoftimes') self.assertEqual(create_key('<?php $bling; $bling; ?>'), 'phpblingbling') self.assertEqual(create_key(''), '') self.assertEqual(create_key('the a an'), '') self.assertEqual(create_key('a'), '')
def remove_object(self, obj, data): title = data["title"] keys = [] for partial_title in partial_complete(title): partial_key = create_key(partial_title) self.client.srem(partial_key, data) key = "%s%s" % (self.prefix, partial_key) self.client.zrem(key, "^")
def autocomplete_keys(self, title): key = create_key(title) current_key = key[:MIN_LENGTH] for char in key[MIN_LENGTH:]: yield (current_key, char, ord(char)) current_key += char yield (current_key, self.terminator, 0)
def suggest(self, phrase, limit): phrase = create_key(phrase) if not phrase: return [] qs = AutocompleteObject.objects.filter( title__startswith=phrase, sites__pk__exact=settings.SITE_ID ).values_list('data', flat=True) if limit is not None: qs = qs[:limit] return qs
def remove_object(self, obj, data): title = data['title'] keys = [] obj_data = self.get_object_data(obj) #...how to figure out if its the final item... for partial_title in partial_complete(title): # get a list of all the keys that would have been set for the tries autocomplete_keys = list(self.autocomplete_keys(partial_title)) # flag for whether ours is the last object at this lookup is_last = False # grab all the members of this lookup set partial_key = create_key(partial_title) objects_at_key = self.client.smembers(partial_key) # check the data at this lookup set to see if ours was the only obj # referenced at this point if obj_data not in objects_at_key: # something weird happened and our data isn't even here continue elif len(objects_at_key) == 1: # only one object stored here, remove the terminal flag zset_key = '%s%s' % (self.prefix, partial_key) self.client.zrem(zset_key, '^') # see if there are any other references to keys here is_last = self.client.zcard(zset_key) == 0 if is_last: for (key, value, score) in reversed(autocomplete_keys): key = '%s%s' % (self.prefix, key) # another lookup ends here, so bail if '^' in self.client.zrange(key, 0, -1): self.client.zrem(key, value) break else: self.client.delete(key) # we can just blow away the lookup key self.client.delete(partial_key) else: # remove only our object's data self.client.srem(partial_key, obj_data) # finally, remove the data from the data key self.client.delete('objdata:%s' % obj_data)
def suggest(self, phrase, limit): """ Wrap our search & results with prefixing """ phrase = create_key(phrase) results = self._suggest("%s%s" % (self.prefix, phrase), limit) prefix_len = len(self.prefix) cleaned_keys = map(lambda x: x[prefix_len:], results) data = [] for key in cleaned_keys: data.extend(self.client.smembers(key)) return data
def remove_object(self, obj, data): title = data["title"] keys = [] obj_data = self.get_object_data(obj) # ...how to figure out if its the final item... for partial_title in partial_complete(title): # get a list of all the keys that would have been set for the tries autocomplete_keys = list(self.autocomplete_keys(partial_title)) # flag for whether ours is the last object at this lookup is_last = False # grab all the members of this lookup set partial_key = create_key(partial_title) objects_at_key = self.client.smembers(partial_key) # check the data at this lookup set to see if ours was the only obj # referenced at this point if obj_data not in objects_at_key: # something weird happened and our data isn't even here continue elif len(objects_at_key) == 1: # only one object stored here, remove the terminal flag zset_key = "%s%s" % (self.prefix, partial_key) self.client.zrem(zset_key, "^") # see if there are any other references to keys here is_last = self.client.zcard(zset_key) == 0 if is_last: for (key, value, score) in reversed(autocomplete_keys): key = "%s%s" % (self.prefix, key) # another lookup ends here, so bail if "^" in self.client.zrange(key, 0, -1): self.client.zrem(key, value) break else: self.client.delete(key) # we can just blow away the lookup key self.client.delete(partial_key) else: # remove only our object's data self.client.srem(partial_key, obj_data) # finally, remove the data from the data key self.client.delete("objdata:%s" % obj_data)
def store_object(self, obj, data): """ Given a title & some data that needs to be stored, make it available for autocomplete via the suggest() method """ title = data['title'] # store actual object data obj_data = self.get_object_data(obj) self.client.set('objdata:%s' % obj_data, data['data']) # create tries using sorted sets and add obj_data to the lookup set for partial_title in partial_complete(title): # store a reference to our object in the lookup set self.client.sadd(create_key(partial_title), obj_data) for (key, value, score) in self.autocomplete_keys(partial_title): self.client.zadd('%s%s' % (self.prefix, key), value, score)
def store_object(self, obj, data): """ Given a title & some data that needs to be stored, make it available for autocomplete via the suggest() method """ self.remove_object(obj, data) title = data['title'] for partial_title in partial_complete(title): key = create_key(partial_title) autocomplete_obj = AutocompleteObject( title=key, object_id=obj.pk, content_type=ContentType.objects.get_for_model(obj), pub_date=data['pub_date'], data=data['data']) autocomplete_obj.save() autocomplete_obj.sites = data['sites']
def store_object(self, obj, data): """ Given a title & some data that needs to be stored, make it available for autocomplete via the suggest() method """ title = data["title"] # store actual object data obj_data = self.get_object_data(obj) self.client.set("objdata:%s" % obj_data, data["data"]) # create tries using sorted sets and add obj_data to the lookup set for partial_title in partial_complete(title): # store a reference to our object in the lookup set self.client.sadd(create_key(partial_title), obj_data) for (key, value, score) in self.autocomplete_keys(partial_title): self.client.zadd("%s%s" % (self.prefix, key), value, score)
def store_object(self, obj, data): """ Given a title & some data that needs to be stored, make it available for autocomplete via the suggest() method """ self.remove_object(obj, data) title = data['title'] for partial_title in partial_complete(title): key = create_key(partial_title) obj = AutocompleteObject( title=key, object_id=obj.pk, content_type=ContentType.objects.get_for_model(obj), pub_date=data['pub_date'], data=data['data'] ) obj.save() obj.sites = data['sites']
def suggest(self, phrase, limit, models): """ Wrap our search & results with prefixing """ phrase = create_key(phrase) # perform the depth-first search over the sorted sets results = self._suggest('%s%s' % (self.prefix, phrase), limit) # strip the prefix off the keys that indicated they matched a lookup prefix_len = len(self.prefix) cleaned_keys = map(lambda x: x[prefix_len:], results) # lookup the data references for each lookup set obj_data_lookups = [] for key in cleaned_keys: obj_data_lookups.extend(self.client.smembers(key)) seen = set() data = [] if models: valid_models = set([str(model_class._meta) for model_class in models]) # grab the data for each object for lookup in obj_data_lookups: if lookup in seen: continue seen.add(lookup) if models: model_class, obj_pk = lookup.split(':') if model_class not in valid_models: continue data.append(self.client.get('objdata:%s' % lookup)) return data
def suggest(self, phrase, limit, models): """ Wrap our search & results with prefixing """ phrase = create_key(phrase) # perform the depth-first search over the sorted sets results = self._suggest("%s%s" % (self.prefix, phrase), limit) # strip the prefix off the keys that indicated they matched a lookup prefix_len = len(self.prefix) cleaned_keys = map(lambda x: x[prefix_len:], results) # lookup the data references for each lookup set obj_data_lookups = [] for key in cleaned_keys: obj_data_lookups.extend(self.client.smembers(key)) seen = set() data = [] if models: valid_models = set([str(model_class._meta) for model_class in models]) # grab the data for each object for lookup in obj_data_lookups: if lookup in seen: continue seen.add(lookup) if models: model_class, obj_pk = lookup.split(":") if model_class not in valid_models: continue data.append(self.client.get("objdata:%s" % lookup)) return data
def suggest(self, phrase, limit, models): phrase = create_key(phrase) if not phrase: return [] query = dict( title__startswith=phrase, sites__pk__exact=settings.SITE_ID, ) if models is not None: query.update(content_type__in=[ ContentType.objects.get_for_model(model_class) \ for model_class in models ]) qs = AutocompleteObject.objects.filter(**query).values_list( 'data', flat=True).distinct() if limit is not None: qs = qs[:limit] return qs
def suggest(self, phrase, limit, models): phrase = create_key(phrase) if not phrase: return [] query = dict( title__startswith=phrase, sites__pk__exact=settings.SITE_ID, ) if models is not None: query.update(content_type__in=[ ContentType.objects.get_for_model(model_class) \ for model_class in models ]) qs = AutocompleteObject.objects.filter( **query ).values_list('data', flat=True).distinct() if limit is not None: qs = qs[:limit] return qs