def try_fuzzy(helper, tokens, include_common=True): if not helper.bucket_dry or not tokens: return helper.debug('Fuzzy on. Trying with %s.', tokens) tokens.sort(key=lambda t: len(t), reverse=True) allkeys = helper.keys[:] if include_common: # As we are in fuzzy, try to narrow as much as possible by adding # unused common tokens. allkeys.extend( [t.db_key for t in helper.common if t.db_key not in helper.keys]) for try_one in tokens: if helper.bucket_full: break keys = allkeys[:] if try_one.db_key in keys: keys.remove(try_one.db_key) if try_one.isdigit(): continue helper.debug('Going fuzzy with %s and %s', try_one, keys) neighbors = make_fuzzy(try_one, max=helper.fuzzy) if len(keys): # Only retain tokens that have been seen in the index at least # once with the other tokens. DB.sadd(helper.pid, *neighbors) interkeys = [pair_key(k[2:]) for k in keys] interkeys.append(helper.pid) fuzzy_words = DB.sinter(interkeys) DB.delete(helper.pid) # Keep the priority we gave in building fuzzy terms (inversion # first, then substitution, etc.). fuzzy_words = [w.decode() for w in fuzzy_words] fuzzy_words.sort(key=lambda x: neighbors.index(x)) else: # The token we are considering is alone. fuzzy_words = [] for neighbor in neighbors: key = dbkeys.token_key(neighbor) count = DB.zcard(key) if count: fuzzy_words.append(neighbor) if fuzzy_words: helper.debug('Found fuzzy candidates %s', fuzzy_words) fuzzy_keys = [dbkeys.token_key(w) for w in fuzzy_words] for key in fuzzy_keys: if helper.bucket_dry: helper.add_to_bucket(keys + [key])
def try_fuzzy(helper, tokens, include_common=True): if not helper.bucket_dry or not tokens: return helper.debug('Fuzzy on. Trying with %s.', tokens) tokens.sort(key=lambda t: len(t), reverse=True) allkeys = helper.keys[:] if include_common: # As we are in fuzzy, try to narrow as much as possible by adding # unused commons tokens. common = [t for t in helper.common if t.db_key not in helper.keys] allkeys.extend([t.db_key for t in common]) for try_one in tokens: if helper.bucket_full: break keys = allkeys[:] if try_one.db_key in keys: keys.remove(try_one.db_key) if try_one.isdigit(): continue helper.debug('Going fuzzy with %s', try_one) neighbors = make_fuzzy(try_one, max=helper.fuzzy) if len(keys): # Only retains tokens that have been seen in the index at least # once with the other tokens. DB.sadd(helper.query, *neighbors) interkeys = [pair_key(k[2:]) for k in keys] interkeys.append(helper.query) fuzzy_words = DB.sinter(interkeys) DB.delete(helper.query) # Keep the priority we gave in building fuzzy terms (inversion # first, then substitution, etc.). fuzzy_words = [w.decode() for w in fuzzy_words] fuzzy_words.sort(key=lambda x: neighbors.index(x)) else: # The token we are considering is alone. fuzzy_words = [] for neighbor in neighbors: key = dbkeys.token_key(neighbor) count = DB.zcard(key) if count: fuzzy_words.append(neighbor) helper.debug('Found fuzzy candidates %s', fuzzy_words) fuzzy_keys = [dbkeys.token_key(w) for w in fuzzy_words] for key in fuzzy_keys: if helper.bucket_dry: helper.add_to_bucket(keys + [key])
def autocomplete(helper, tokens, skip_commons=False, use_geohash=False): helper.debug('Autocompleting %s', helper.last_token) keys = [t.db_key for t in tokens if not t.is_last] pair_keys = [pair_key(t) for t in tokens if not t.is_last] key = edge_ngram_key(helper.last_token) autocomplete_tokens = DB.sinter(pair_keys + [key]) helper.debug('Found tokens to autocomplete %s', autocomplete_tokens) for token in autocomplete_tokens: key = dbkeys.token_key(token.decode()) if skip_commons\ and token_key_frequency(key) > config.COMMON_THRESHOLD: helper.debug('Skip common token to autocomplete %s', key) continue if not helper.bucket_overflow or helper.last_token in helper.not_found: helper.debug('Trying to extend bucket. Autocomplete %s', key) extra_keys = [key] if use_geohash and helper.geohash_key: extra_keys.append(helper.geohash_key) helper.add_to_bucket(keys + extra_keys)
def autocomplete(helper, tokens, skip_commons=False, use_geohash=False): helper.debug('Autocompleting %s', helper.last_token) # helper.last_token.autocomplete() keys = [t.db_key for t in tokens if not t.is_last] pair_keys = [pair_key(t) for t in tokens if not t.is_last] key = edge_ngram_key(helper.last_token) autocomplete_tokens = DB.sinter(pair_keys + [key]) helper.debug('Found tokens to autocomplete %s', autocomplete_tokens) for token in autocomplete_tokens: key = dbkeys.token_key(token.decode()) if skip_commons\ and token_key_frequency(key) > config.COMMON_THRESHOLD: helper.debug('Skip common token to autocomplete %s', key) continue if not helper.bucket_overflow or helper.last_token in helper.not_found: helper.debug('Trying to extend bucket. Autocomplete %s', key) extra_keys = [key] if use_geohash and helper.geohash_key: extra_keys.append(helper.geohash_key) helper.add_to_bucket(keys + extra_keys)