def define_autocomplete_dictionary(): """Creates an autocomplete search dictionary for `name`. Also creats autocomplete dictionary for `categories` if checked in E Commerce Settings""" cache = frappe.cache() name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) cat_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache) ac_categories = frappe.db.get_single_value( 'E Commerce Settings', 'show_categories_in_search_autocomplete') # Delete both autocomplete dicts try: cache.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE)) cache.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE)) except Exception: return False items = frappe.get_all('Website Item', fields=['web_item_name', 'item_group'], filters={"published": 1}) for item in items: name_ac.add_suggestions(Suggestion(item.web_item_name)) if ac_categories and item.item_group: cat_ac.add_suggestions(Suggestion(item.item_group)) return True
def get_category_suggestions(query): search_results = {"results": []} if not is_redisearch_enabled(): # Redisearch module not enabled, query db categories = frappe.db.get_all( "Item Group", filters={ "name": ["like", "%{0}%".format(query)], "show_in_website": 1 }, fields=["name", "route"], ) search_results["results"] = categories return search_results if not query: return search_results ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache()) suggestions = ac.get_suggestions(query, num=10, with_payloads=True) results = [json.loads(s.payload) for s in suggestions] search_results["results"] = results return search_results
def __init__(self, args): self.host = args.host self.port = args.port self.key = args.key self.file = args.file self.rows = args.rows self.client = AutoCompleter(self.key, self.host, self.port)
def get_category_suggestions(query): search_results = {"results": []} if not is_search_module_loaded(): # Redisearch module not loaded, query db categories = frappe.db.get_all("Item Group", filters={ "name": ["like", "%{0}%".format(query)], "show_in_website": 1 }, fields=["name", "route"]) search_results['results'] = categories return search_results if not query: return search_results ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache()) suggestions = ac.get_suggestions(query, num=10) search_results['results'] = [s.string for s in suggestions] return search_results
def post(self): mydata = request.json location = str(mydata['location']) ac = AutoCompleter(current_app.config["REDISSEARCH_INDEX"], current_app.config["REDISSEARCH_URI"]) res = ac.add_suggestions(Suggestion(location, 1.0), increment = False) data = {'msg': res} return data, 200
def __init__(self, autocomplete_name, host='localhost', port=6379): try: self.host = host self.port = port self.table_name = autocomplete_name self.ac = AutoCompleter(autocomplete_name, host, port) except Exception as e: print >> sys.stderr, "TAS_Autocompleter Error inside constructor Index:\'", self.table_name, "\' HOST:\'", self.host, "\' PORT:\'", self.port, "\'\n" print >> sys.stderr, e
def define_autocomplete_dictionary(): """ Defines/Redefines an autocomplete search dictionary for Website Item Name. Also creats autocomplete dictionary for Published Item Groups. """ cache = frappe.cache() item_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) item_group_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache) # Delete both autocomplete dicts try: cache.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE)) cache.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE)) except Exception: raise_redisearch_error() create_items_autocomplete_dict(autocompleter=item_ac) create_item_groups_autocomplete_dict(autocompleter=item_group_ac)
def __init__(self, host=ip, port=port, db=db, autocomplete_name='Default'): self.client = Client(autocomplete_name,host,port) self.ipAdd = host self.ipPort = port self.db = db self.redisConn = redis.StrictRedis(host=self.ipAdd, port=self.ipPort, db=self.db) self.autocomplete = AutoCompleter(autocomplete_name, host, port) self.escape1 = re.compile(r'&#\d+;') self.escape2 = re.compile(r',|\.|<|>|{|}|[|]|"|\'|:|;|!|@|#|\$|%|\^|&|\*|\(|\)|-|\+|=|~') self.escape3 = re.compile(r'\s+')
def post(self): mydata = request.json location = str(mydata['location']) ac = AutoCompleter(current_app.config["REDISSEARCH_INDEX"], current_app.config["REDISSEARCH_URI"]) res = ac.delete(location); if res == 1: data = {'msg': 'Location deleted'} else: data = {'msg': 'Location not found'} return data, 200
def _setup_client(self, hostname: str, idx_name: str, port=6379) -> None: try: self._client = Client(idx_name, host=hostname, port=port) self._auto_compl = AutoCompleter(idx_name, hostname, port=port) self._hostname = hostname self._port = port self._idx = idx_name self._ready = True LOGGER.info("Cache engine is ready") except: self._client = None LOGGER.error("Cache engine is faulty!")
def insert(): # insertion of search/suggestion data suggestion_client = Client('movie') suggestion_client.create_index([TextField('title'), TagField('genres', separator = '|')]) for i in range(0, len(movie_df)): suggestion_client.add_document(movie_df['tmdbId'][i], title = movie_df['title'][i], genres = movie_df['genres'][i]) # insertion of auto-completion data completion_client = AutoCompleter('ac') for i in range(0, len(movie_df)): completion_client.add_suggestions(Suggestion(movie_df['title'][i]))
def product_autocomplete(request): autocomplete_key = request.POST.get('autocomplete_key', "").strip() if len(autocomplete_key) == 0: return JsonResponse({'autocomplete_values': []}) auto_completer = AutoCompleter('productAutocompleter') autocomplete_values = [] if auto_completer.len() == 0: create_product_autocompleter() res = auto_completer.get_suggestions(autocomplete_key, fuzzy=True) for acv in res: autocomplete_values.append(str(acv)) context = {'autocomplete_values': autocomplete_values} return JsonResponse(context)
def create_product_autocompleter(): auto_completer = AutoCompleter('productAutocompleter') products = Product.objects.filter(active=True) for product in products: title = product.name category = product.category.name auto_completer.add_suggestions(Suggestion(title, 5.0), Suggestion(category, 3.0)) for tag in Tag.objects.all(): auto_completer.add_suggestions(Suggestion(tag.tag, 5.0)) for cv in CategoryVarient.objects.all(): auto_completer.add_suggestions(Suggestion(cv.value, 2.0)) return True
def refresh_search_keys(request): if (request.user.is_authenticated() and request.user.is_staff): client = Client('productIndex') total_old_docts = client.info()['num_docs'] delete_status = client.drop_index() new_index = False if delete_status == 'OK': new_index = create_product_search_index() auto_completer = AutoCompleter('productAutocompleter') auto_completer_old_count = auto_completer.len() create_product_autocompleter() auto_completer_new_count = auto_completer.len() return JsonResponse({'success': True}) else: return JsonResponse({'success': False})
def product_search(query, limit=10, fuzzy_search=True): search_results = {"from_redisearch": True, "results": []} if not is_redisearch_enabled(): # Redisearch module not enabled search_results["from_redisearch"] = False search_results["results"] = get_product_data(query, 0, limit) return search_results if not query: return search_results red = frappe.cache() query = clean_up_query(query) # TODO: Check perf/correctness with Suggestions & Query vs only Query # TODO: Use Levenshtein Distance in Query (max=3) ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red) client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red) suggestions = ac.get_suggestions( query, num=limit, fuzzy=fuzzy_search and len(query) > 3 # Fuzzy on length < 3 can be real slow ) # Build a query query_string = query for s in suggestions: query_string += f"|('{clean_up_query(s.string)}')" q = Query(query_string) results = client.search(q) search_results["results"] = list(map(convert_to_dict, results.docs)) search_results["results"] = sorted( search_results["results"], key=lambda k: frappe.utils.cint(k["ranking"]), reverse=True) return search_results
from redisearch import TextField, NumericField, Query, AutoCompleter, Suggestion ac = AutoCompleter('ac', 'redis-search') ac.add_suggestions(Suggestion('google', 5.0), Suggestion('goo', 1.0)) f = open("location-cnt.txt", "r") for line in f: keywords = line.split(" ") keywords[1] = keywords[1].replace("_", " ") ac.add_suggestions(Suggestion(keywords[1].rstrip("\n"), float(keywords[0])))
else: redis_port = 6379 if environ.get('REDIS_PASSWORD') is not None: redis_password = environ.get('REDIS_PASSWORD') else: redis_password = '' client = Client( 'fortune500', host=redis_server, password=redis_password, port=redis_port ) ac = AutoCompleter( 'ac', conn = client.redis ) nav = Nav() topbar = Navbar('', View('Home', 'index'), View('Aggregations', 'show_agg'), View('CEO Search', 'search_ceo'), View('Tag Search', 'search_tags'), ) nav.register_element('top', topbar) def agg_by(field): ar = aggregation.AggregateRequest().group_by(field, reducers.count().alias('my_count')).sort_by(aggregation.Desc('@my_count'))
def __init__(self, ghlogin_or_token=None, docs_url=None, search_url=None, queue_url=None, repo=None): timestamp = datetime.utcnow() logger.info('Initializing temporary hub {}'.format(timestamp)) if ghlogin_or_token: self.gh = Github(ghlogin_or_token) elif 'GITHUB_TOKEN' in os.environ: self.gh = Github(os.environ['GITHUB_TOKEN']) else: logger.info('Env var ' 'GITHUB_TOKEN' ' not found') if docs_url: pass elif 'DOCS_REDIS_URL' in os.environ: docs_url = os.environ['DOCS_REDIS_URL'] else: logger.critical('No Redis for document storage... bye bye.') raise RuntimeError('No Redis for document storage... bye bye.') self.dconn = ReJSONClient().from_url(docs_url) if search_url: pass elif 'SEARCH_REDIS_URL' in os.environ: search_url = os.environ['SEARCH_REDIS_URL'] else: search_url = docs_url conn = Redis(connection_pool=ConnectionPool().from_url(search_url)) self.sconn = RediSearchClient(self._ixname, conn=conn) self.autocomplete = AutoCompleter(self._acname, conn=conn) if queue_url: pass elif 'QUEUE_REDIS_URL' in os.environ: queue_url = os.environ['QUEUE_REDIS_URL'] else: queue_url = docs_url self.qconn = StrictRedis.from_url(queue_url) if repo: pass elif 'REDISMODULES_REPO' in os.environ: repo = os.environ['REDISMODULES_REPO'] else: logger.critical('No REDISMODULES_REPO... bye bye.') raise RuntimeError('No REDISMODULES_REPO... bye bye.') self.repo = repo # Check if hub exists if self.dconn.exists(self._hubkey): self._ts = datetime.fromtimestamp( float(self.dconn.jsonget(self._hubkey, Path('.created')))) logger.info('Latching to hub {}'.format(self._ts)) else: self._ts = timestamp logger.info('Creating hub {}'.format(self._ts)) self.createHub() self.addModulesRepo(self.repo)
def delete_from_ac_dict(website_item_doc): """Removes this items's name from autocomplete dictionary""" cache = frappe.cache() name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) name_ac.delete(website_item_doc.web_item_name)
def insert_to_name_ac(web_name, doc_name): ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache()) ac.add_suggestions(Suggestion(web_name, payload=doc_name))
def __init__(self, host=ip, port=port, db=db, autocomplete_name='Default'): self.ipAdd = host self.ipPort = port self.db = db self.autocomplete = AutoCompleter(autocomplete_name, host, port)
def load_data(redis_server, redis_port, redis_password): load_client = Client( 'fortune500-v1', host=redis_server, password=redis_password, port=redis_port ) load_ac = AutoCompleter( 'ac', conn = load_client.redis ) definition = IndexDefinition( prefix=['fortune500:'], language='English', score_field='title', score=0.5 ) load_client.create_index( ( TextField("title", weight=5.0), TextField('website'), TextField('company'), NumericField('employees', sortable=True), TextField('industry', sortable=True), TextField('sector', sortable=True), TextField('hqcity', sortable=True), TextField('hqstate', sortable=True), TextField('ceo'), TextField('ceoTitle'), NumericField('rank', sortable=True), NumericField('assets', sortable=True), NumericField('revenues', sortable=True), NumericField('profits', sortable=True), NumericField('equity', sortable=True), TagField('tags'), TextField('ticker') ), definition=definition) with open('./fortune500.csv', encoding='utf-8') as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') line_count = 0 for row in csv_reader: if line_count > 0: load_ac.add_suggestions(Suggestion(row[1].replace('"', ''), 1.0)) load_client.redis.hset( "fortune500:%s" %(row[1].replace(" ", '')), mapping = { 'title': row[1], 'company': row[1], 'rank': row[0], 'website': row[2], 'employees': row[3], 'sector': row[4], 'tags': ",".join(row[4].replace('&', '').replace(',', '').replace(' ', ' ').split()).lower(), 'industry': row[5], 'hqcity': row[8], 'hqstate': row[9], 'ceo': row[12], 'ceoTitle': row[13], 'ticker': row[15], 'revenues': row[17], 'profits': row[19], 'assets': row[21], 'equity': row[22] }) line_count += 1 # Finally Create the alias load_client.aliasadd("fortune500")
from redisearch import Client, AutoCompleter, TextField, IndexDefinition, Query from flask import Flask, escape, request, jsonify from flask_cors import CORS app = Flask(__name__) CORS(app) client = Client('idx:addr', 'redis') ac = AutoCompleter('ac', 'redis') @app.route('/recherche') def recherches_adresses(): query = request.args.get("q") q = Query(query).language('french').paging(0, 10) res = client.search(q) adresses = {} for i, doc in enumerate(res.docs): adresses[i] = { "value": doc.id.replace("addr:", ""), "label": doc.adresse } return jsonify(adresses=adresses) @app.route('/suggestions') def suggestions_adresses(): query = request.args.get("q") suggs = ac.get_suggestions(query, fuzzy=True, with_payloads=True) adresses = {} for i, sugg in enumerate(suggs): adresses[i] = {"value": sugg.payload, "label": sugg.string}
def get(self): keyword = request.args['term'] ac = AutoCompleter(current_app.config["REDISSEARCH_INDEX"], current_app.config["REDISSEARCH_URI"]) res = ac.get_suggestions(keyword, fuzzy = True) return {"suggestion": [x.string for x in res]}, 200