def main():
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--url', help='Redis URL', type=str, default='redis://127.0.0.1:6379')
args = parser.parse_args()
# Set up Redis connection
url = urlparse(args.url)
r = redis.StrictRedis(host=url.hostname, port=url.port)
rsbeer = Client(ftbeeridx, conn=r)
rsbrewery = Client(ftbreweryidx, conn=r)
for rsclient in [rsbeer, rsbrewery]:
try:
cinfo = rsclient.info()
except:
continue
print("dropping index {}".format(cinfo['index_name']))
rsclient.drop_index()
print ("Importing categories...")
import_csv(r, category, catfile)
print ("Importing styles...")
import_csv(r, style, stylefile)
print ("Importing breweries...")
import_csv(r, brewery, breweryfile)
print ("Adding brewery geo data to RediSearch...")
import_brewery_geo(r, rsbrewery)
print ("Adding beer data to RediSearch...")
ftadd_beers(r, rsbeer)
print ("Done.")
def before_request():
g.redis = redis.StrictRedis(
host=app.config['REDIS_HOST'],
port=app.config['REDIS_PORT'],
)
g.rsbeer = Client(
'beerIdx',
conn=g.redis
)
g.rsbrewery = Client(
'breweryIdx',
conn=g.redis
)
def __init__(self):
self.r = redis.from_url(config.EVENT_BROKER_URL)
self.client = Client('CCTV_DATA')
try:
self.client.create_index([TextField('CCTV_ID'), TagField('TAGS')])
except Exception as error:
print("Error while creatign index", error)
def index_size(index_name):
# Creating a client with a given index name
# print("in index_size index_name=" + index_name)
client = Client(index_name, 'redis', 6379)
all_info = client.info()
# print(all_info)
return all_info
def create_product_search_index_async():
print 'Creating Search Index'
client = Client('productIndex')
client.create_index([
TextField('title', weight=5.0),
TextField('description'),
TextField('tags'),
TextField('category')
])
products = Product.objects.filter(active=True)
cache.set('Search_index_total', len(products), timeout=None)
index = 0
for product in products:
title = product.name
description = product.description
category = ','.join([cat.name for cat in product.category.all()])
tag = product.tags
tag_maps = ProductTagMap.objects.filter(product=product)
for tag_map in tag_maps:
tag = tag + tag_map.tag.tag + ' '
category_varients = []
for pv in ProductVarientList.objects.filter(product=product):
for cv in pv.key.all():
category_varients.append(cv.value)
tag += ' '.join(list(set(category_varients)))
client.add_document(str(product.id),
title=title,
description=description,
tags=tag,
category=category)
cache.set('Search_index_index', index, timeout=None)
index += 1
return True
def test():
# Creating a client with a given index name
client = Client('myIndex')
# Creating the index definition and schema
client.drop_index()
client.create_index([TextField('title', weight=5.0), TextField('body')])
# Indexing a document
client.add_document(
'doc1',
title='RediSearch',
body='Redisearch implements a search engine on top of redis')
# Simple search
res = client.search("search engine")
# the result has the total number of results, and a list of documents
print res.total # "1"
print res.docs[0]
# Searching with snippets
# res = client.search("search engine", snippet_sizes={'body': 50})
# Searching with complex parameters:
q = Query("search engine").verbatim().no_content().paging(0, 5)
res = client.search(q)
def create_website_items_index():
"Creates Index Definition."
# CREATE index
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
# DROP if already exists
try:
client.drop_index()
except Exception:
pass
idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)])
# Based on e-commerce settings
idx_fields = frappe.db.get_single_value('E Commerce Settings',
'search_index_fields')
idx_fields = idx_fields.split(',') if idx_fields else []
if 'web_item_name' in idx_fields:
idx_fields.remove('web_item_name')
idx_fields = list(map(to_search_field, idx_fields))
client.create_index(
[TextField("web_item_name", sortable=True)] + idx_fields,
definition=idx_def,
)
reindex_all_web_items()
define_autocomplete_dictionary()
def start(self, data, index_name):
status = 1
self.drop_index()
self.client = Client(index_name, self.host, self.port)
status = 2
schema = [
NumericField('INDEX'),
TextField('DATA'),
TextField('SECTION_TYPE'),
TextField('DOCID'),
TextField('PAGE'),
TextField('GRIDID'),
TextField("ROWCOL"),
TextField('BBOX'),
TextField("PAGE_GRID_SE"),
TextField('Rowspan'),
TextField('Colspan')
]
#rsObj.set_schema([NumericField('INDEX'), TextField('DOCID'), TextField('CATEGORY'), TextField('TAXONAME'), TextField('VALUE'), TextField('XML_REF'), TextField('REF_KEY')])
status = 3
self.add_indexing_schema(schema)
status = 4
self.add_data(data, index_name)
status = 5
return [status]
def cache_to_redis(data: dict):
if REDIS_HOSTNAME == '':
print('REDIS_HOSTNAME environment variable is not set')
return
client = Client('games', host=REDIS_HOSTNAME, port=REDIS_PORT)
indexCreated = False
maxAltNames = len(max(data.values(), key=lambda d: len(d['alt_names']))['alt_names'])
while not indexCreated:
try:
client.create_index([TextField('name', weight=10),
*[TextField('alt_name_%d' % i, weight=10) for i in range(maxAltNames)],
TextField('summary', weight=1)],
TextField('cover', weight=0),
TextField('thumb', weight=0))
indexCreated = True
except Exception:
print('Failed to create index, retrying %s')
time.sleep(3)
for k, v in data.items():
client.add_document(k,
name=v['name'],
**{'alt_name_%d' % i: n for i, n in enumerate(v['alt_names'])},
cover=v['cover'],
thumb=v['thumb'],
summary=v['summary'])
print('done')
def create_website_items_index():
"Creates Index Definition."
# CREATE index
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
try:
client.drop_index() # drop if already exists
except ResponseError:
# will most likely raise a ResponseError if index does not exist
# ignore and create index
pass
except Exception:
raise_redisearch_error()
idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)])
# Index fields mentioned in e-commerce settings
idx_fields = frappe.db.get_single_value("E Commerce Settings",
"search_index_fields")
idx_fields = idx_fields.split(",") if idx_fields else []
if "web_item_name" in idx_fields:
idx_fields.remove("web_item_name")
idx_fields = list(map(to_search_field, idx_fields))
client.create_index(
[TextField("web_item_name", sortable=True)] + idx_fields,
definition=idx_def,
)
reindex_all_web_items()
define_autocomplete_dictionary()
def general_search(request) -> Response:
"""
Default full text search on all resources if no sources are specified.
Faceted search if sources are specified.
**query**: Query to search.
**source**: Multiple sources can be specifed.
"""
client = Client(INDEX_NAME, conn=get_redis_connection())
query = request.GET.get('query')
sort_stars = request.GET.get('sort-stars')
resources = request.GET.getlist('source')
languages = request.GET.getlist('language')
awesome_lists = request.GET.getlist('awesome-list')
query = format_query(query, resources, languages, awesome_lists)
results = client.search(Query(query))
results = [doc.__dict__ for doc in results.docs]
if sort_stars == "true":
results.sort(key=lambda x: int(x['stargazers_count']), reverse=True)
return Response({
"docs": results
})
def get_indexed_client(self):
return Client(
"idx:stock",
host=self.hostname,
port=self.port,
password=self.password,
)
def __init__(self, args):
self.index = args.index
self.client = Client(self.index, host=args.host, port=args.port)
self.redis = redis.Redis(args.host, args.port)
self.fields = []
info = self.client.info()['fields']
for f in info:
self.fields.append(f[0])
def __init__(self, urls: List[str], max_per_list: int = MAX_RES_PER_LIST):
self.urls = urls
self.client = Client(INDEX_NAME,
host=REDIS_HOST,
port=REDIS_PORT,
password=REDIS_PASSWORD)
self.keys = Keys(KEY_PREFIX)
self.max = max_per_list
def build_ipa_index():
start_time = time.time()
rc = redis.Redis(password=os.environ.get('REDIS_PASSWORD', ''))
rs_client = Client('IPAIndex', conn=rc)
print(
'Getting file `amministrazioni.txt` from https://www.indicepa.gov.it',
flush=True)
ipa_index_amm_url = 'https://www.indicepa.gov.it/public-services/opendata-read-service.php?dstype=FS&filename=amministrazioni.txt'
ipa_index_amm = pd.read_csv(ipa_index_amm_url, sep='\t', dtype=str)
print('Getting file `ou.txt` from https://www.indicepa.gov.it', flush=True)
ipa_index_ou_url = 'https://www.indicepa.gov.it/public-services/opendata-read-service.php?dstype=FS&filename=ou.txt'
ipa_index_ou = pd.read_csv(ipa_index_ou_url,
sep='\t',
na_values=['da_indicare', '*****@*****.**'],
dtype=str)
ipa_index_ou = ipa_index_ou.loc[lambda ipa_index_ou: ipa_index_ou['cod_ou']
== 'Ufficio_Transizione_Digitale']
try:
rs_client.drop_index()
except:
pass # Index already dropped
rs_client.create_index([
TextField('ipa_code', weight=2.0),
TextField('name', weight=2.0, sortable=True),
TextField('site'),
TextField('pec'),
TextField('city', weight=1.4),
TextField('county'),
TextField('region'),
TagField('type'),
TextField('rtd_name'),
TextField('rtd_pec'),
TextField('rtd_mail'),
])
print('Created index `IPAIndex`', flush=True)
print('Feeding `IPAIndex` with data from `amministrazioni.txt`',
flush=True)
for index, row in ipa_index_amm.iterrows():
rs_client.add_document(row['cod_amm'],
language='italian',
replace=True,
**get_ipa_amm_item(row))
print('Feeding `IPAIndex` with data from `ou.txt`', flush=True)
for index, row in ipa_index_ou.iterrows():
rs_client.add_document(row['cod_amm'],
partial=True,
**get_ipa_rtd_item(row))
finish_time = time.time()
print('`IPAIndex` build completed in {0} seconds'.format(
round(finish_time - start_time, 2)),
flush=True)
def __init__(self, host=ip, port=port, db=db, autocomplete_name='Default'):
self.client = Client(autocomplete_name,host,port)
self.ipAdd = host
self.ipPort = port
self.db = db
self.redisConn = redis.StrictRedis(host=self.ipAdd, port=self.ipPort, db=self.db)
self.autocomplete = AutoCompleter(autocomplete_name, host, port)
self.escape1 = re.compile(r'&#\d+;')
self.escape2 = re.compile(r',|\.|<|>|{|}|[|]|"|\'|:|;|!|@|#|\$|%|\^|&|\*|\(|\)|-|\+|=|~')
self.escape3 = re.compile(r'\s+')
def _setup_client(self, hostname: str, idx_name: str, port=6379) -> None:
try:
self._client = Client(idx_name, host=hostname, port=port)
self._auto_compl = AutoCompleter(idx_name, hostname, port=port)
self._hostname = hostname
self._port = port
self._idx = idx_name
self._ready = True
LOGGER.info("Cache engine is ready")
except:
self._client = None
LOGGER.error("Cache engine is faulty!")
def __init__(self):
self.redis_info = config.Config.redis_info
self.ip, self.port, self.db = self.redis_info["host"], self.redis_info[
"port"], self.redis_info["db"]
index_name = self.redis_info["tb_name"]
self.client = Client(index_name, self.ip, self.port)
#self.rd_con = self.make_redis_connection()
self.escape1 = re.compile(r'&#\d+;')
self.escape2 = re.compile(
r',|\.|<|>|{|}|[|]|"|\'|:|;|!|@|#|\$|%|\^|&|\*|\(|\)|-|\+|=|~')
self.escape3 = re.compile(r'\s+')
pass
def __init__(self, table_name, host="localhost", port=6381):
try:
self.client = Client(table_name, host, port)
self.host = host
self.port = port
self.table_name = table_name
self.redis = Redis()
self.LIMIT = 10
except Exception as e:
print 'yyy'
print >> sys.stderr, "TAS_Redisearch Error inside Constructor Index:\'", table_name, "\' HOST:\'", host, "\' PORT:\'", port, "\'\n"
print >> sys.stderr, e
def __init__(self, args):
self.host = args.host
self.port = args.port
self.index = args.index
self.file = open(args.file, 'r')
self.delimiter = args.delimiter
self.rows = args.rows
self.hasHeader = args.header
self.ignore = args.ignore
self.docid = args.docid
self.client = Client(self.index, self.host, self.port)
self.fields = self.client.info()['fields']
def get(self, request):
# data=request.data
mes = {}
search_key = request.GET.get('key')
print(search_key)
all_classes = Course.objects.all()
print("开始创建索引——————————————————————————")
# 创建一个客户端与给定索引名称
client = Client('CII' + str(datetime.now()), host=settings.SIP, port='6666')
# 创建索引定义和模式
client.create_index((TextField('title'), TextField('body')))
print('索引创建完毕————————————————————————————————')
print('开始添加数据————————————————————————————————')
for i in all_classes:
print(str(i.id) + str(i.title))
# 索引文
client.add_document('result' + str(datetime.now()), title=i.title + '@' + str(i.id), info=i.info,
language='chinese')
print(333333333)
print('数据添加完毕————————————————————————————————')
print(client.info())
# 查找搜索
client = Client('CII' + str(datetime.now()), host=settings.SIP, port='6666')
res = client.search(search_key)
print('查询结束————————————————————————————————————————————————')
id_list = []
print(res.docs)
for i in res.docs:
# print(i.title) # 取出title，以@切割，取课程ID查询，然后序列化展示
id = i.title.split('@')[1]
id_list.append(id)
course = Course.objects.filter(id__in=id_list).all()
c = CourseSerializersModel(course, many=True)
mes['course'] = c.data
mes['code'] = 200
mes['message'] = '搜索完毕'
return Response(mes)
def insert():
# insertion of search/suggestion data
suggestion_client = Client('movie')
suggestion_client.create_index([TextField('title'), TagField('genres', separator = '|')])
for i in range(0, len(movie_df)):
suggestion_client.add_document(movie_df['tmdbId'][i], title = movie_df['title'][i], genres = movie_df['genres'][i])
# insertion of auto-completion data
completion_client = AutoCompleter('ac')
for i in range(0, len(movie_df)):
completion_client.add_suggestions(Suggestion(movie_df['title'][i]))
def create_index(cls):
error_message = "Unable to create Index. Try Again"
redis_enabled = os.getenv("REDIS_SEARCH", False)
if redis_enabled:
client = Client("tower", port=6379, host=os.getenv('REDIS_HOST'))
try:
client.create_index(document)
cls.build_index(client)
print("Watcher Index created successfully")
except ResponseError as err:
print(err)
else:
print(error_message)
def clientpush(self):
client = Client('Checkout')
client.create_index([
NumericField('Key'),
TextField('UsageClass'),
TextField('CheckoutType'),
TextField('MaterialType'),
NumericField('CheckoutYear'),
NumericField('CheckoutMonth'),
NumericField('Checkouts'),
TextField('Title'),
TextField('Creator'),
TextField('Subjects'),
TextField('Publisher'),
TextField('PublicationYear')
])
db_connection, _ = self.connect()
cursor = db_connection.cursor()
cursor.execute('SELECT * FROM customers')
results = cursor.fetchall()
i = 0
for result in results:
client.add_document('doc%s' % i,
Key=result[0],
UsageClass=result[1],
CheckoutType=result[2],
MaterialType=result[3],
CheckoutYear=result[4],
CheckoutMonth=result[5],
Checkouts=result[6],
Title=result[7],
Creator=result[8],
Subjects=result[9],
Publisher=result[10],
PublicationYear=result[11])
i += 1
print(i)
res = client.search('BOOK')
print("{} {}".format(res.total, res.docs[0].Title))
res1 = client.search("use")
print(res1)
q = Query('use').verbatim().no_content().paging(0, 5)
res1 = client.search(q)
print(res1)
cursor.close()
db_connection.close()
def refresh_search_keys(request):
if (request.user.is_authenticated() and request.user.is_staff):
client = Client('productIndex')
total_old_docts = client.info()['num_docs']
delete_status = client.drop_index()
new_index = False
if delete_status == 'OK':
new_index = create_product_search_index()
auto_completer = AutoCompleter('productAutocompleter')
auto_completer_old_count = auto_completer.len()
create_product_autocompleter()
auto_completer_new_count = auto_completer.len()
return JsonResponse({'success': True})
else:
return JsonResponse({'success': False})
def __init__(self, args):
self.host = args.host
self.port = args.port
self.search = False
self.index = args.index
if self.index is not None:
self.search = True
self.search_client = Client(self.index, self.host, self.port)
self.info = self.search_client.info()['fields']
self.file = open(args.file, 'r')
self.delimiter = args.delimiter
self.rows = args.rows
self.ignore = args.ignore
self.docid = args.docid
self.client = redis.Redis(args.host, args.port)
self.fields = []
def searchdb(search_content):
global total
client = Client("BoxGroup", port=6379)
search_content = ' '.join(jieba.cut(search_content))
q = Query(search_content).verbatim().paging(0, 500)
res = client.search(q)
total = res.total
titlelist = []
i = 0
while i < res.total:
titlelist.append(res.docs[i].title)
i += 1
if res.total > 0:
return titlelist
elif res.total == 0:
return "No result found"
def search(cls, query, offset=0, paginate=10):
client = Client("tower", port=6379, host=os.getenv('REDIS_HOST'))
q = Query(query).paging(offset, paginate)
res = client.search(q)
result = []
for doc in res.docs:
value_dict = {
'id': doc.id,
'client_ip': doc.clientIp,
'service': doc.service,
'error_message': doc.errorMessage,
'stack_trace': doc.stackTrace,
'numberRange': doc.numberRange
}
result.append(value_dict)
print(res)
return result
def start(self, data, doc_id, company, project):
status = 1
index_name = project + "_DOCUMENT_" + str(doc_id)
self.drop_index()
self.client = Client(index_name, self.host, self.port)
status = 2
schema = [
NumericField('INDEX'),
TextField('DATA'),
TextField('PAGE'),
TextField('BBOX')
]
status = 3
self.add_indexing_schema(schema)
status = 4
self.add_data(data, company, doc_id, project)
status = 5
return [status]
def product_search(query, limit=10, fuzzy_search=True):
search_results = {"from_redisearch": True, "results": []}
if not is_redisearch_enabled():
# Redisearch module not enabled
search_results["from_redisearch"] = False
search_results["results"] = get_product_data(query, 0, limit)
return search_results
if not query:
return search_results
red = frappe.cache()
query = clean_up_query(query)
# TODO: Check perf/correctness with Suggestions & Query vs only Query
# TODO: Use Levenshtein Distance in Query (max=3)
ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red)
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red)
suggestions = ac.get_suggestions(
query,
num=limit,
fuzzy=fuzzy_search
and len(query) > 3 # Fuzzy on length < 3 can be real slow
)
# Build a query
query_string = query
for s in suggestions:
query_string += f"|('{clean_up_query(s.string)}')"
q = Query(query_string)
results = client.search(q)
search_results["results"] = list(map(convert_to_dict, results.docs))
search_results["results"] = sorted(
search_results["results"],
key=lambda k: frappe.utils.cint(k["ranking"]),
reverse=True)
return search_results
