def test():
# Creating a client with a given index name
client = Client('myIndex')
# Creating the index definition and schema
client.drop_index()
client.create_index([TextField('title', weight=5.0), TextField('body')])
# Indexing a document
client.add_document(
'doc1',
title='RediSearch',
body='Redisearch implements a search engine on top of redis')
# Simple search
res = client.search("search engine")
# the result has the total number of results, and a list of documents
print res.total # "1"
print res.docs[0]
# Searching with snippets
# res = client.search("search engine", snippet_sizes={'body': 50})
# Searching with complex parameters:
q = Query("search engine").verbatim().no_content().paging(0, 5)
res = client.search(q)
class TAS_Import():
def __init__(self, index_name, host="172.16.20.7", port=6382, db=0):
self.client = Client(index_name, host, port)
self.host = host
self.port = port
self.index_name = index_name
self.redis = Redis()
def add_indexing(self, schema):
self.client.create_index(schema, False, False, [])
return ["Done"]
def add_data(self, data):
for i, rr in enumerate(data):
index = i + 1
print rr
name, age, location = rr['name'], rr['age'], rr['location']
self.client.add_document(index,
NAME=name,
AGE=age,
LOCATION=location)
return ["Done"]
def drop_index(self):
try:
self.client.drop_index()
except:
pass
def create_website_items_index():
"Creates Index Definition."
# CREATE index
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
try:
client.drop_index() # drop if already exists
except ResponseError:
# will most likely raise a ResponseError if index does not exist
# ignore and create index
pass
except Exception:
raise_redisearch_error()
idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)])
# Index fields mentioned in e-commerce settings
idx_fields = frappe.db.get_single_value("E Commerce Settings",
"search_index_fields")
idx_fields = idx_fields.split(",") if idx_fields else []
if "web_item_name" in idx_fields:
idx_fields.remove("web_item_name")
idx_fields = list(map(to_search_field, idx_fields))
client.create_index(
[TextField("web_item_name", sortable=True)] + idx_fields,
definition=idx_def,
)
reindex_all_web_items()
define_autocomplete_dictionary()
def create_product_search_index_async():
print 'Creating Search Index'
client = Client('productIndex')
client.create_index([
TextField('title', weight=5.0),
TextField('description'),
TextField('tags'),
TextField('category')
])
products = Product.objects.filter(active=True)
cache.set('Search_index_total', len(products), timeout=None)
index = 0
for product in products:
title = product.name
description = product.description
category = ','.join([cat.name for cat in product.category.all()])
tag = product.tags
tag_maps = ProductTagMap.objects.filter(product=product)
for tag_map in tag_maps:
tag = tag + tag_map.tag.tag + ' '
category_varients = []
for pv in ProductVarientList.objects.filter(product=product):
for cv in pv.key.all():
category_varients.append(cv.value)
tag += ' '.join(list(set(category_varients)))
client.add_document(str(product.id),
title=title,
description=description,
tags=tag,
category=category)
cache.set('Search_index_index', index, timeout=None)
index += 1
return True
class SearchDemo:
def __init__(self, args):
self.index = args.index
self.client = Client(self.index, host=args.host, port=args.port)
def create(self):
try:
self.client.drop_index()
except:
pass
self.client.create_index([
NumericField('WORDCOUNT', sortable=True),
TextField('BYLINE', no_stem=True, sortable=True),
TextField('DOCUMENTTYPE', sortable=True),
TextField('HEADLINE', sortable=True),
TagField('KEYWORDS', separator=';'),
NumericField('MULTIMEDIA', sortable=True),
TextField('NEWDESK', sortable=True),
NumericField('PRINTPAGE', sortable=True),
NumericField('PUBDATE', sortable=True),
TextField('SECTIONNAME', sortable=True),
TextField('SNIPPET', sortable=True),
TextField('TYPEOFMATERIAL', sortable=True),
TextField('WEBURL')
])
class RediSearchClient(object):
def __init__(self, index_name):
self.client = Client(index_name)
self.index_name = index_name
def build_index(self, line_doc_path, n_docs):
line_pool = LineDocPool(line_doc_path)
try:
self.client.drop_index()
except:
pass
self.client.create_index([TextField('title'), TextField('url'), TextField('body')])
for i, d in enumerate(line_pool.doc_iterator()):
self.client.add_document(i, nosave = True, title = d['doctitle'],
url = d['url'], body = d['body'])
if i + 1 == n_docs:
break
if i % 1000 == 0:
print "{}/{} building index".format(i, n_docs)
def search(self, query):
q = Query(query).paging(0, 5).verbatim()
res = self.client.search(q)
# print res.total # "1"
return res
def create_website_items_index():
"Creates Index Definition."
# CREATE index
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
# DROP if already exists
try:
client.drop_index()
except Exception:
pass
idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)])
# Based on e-commerce settings
idx_fields = frappe.db.get_single_value('E Commerce Settings',
'search_index_fields')
idx_fields = idx_fields.split(',') if idx_fields else []
if 'web_item_name' in idx_fields:
idx_fields.remove('web_item_name')
idx_fields = list(map(to_search_field, idx_fields))
client.create_index(
[TextField("web_item_name", sortable=True)] + idx_fields,
definition=idx_def,
)
reindex_all_web_items()
define_autocomplete_dictionary()
def cache_to_redis(data: dict):
if REDIS_HOSTNAME == '':
print('REDIS_HOSTNAME environment variable is not set')
return
client = Client('games', host=REDIS_HOSTNAME, port=REDIS_PORT)
indexCreated = False
maxAltNames = len(max(data.values(), key=lambda d: len(d['alt_names']))['alt_names'])
while not indexCreated:
try:
client.create_index([TextField('name', weight=10),
*[TextField('alt_name_%d' % i, weight=10) for i in range(maxAltNames)],
TextField('summary', weight=1)],
TextField('cover', weight=0),
TextField('thumb', weight=0))
indexCreated = True
except Exception:
print('Failed to create index, retrying %s')
time.sleep(3)
for k, v in data.items():
client.add_document(k,
name=v['name'],
**{'alt_name_%d' % i: n for i, n in enumerate(v['alt_names'])},
cover=v['cover'],
thumb=v['thumb'],
summary=v['summary'])
print('done')
def build_ipa_index():
start_time = time.time()
rc = redis.Redis(password=os.environ.get('REDIS_PASSWORD', ''))
rs_client = Client('IPAIndex', conn=rc)
print(
'Getting file `amministrazioni.txt` from https://www.indicepa.gov.it',
flush=True)
ipa_index_amm_url = 'https://www.indicepa.gov.it/public-services/opendata-read-service.php?dstype=FS&filename=amministrazioni.txt'
ipa_index_amm = pd.read_csv(ipa_index_amm_url, sep='\t', dtype=str)
print('Getting file `ou.txt` from https://www.indicepa.gov.it', flush=True)
ipa_index_ou_url = 'https://www.indicepa.gov.it/public-services/opendata-read-service.php?dstype=FS&filename=ou.txt'
ipa_index_ou = pd.read_csv(ipa_index_ou_url,
sep='\t',
na_values=['da_indicare', '*****@*****.**'],
dtype=str)
ipa_index_ou = ipa_index_ou.loc[lambda ipa_index_ou: ipa_index_ou['cod_ou']
== 'Ufficio_Transizione_Digitale']
try:
rs_client.drop_index()
except:
pass # Index already dropped
rs_client.create_index([
TextField('ipa_code', weight=2.0),
TextField('name', weight=2.0, sortable=True),
TextField('site'),
TextField('pec'),
TextField('city', weight=1.4),
TextField('county'),
TextField('region'),
TagField('type'),
TextField('rtd_name'),
TextField('rtd_pec'),
TextField('rtd_mail'),
])
print('Created index `IPAIndex`', flush=True)
print('Feeding `IPAIndex` with data from `amministrazioni.txt`',
flush=True)
for index, row in ipa_index_amm.iterrows():
rs_client.add_document(row['cod_amm'],
language='italian',
replace=True,
**get_ipa_amm_item(row))
print('Feeding `IPAIndex` with data from `ou.txt`', flush=True)
for index, row in ipa_index_ou.iterrows():
rs_client.add_document(row['cod_amm'],
partial=True,
**get_ipa_rtd_item(row))
finish_time = time.time()
print('`IPAIndex` build completed in {0} seconds'.format(
round(finish_time - start_time, 2)),
flush=True)
def create_index(cls):
error_message = "Unable to create Index. Try Again"
redis_enabled = os.getenv("REDIS_SEARCH", False)
if redis_enabled:
client = Client("tower", port=6379, host=os.getenv('REDIS_HOST'))
try:
client.create_index(document)
cls.build_index(client)
print("Watcher Index created successfully")
except ResponseError as err:
print(err)
else:
print(error_message)
def insert():
# insertion of search/suggestion data
suggestion_client = Client('movie')
suggestion_client.create_index([TextField('title'), TagField('genres', separator = '|')])
for i in range(0, len(movie_df)):
suggestion_client.add_document(movie_df['tmdbId'][i], title = movie_df['title'][i], genres = movie_df['genres'][i])
# insertion of auto-completion data
completion_client = AutoCompleter('ac')
for i in range(0, len(movie_df)):
completion_client.add_suggestions(Suggestion(movie_df['title'][i]))
def clientpush(self):
client = Client('Checkout')
client.create_index([
NumericField('Key'),
TextField('UsageClass'),
TextField('CheckoutType'),
TextField('MaterialType'),
NumericField('CheckoutYear'),
NumericField('CheckoutMonth'),
NumericField('Checkouts'),
TextField('Title'),
TextField('Creator'),
TextField('Subjects'),
TextField('Publisher'),
TextField('PublicationYear')
])
db_connection, _ = self.connect()
cursor = db_connection.cursor()
cursor.execute('SELECT * FROM customers')
results = cursor.fetchall()
i = 0
for result in results:
client.add_document('doc%s' % i,
Key=result[0],
UsageClass=result[1],
CheckoutType=result[2],
MaterialType=result[3],
CheckoutYear=result[4],
CheckoutMonth=result[5],
Checkouts=result[6],
Title=result[7],
Creator=result[8],
Subjects=result[9],
Publisher=result[10],
PublicationYear=result[11])
i += 1
print(i)
res = client.search('BOOK')
print("{} {}".format(res.total, res.docs[0].Title))
res1 = client.search("use")
print(res1)
q = Query('use').verbatim().no_content().paging(0, 5)
res1 = client.search(q)
print(res1)
cursor.close()
db_connection.close()
class RandomWikipediaImport(object):
def __init__(self):
self.rs = Client('wikipedia')
self.rs.create_index((TextField('title', weight=5.0), TextField('body')))
print(f'>>> Created index')
def insert_random_loop(self):
i = 1
while True:
ra = wikipedia.random()
article = wikipedia.page(ra)
self.rs.add_document(f'doc{i}', title=article.title, body=article.content)
print(f'>>> Inserted {article.title}')
i += 1
class TAS_Import():
def __init__(self, index_name, host=ip, port=port, db=db):
self.client = Client(index_name, host, port)
self.host = host
self.port = port
#self.redis = Redis()
def add_indexing_schema(self, schema):
self.client.create_index(schema, False, False, [])
return ["Done"]
def add_data(self, rdata, company, doc_id, project):
for i, rr in enumerate(rdata):
index = doc_id + company + "CMDIC" + str(i + 1) + project
l1, l2, l3 = rr
l1 = config_obj.StringEscape(l1)
self.client.add_document(index, DATA=l1, PAGE=l2, BBOX=l3)
return ["Done"]
def drop_index(self):
try:
self.client.drop_index()
except Exception as e:
#print 'Error',e
pass
def start(self, data, doc_id, company, project):
status = 1
index_name = project + "_DOCUMENT_" + str(doc_id)
self.drop_index()
self.client = Client(index_name, self.host, self.port)
status = 2
schema = [
NumericField('INDEX'),
TextField('DATA'),
TextField('PAGE'),
TextField('BBOX')
]
status = 3
self.add_indexing_schema(schema)
status = 4
self.add_data(data, company, doc_id, project)
status = 5
return [status]
class EventProcessor():
def __init__(self):
self.r = redis.from_url(config.EVENT_BROKER_URL)
self.client = Client('CCTV_DATA')
try:
self.client.create_index([TextField('CCTV_ID'), TagField('TAGS')])
except Exception as error:
print("Error while creatign index", error)
# self.client.create_index([TextField('title', weight=5.0), TextField('body')])
def get_objects_in_image(self, image):
# TODO: call RedisAI module
objects = [
"key", "passport", "wallet", "car", "bag", "watch", "book",
"satchel", "laptop", "camera", "mobile_phone"
]
tags = []
tags.append(objects[r.randint(0, 10)])
tags.append(objects[r.randint(0, 10)])
tags.append(objects[r.randint(0, 10)])
tags.append(objects[r.randint(0, 10)])
return tags
def process(self, msg):
print("Going to process message and and store it", msg)
# print(float(msg["LON"]), float(msg["LAT"]), msg["CCTV_ID"])
# print(type(float(msg["LON"])), type(float(msg["LAT"])), msg["CCTV_ID"])
try:
self.r.geoadd("CCTV_LOCATION", float(msg["LON"]),
float(msg["LAT"]), msg["CCTV_ID"])
msg["TAGS"] = self.get_objects_in_image(msg.get("IMAGE", ""))
# print("Going to store this in search", msg)
doc_unique_key = msg["CCTV_ID"] + "_" + msg["TS"]
self.client.add_document(doc_unique_key,
CCTV_ID=doc_unique_key,
TAGS=",".join(msg["TAGS"]))
except Exception as error:
print("Error while adding ccty data", error)
def index():
client = Client('sh')
# client.drop_index()
client.create_index(txt=1.0)
chapters = {}
with open('will_play_text.csv') as fp:
r = csv.reader(fp, delimiter=';')
for line in r:
#['62816', 'Merchant of Venice', '9', '3.2.74', 'PORTIA', "I'll begin it,--Ding, dong, bell."]
play, chapter, character, text = line[1], line[2], line[4], line[5]
d = chapters.setdefault('{}:{}'.format(play, chapter), {})
d['play'] = play
d['text'] = d.get('text', '') + ' ' + text
for chapter, doc in chapters.iteritems():
print chapter, doc
client.add_document(chapter, nosave=True, txt=doc['text'])
def index():
client = Client('sh')
# client.drop_index()
client.create_index(txt=1.0)
chapters = {}
with open('will_play_text.csv') as fp:
r = csv.reader(fp, delimiter=';')
for line in r:
#['62816', 'Merchant of Venice', '9', '3.2.74', 'PORTIA', "I'll begin it,--Ding, dong, bell."]
play, chapter, character, text = line[1], line[2], line[4], line[5]
d = chapters.setdefault('{}:{}'.format(play, chapter), {})
d['play'] = play
d['text'] = d.get('text', '') + ' ' + text
for chapter, doc in chapters.iteritems():
print chapter, doc
client.add_document(chapter, nosave=True, txt=doc['text'])
def get(self, request):
# data=request.data
mes = {}
search_key = request.GET.get('key')
print(search_key)
all_classes = Course.objects.all()
print("开始创建索引——————————————————————————")
# 创建一个客户端与给定索引名称
client = Client('CII' + str(datetime.now()), host=settings.SIP, port='6666')
# 创建索引定义和模式
client.create_index((TextField('title'), TextField('body')))
print('索引创建完毕————————————————————————————————')
print('开始添加数据————————————————————————————————')
for i in all_classes:
print(str(i.id) + str(i.title))
# 索引文
client.add_document('result' + str(datetime.now()), title=i.title + '@' + str(i.id), info=i.info,
language='chinese')
print(333333333)
print('数据添加完毕————————————————————————————————')
print(client.info())
# 查找搜索
client = Client('CII' + str(datetime.now()), host=settings.SIP, port='6666')
res = client.search(search_key)
print('查询结束————————————————————————————————————————————————')
id_list = []
print(res.docs)
for i in res.docs:
# print(i.title) # 取出title，以@切割，取课程ID查询，然后序列化展示
id = i.title.split('@')[1]
id_list.append(id)
course = Course.objects.filter(id__in=id_list).all()
c = CourseSerializersModel(course, many=True)
mes['course'] = c.data
mes['code'] = 200
mes['message'] = '搜索完毕'
return Response(mes)
class SearchDemo:
def __init__(self, args):
self.index = args.index
self.client = Client(self.index, host=args.host, port=args.port)
def create(self):
try:
self.client.drop_index()
except:
pass
self.client.create_index([
NumericField('ORDERNUMBER'),
NumericField('QUANTITYORDERED', sortable=True),
NumericField('PRICEEACH', sortable=True),
NumericField('ORDERLINENUMBER'),
NumericField('SALES', sortable=True),
TextField('ORDERDATE'),
TextField('STATUS', sortable=True),
NumericField('QTR_ID', sortable=True),
NumericField('MONTH_ID', sortable=True),
NumericField('YEAR_ID', sortable=True),
TextField('PRODUCTLINE', sortable=True),
NumericField('MSRP', sortable=True),
TextField('PRODUCTCODE', sortable=True),
TextField('CUSTOMERNAME', sortable=True),
TextField('PHONE'),
TextField('ADDRESSLINE1'),
TextField('ADDRESSLINE2'),
TextField('CITY', sortable=True),
TextField('STATE', sortable=True),
TextField('POSTALCODE', sortable=True),
TextField('COUNTRY', sortable=True),
TextField('TERRITORY', sortable=True),
TextField('CONTACTLASTNAME'),
TextField('CONTACTFIRSTNAME'),
TextField('DEALSIZE', sortable=True)
])
class Hub(object):
dconn = None # document store connection
sconn = None # search index connection
qconn = None # queue connection
gh = None
autocomplete = None
repo = None
_ts = None
_hubkey = 'hub:catalog'
_ixname = 'ix'
_acname = 'ac'
def __init__(self,
ghlogin_or_token=None,
docs_url=None,
search_url=None,
queue_url=None,
repo=None):
timestamp = datetime.utcnow()
logger.info('Initializing temporary hub {}'.format(timestamp))
if ghlogin_or_token:
self.gh = Github(ghlogin_or_token)
elif 'GITHUB_TOKEN' in os.environ:
self.gh = Github(os.environ['GITHUB_TOKEN'])
else:
logger.info('Env var ' 'GITHUB_TOKEN' ' not found')
if docs_url:
pass
elif 'DOCS_REDIS_URL' in os.environ:
docs_url = os.environ['DOCS_REDIS_URL']
else:
logger.critical('No Redis for document storage... bye bye.')
raise RuntimeError('No Redis for document storage... bye bye.')
self.dconn = ReJSONClient().from_url(docs_url)
if search_url:
pass
elif 'SEARCH_REDIS_URL' in os.environ:
search_url = os.environ['SEARCH_REDIS_URL']
else:
search_url = docs_url
conn = Redis(connection_pool=ConnectionPool().from_url(search_url))
self.sconn = RediSearchClient(self._ixname, conn=conn)
self.autocomplete = AutoCompleter(self._acname, conn=conn)
if queue_url:
pass
elif 'QUEUE_REDIS_URL' in os.environ:
queue_url = os.environ['QUEUE_REDIS_URL']
else:
queue_url = docs_url
self.qconn = StrictRedis.from_url(queue_url)
if repo:
pass
elif 'REDISMODULES_REPO' in os.environ:
repo = os.environ['REDISMODULES_REPO']
else:
logger.critical('No REDISMODULES_REPO... bye bye.')
raise RuntimeError('No REDISMODULES_REPO... bye bye.')
self.repo = repo
# Check if hub exists
if self.dconn.exists(self._hubkey):
self._ts = datetime.fromtimestamp(
float(self.dconn.jsonget(self._hubkey, Path('.created'))))
logger.info('Latching to hub {}'.format(self._ts))
else:
self._ts = timestamp
logger.info('Creating hub {}'.format(self._ts))
self.createHub()
self.addModulesRepo(self.repo)
def get_repo_url(self):
return 'https://github.com/{}'.format(self.repo)
def createHub(self):
logger.info('Creating the hub in the database {}'.format(self._ts))
# Store the master modules catalog as an object
self.dconn.jsonset(
self._hubkey, Path.rootPath(), {
'created': str(_toepoch(self._ts)),
'modules': {},
'submissions': [],
'submit_enabled': False
})
# Create a RediSearch index for the modules
# TODO: catch errors
self.sconn.create_index(
(TextField('name', sortable=True), TextField('description'),
NumericField('stargazers_count', sortable=True),
NumericField('forks_count', sortable=True),
NumericField('last_modified', sortable=True)),
stopwords=stopwords)
def deleteHub(self):
# TODO
pass
def addModule(self, mod):
logger.info('Adding module to hub {}'.format(mod['name']))
# Store the module object as a document
m = RedisModule(self.dconn, self.sconn, self.autocomplete, mod['name'])
m.save(mod)
# Add a reference to it in the master catalog
self.dconn.jsonset(
self._hubkey, Path('.modules["{}"]'.format(m.get_id())), {
'id': m.get_id(),
'key': m.get_key(),
'created': str(_toepoch(self._ts)),
})
# Schedule a job to refresh repository statistics, starting from now and every hour
s = Scheduler(connection=self.qconn)
job = s.schedule(
scheduled_time=datetime(1970, 1, 1),
func=callRedisModuleUpateStats,
args=[m.get_id()],
interval=60 * 60, # every hour
repeat=None, # indefinitely
ttl=0,
result_ttl=0)
return m
"""
Adds modules to the hub from a local directory
TODO: deprecate asap
"""
def addModulesPath(self, path):
logger.info('Loading modules from local path {}'.format(path))
# Iterate module JSON files
for filename in os.listdir(path):
if filename.endswith(".json"):
with open('{}/{}'.format(path, filename)) as fp:
mod = json.load(fp)
m = self.addModule(mod['name'], mod)
"""
Adds a modules to the hub from a github repository
"""
def addModulesRepo(self, name, path='/modules/'):
# TODO: check for success
q = Queue(connection=self.qconn)
q.enqueue(callLoadModulesFromRepo, name, path)
def loadModulesFromRepo(self, name, path):
logger.info('Loading modules from Github {} {}'.format(name, path))
# TODO: error handling, sometimes not all contents are imported?
repo = self.gh.get_repo(name)
files = repo.get_dir_contents(path)
for f in files:
mod = json.loads(f.decoded_content)
m = self.addModule(mod)
"""
Submits a module to the hub
"""
def submitModule(self, repo_id, **kwargs):
logger.info('Module submitted to hub {}'.format(repo_id))
repo_id = repo_id.lower()
ts = datetime.utcnow()
res = {'id': repo_id, 'status': 'failed'}
if not self.dconn.jsonget(self._hubkey, Path('submit_enabled')):
res['message'] = 'Module submission is currently disabled'
return res
# Check if the module is already listed
m = RedisModule(self.dconn, self.sconn, self.autocomplete, repo_id)
if m.exists:
# TODO: return in search results
res['message'] = 'Module already listed in the hub'
return res
# Check if there's an active submission, or if the failure was too recent
submission = Submission(self.dconn, repo_id)
if submission.exists:
status = submission.status
if status != 'failed':
res['status'] = 'active'
res['message'] = 'Active submission found for module'
return res
else:
# TODO: handle failed submissions
res['message'] = 'Module already submitted to the hub and had failed, please reset manually for now'
return res
# Store the new submission
submission.save(**kwargs)
# Record the submission in the catalog
# TODO: find a good use for that, e.g. 5 last submissions
self.dconn.jsonarrappend(self._hubkey, Path('.submissions'), {
'id': submission.get_id(),
'created': submission.created,
})
# Add a job to process the submission
q = Queue(connection=self.qconn)
job = q.enqueue(callProcessSubmission, submission.get_id())
if job is None:
res['message'] = 'Submission job could not be created'
# TODO: design retry path
logger.error(
'Could not create submission processing job for {}'.format(
submission.get_id()))
else:
res['status'] = 'queued'
submission.status = res['status']
submission.job = job.id
return res
def viewSubmissionStatus(self, repo_id):
submission = Submission(self.dconn, repo_id)
if submission.exists:
res = {
'id': submission.get_id(),
'status': submission.status,
'message': submission.message,
}
if 'finished' == res['status']:
res['pull_number'] = submission.pull_number
res['pull_url'] = submission.pull_url
return res
def processSubmission(self, repo_id):
logger.info('Processing submision for {}'.format(repo_id))
submission = Submission(self.dconn, repo_id)
if submission.exists:
return submission.process(self.gh, self.repo)
def viewModules(self, query=None, sort=None):
if not query:
# Use a purely negative query to get all modules
query = '-etaoinshrdlu'
q = Query(query).no_content().paging(0, 1000)
if sort:
if sort == 'relevance':
pass
elif sort == 'update':
q.sort_by('last_modified')
elif sort == 'stars':
q.sort_by('stargazers_count', asc=False)
elif sort == 'forks':
q.sort_by('forks_count', asc=False)
elif sort == 'name':
q.sort_by('name')
results = self.sconn.search(q)
mods = []
fetch_duration = 0
# TODO: this should be pipelined
for doc in results.docs:
m = RedisModule(self.dconn, self.sconn, self.autocomplete, doc.id)
res, duration = _durationms(m.to_dict)
mods.append(res)
fetch_duration += duration
return {
'results': results.total,
'search_duration': '{:.3f}'.format(results.duration),
'fetch_duration': '{:.3f}'.format(fetch_duration),
'total_duration':
'{:.3f}'.format(fetch_duration + results.duration),
'modules': mods,
}
def viewSearchSuggestions(self, prefix):
suggestions = self.autocomplete.get_suggestions(prefix)
return [s.string for s in suggestions]
#--------------------------------------------
# Import the whole dirctory to redisearch
# Create the index and the documents
# Change the the dirt to your document's path
#--------------------------------------------
import os
from redisearch import Client, Query, TextField
dirt = "/path/to/the/documents/" # Change it to your own path
client = Client("BoxGroup", port=6379) # 6379 as default
client.create_index([TextField('title'), TextField('body')])
filelist = os.listdir(dirt)
filelist = sorted(filelist)
try:
filelist.remove(".git")
except:
print("git目录不存在，已跳过")
filecounter = 0
for filename in filelist:
openfilename = dirt + filename
with open(openfilename, "r+") as f:
data = f.read()
try:
client.add_document(filecounter,
title=filename,
body=data,
language="chinese")
except:
import pandas as pd
import json
from tqdm import tqdm
from redisearch import Client, TextField, NumericField, Query
from time import sleep
from rediscluster import StrictRedisCluster
sleep(15)
nodes = [{'host': "173.17.0.2", 'port': "7000"}]
rc = StrictRedisCluster(startup_nodes=nodes, decode_responses=True)
client=Client('week1', conn=rc)
client.create_index([TextField('name'), TextField('surname'), TextField('job')])
dat = pd.read_csv("test.csv")
for idx, row in tqdm(dat.iterrows()):
client.add_document(f"{row['index']}", replace=True, partial=True, name = f"{row['name']}", surname = f"{row['surname']}", job = f"{row['job']}")
import hashlib
import gpxpy
import gpxpy.gpx
from redisearch import Client, Query, TextField, GeoField, NumericField
client = Client(
'attractions',
host='127.0.0.1',
password='',
port=6379
)
client.create_index([
TextField('title', weight=5.0),
TextField('description'),
NumericField('verified', sortable=True),
GeoField('geo'),
])
gpx_file = open('All_States_Offbeat_Tourist_Attractions.gpx', 'r', encoding='utf-8')
gpx = gpxpy.parse(gpx_file)
for waypoint in gpx.waypoints:
if "Verified" in waypoint.comment:
v = 1
else:
v = 0
t = "%s,%s,%s" %(waypoint.name, waypoint.longitude, waypoint.latitude)
client.add_document(
class UserCache:
def __init__(self):
self.client = Client("api_user_index", app.config["REDIS_HOST"],
app.config["REDIS_PORT"])
def create_user_index(self, users):
"""
Creates a new user index if not exists
:param users:
:return:
"""
definition = IndexDefinition(prefix=['doc:', 'user:'])
try:
self.client.create_index(
(TextField("first_name"), TextField("last_name"),
TextField("email"), NumericField("age"),
NumericField("is_employee"),
NumericField("user_id", sortable=True)),
definition=definition)
except redis.exceptions.ResponseError:
return False
indexer = self.client.batch_indexer(chunk_size=len(users))
for user in users:
fields = {
"first_name":
user.first_name.translate(str.maketrans({"-": r"\-"})),
"last_name":
user.last_name.translate(str.maketrans({"-": r"\-"})),
"email":
user.email.translate(str.maketrans({"-": r"\-"})),
"age":
user.age,
"user_id":
user.id,
"is_employee":
int(user.is_employee),
}
indexer.add_document(f"doc:{user.id}", **fields)
indexer.commit()
return True
def cache_single_user(self, user):
"""
Caches a single user
:param user:
:return:
"""
self.client.redis.hset(
f"doc:{user.id}",
mapping={
"first_name":
user.first_name.translate(str.maketrans({"-": r"\-"})),
"last_name":
user.last_name.translate(str.maketrans({"-": r"\-"})),
"email":
user.email.translate(str.maketrans({"-": r"\-"})),
"age":
user.age,
"user_id":
user.id,
"is_employee":
int(user.is_employee),
})
return True
def search(self, filters, page, per_page):
"""
Searches through redis
:return:
"""
q = Query(self.build_query(filters)).paging(
(page - 1) * per_page, per_page).sort_by("user_id")
return self.client.search(q)
def build_query(self, filters):
query = []
age = "+@age:[minAge maxAge]"
for filter_name, value in filters.items():
# Ugly non-solid way
if value is not None:
if filter_name == "firstName" and len(value) > 1:
query.append(f"+@first_name:{value}*")
if filter_name == "lastName" and len(value) > 1:
query.append(f"+@last_name:{value}*")
if filter_name == "email" and len(value) > 1:
query.append(f"+@email:{value}*")
if filter_name == "minAge":
age = age.replace("minAge", str(value))
if filter_name == "maxAge":
age = age.replace("maxAge", str(value))
if filter_name == "isEmployee":
query.append(f"+@is_employee:{int(value)}")
age = age.replace("minAge", "0")
age = age.replace("maxAge", "100")
query.append(age)
return " ".join(query)
# Creating a client with a given index name
client = Client("cveIndex")
try:
client.info()
except Exception as e:
if e.args[0] != "Unknown Index name":
print("You must be running a redis server with the redisearch module installed")
exit()
# IndexDefinition is avaliable for RediSearch 2.0+
definition = IndexDefinition(prefix=['cve:'])
# Creating the index definition and schema
try:
client.create_index((TextField("id"), TextField("description"), TextField("configurations")), definition=definition)
except:
# Index already exists. Delete and recreate
client.drop_index()
print("Index already exists. Dropping. Delete keys and try again.")
exit()
def process_CVE_file(file):
with open(file, 'r', encoding="utf8") as f:
json = ujson.decode(f.read())
cve_items = json['CVE_Items']
for cve_item in cve_items:
cve_id = cve_item['cve']['CVE_data_meta']['ID']
cve_desc = cve_item['cve']['description']['description_data'][0]['value']
cve_configurations = str(cve_item['configurations']['nodes'])
# Sanitizing special characters to prevent them from being tokenized away
def open_redis():
if not os.path.isdir('./nvd_data_feeds/'):
os.mkdir('./nvd_data_feeds/')
print('Creating the docker container with redislabs/redisearch\n')
Popen([
'docker', 'run', '--rm', '--name', 'amadeus', '-p', '6379:6379',
'redislabs/redisearch:latest'
])
sleep(6)
urls = [
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2021.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2020.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2019.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2018.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2017.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2016.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2015.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2014.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2013.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2012.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2011.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2010.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2009.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2008.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2007.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2006.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2005.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2004.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2003.json.zip',
'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-2002.json.zip',
'https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip'
]
print('\nDownloading and unziping json feeds')
if not os.path.isdir('./downloads/'):
os.mkdir('./downloads/')
tam = len(urls)
dl = 0
for url in urls:
name = url.split('/')[-1]
response = get(url)
open('./downloads/' + name, 'wb').write(response.content)
with ZipFile('./downloads/' + name, 'r') as zip_ref:
zip_ref.extractall('./nvd_data_feeds/')
dl += 1
prog = dl / tam
done = int(50 * prog)
stdout.write('\r[%s%s%s]%s' %
('Progres > ', '=' * (done - 1) + '>', ' ' *
(50 - done), str(round(prog * 100)) + '%'))
rmtree('./downloads/')
print('\n')
print('Start processing CVE feeds')
# Create a normal redis connection
conn = redis.Redis('localhost')
# Creating a client with a given index name
client = Client('cveIndex')
# IndexDefinition is avaliable for RediSearch 2.0+
definition = IndexDefinition(prefix=['cve:'])
# Creating the index definition and schema
try:
client.create_index((TextField('id'), TextField('description'),
TextField('configurations')),
definition=definition)
except:
# Index already exists. Delete and recreate
client.drop_index()
print('Index already exists\nDropping\nDelete keys and try again')
exit()
def process_CVE_file(file):
with open(file, 'r', encoding='utf8') as f:
json = ujson.decode(f.read())
cve_items = json['CVE_Items']
for cve_item in cve_items:
cve_id = cve_item['cve']['CVE_data_meta']['ID']
cve_desc = cve_item['cve']['description']['description_data'][
0]['value']
cve_configurations = str(cve_item['configurations']['nodes'])
# Sanitizing special characters to prevent them from being tokenized away
cve_desc_sanitized = cve_desc.replace(':', 'cc11').replace(
'.', 'pp22').replace('*', 'ss33')
cve_configurations_sanitized = cve_configurations.replace(
':', 'cc11').replace('.', 'pp22').replace('*', 'ss33')
# Indexing a document for RediSearch 2.0+
client.redis.hset('cve:' + cve_id,
mapping={
'id':
cve_id,
'description':
cve_desc_sanitized,
'configurations':
cve_configurations_sanitized
})
print('Processed ' + file)
with ThreadPoolExecutor(max_workers=20) as pool:
futures = []
for i in range(2002, 2021):
future = pool.submit(
process_CVE_file,
'./nvd_data_feeds/nvdcve-1.1-{0}.json'.format(i))
futures.append(future)
json_list = [x.result() for x in as_completed(futures)]
print('Done processing CVE feeds\nProcessing NVD CPE match feed')
with open('./nvd_data_feeds/nvdcpematch-1.0.json', 'r',
encoding='utf8') as f:
json = ujson.decode(f.read())
matches = json['matches']
for match in matches:
rootUri = match['cpe23Uri']
keyName = rootUri
if 'versionStartIncluding' in match:
keyName += ';;versionStartIncluding=' + match[
'versionStartIncluding']
if 'versionStartExcluding' in match:
keyName += ';;versionStartExcluding=' + match[
'versionStartExcluding']
if 'versionEndIncluding' in match:
keyName += ';;versionEndIncluding=' + match[
'versionEndIncluding']
if 'versionEndExcluding' in match:
keyName += ';;versionEndExcluding=' + match[
'versionEndExcluding']
if len(match['cpe_name']) > 0:
# if CPE list is empty no need to include it in cache
valueString = ';;'.join(x['cpe23Uri']
for x in match['cpe_name'])
conn.set(keyName, valueString)
print('\nAMADEUS is already launched!')
import re
from datetime import datetime
import json
import logging
from random import randint
from time import sleep
#stagger reading and indexing for parallel
sleep(randint(1, 10))
logging.basicConfig(filename='parse.log',level=logging.INFO)
client = Client('medline')
try:
client.create_index([TextField('abstract')])
except ResponseError:
pass
with open(sys.argv[1], 'r') as f:
data=f.read()
recs = data.split("<PubmedArticle>");
recs = recs[1:]
indexer = client.batch_indexer(chunk_size=500)
count = 0
for r in recs:
from redisearch import Client, TextField
# Creating a client with a given index name
client = Client('myIndex')
# Creating the index definition and schema
client.create_index((TextField('title', weight=5.0), TextField('body')))
# Indexing a document
client.add_document(
'doc1',
title='RediSearch',
body='Redisearch impements a search engine on top of redis')
import time
from redisearch import Client, TextField, NumericField, Query
from redis.exceptions import ResponseError
file = open('test_set_tweets.txt', 'r')
client = Client('Tweets')
client.redis.flushdb()
client.create_index([TextField('tweet'), TextField('timestamp')])
start = time.time()
for x, line in enumerate(file.readlines()):
content = line.strip().split('\t')
try:
if len(content) == 4: # tem data
client.add_document('-'.join(content[:2]),
tweet=content[-2],
timestamp=content[-1])
else:
client.add_document('-'.join(content[:2]),
tweet=content[-1],
timestamp='')
except ResponseError:
pass
if x % 1000 == 0:
print(x, 'lines indexed...')
end = time.time()
print("Indexing time elapsed", end - start)
total = 0
for i in range(30):
start = time.time()
class BaseSearchHandler(BaseSearchHandlerSupport):
def __init__(self):
super().__init__()
self._entity = None
# This will only be here as an example
self.is_replacement = False
self.current_replacement = None
self.is_set_entity = False
# Subs are all of the subfields we would need to search through
self.subs: Dict[str, BaseSearchHandler] = {}
self.current_doc_id = None
self.current_doc_id_list = set()
self.current_client = None
self.print_sub = False
self.use_sub_query = False
self._super_ids = []
self._sub_ids = []
self.finished_alter = False
self.search_sub = False
self._processor: Optional[Processor] = None
def __setitem__(self, key: str, value: Any):
if key not in self.requirements.keys() and (not self.is_replacement):
return
self.is_set_entity = False
self.current_client = None
if isinstance(value, dict):
if len(value) == 0:
return
self.handle_input_dict_key(key, value)
else:
_instance_type = type(value)
# check that the value is the right type
if is_generic(_instance_type):
_str_type = to_str(_instance_type)
self.query_builder.insert_by_type_str(_str_type, key, value)
self.insert_builder.insert_by_type_str(_str_type, key, value)
@property
def replacement(self):
if self.current_replacement is None:
self.current_replacement = BaseSearchHandler()
self.current_replacement.is_replacement = True
self.current_replacement.insert_builder.is_replacement = True
# self.current_replacement.
return self.current_replacement
@property
def entity(self):
if self._entity is None:
raise AttributeError("You haven't set entity yet")
return self._entity
@entity.setter
def entity(self, _entity: str):
self._entity = _entity
@property
def processor(self):
if self._processor is None:
raise AttributeError("The processor hasn't been set yet.")
return self._processor
@processor.setter
def processor(self, _processor: Processor):
self._processor = _processor
self.set_sub_processors()
@property
def requirements(self):
return self._requirements_str
@requirements.setter
def requirements(self, _requirements: dict):
"""If we set it here we'd go through each dict item and create string version of each key"""
# Document id will allow us to figure out which documents are involved with subkeys
_requirements["entity"] = str
_requirements["super_id"] = str
self.process_requirements(_requirements)
if not self.is_sub_key:
self.create_sub_handlers()
@property
def dreq(self):
return self._dreq
@dreq.setter
def dreq(self, _req):
self._dreq = _req
self.reset()
self.requirements = _req
self.replacement.requirements = _req
@property
def allrequirements(self):
return self._dreq
@allrequirements.setter
def allrequirements(self, _req):
self._dreq = _req
self.reset()
self.requirements = _req
self.replacement.requirements = _req
@property
def doc_id(self):
""" We get the current doc_id if it exists"""
return self.current_doc_id
@doc_id.setter
def doc_id(self, _doc_id: str):
self.current_doc_id = _doc_id
@property
def verbatim(self):
return self.query_builder.build_exact()
@property
def client(self):
"""client
Get the client for the user. If it doesn't exist yet, create a new one with the given stop words.
For subkeys it adds fields if we've created them recently.
.. code-block:: python
>>> self.client.add_document(_id, payload, **records)
Returns
-------
[type]
A redis connected client. Gets connection from Jamboree processor.
"""
# self.processor
if self.current_client is None:
# We would insert a connection here. Use the connection from the search processor to operate.
with logger.catch(ResponseError):
self.current_client = Client(self.index,
conn=self.processor.rconn)
if len(self.indexable) > 0:
self.current_client.create_index(
self.indexable,
stopwords=[
"but", "there", "these", "they", "this", "to"
],
)
if self.is_sub_key:
if not self.finished_alter:
for i in self.indexable:
with suppress(ResponseError):
self.current_client.alter_schema_add([i])
self.finished_alter = True
return self.current_client
@property
def general(self):
return self.query_builder.general
@general.setter
def general(self, term: str):
""" Push a general term into the query. It can only be done once. Don't put it to a filter key."""
if not isinstance(term, str):
logger.error("Term isn't a string")
return
self.query_builder.general = term
"""
This is when things get weird
"""
def create_sub_handlers(self):
""" Creates subhandlers for the given index"""
for name, subkey in self.subfields.items():
subhandler = BaseSearchHandler()
subhandler.is_sub_key = True
subhandler.index = subkey
subhandler.insert_builder.is_sub = True
self.replacement.subs[name] = copy(subhandler)
self.subs[name] = subhandler
def set_sub_processors(self):
""" If there are any sub queries, set processors to them """
if len(self.subfields) > 0:
self.use_sub_query = True
for name in self.subfields.keys():
self.subs[name].processor = self.processor
with suppress(Exception):
self.replacement.subs[name].processor = self.processor
def set_entity(self):
if self.is_set_entity is False:
self["entity"] = {
"type": "TEXT",
"is_filter": True,
"values": {
"is_exact": True,
"term": self.entity
},
}
self.is_set_entity = True
def verbatim_docs(self):
built = self.query_builder.build_exact()
q = Query(built).no_stopwords().paging(0, 1000000)
results = self.client.search(q)
result_docs = results.docs
return result_docs
def general_docs(self):
built = self.query_builder.build()
q = Query(built).paging(0, 1000000)
results = self.client.search(q)
result_docs = results.docs
return result_docs
def verbatim_sub_ids(self):
super_id_set = set()
sub_id_set = set()
for key, sub in self.subs.items():
sub.print_sub = True
built = sub.query_builder.build()
# logger.warning(built)
built = built.strip(" ")
is_falsy = not built
if is_falsy:
continue
# logger.error(built)
verb_items = sub.general_docs()
current_super_ids = []
current_subs = []
for verb in verb_items:
try:
_verb_id = verb.id
_super_id = verb.super_id
full_dict = verb.__dict__
self.keystore.add(_super_id, key, full_dict)
current_subs.append(_verb_id)
current_super_ids.append(_super_id)
except Exception as e:
logger.error(str(e))
if len(current_super_ids) > 0:
if len(super_id_set) == 0:
super_id_set.update(current_super_ids)
else:
super_id_set = super_id_set.intersection(current_super_ids)
sub_id_set.update(current_subs)
return list(super_id_set), list(sub_id_set)
def verbatim_doc_ids(self):
q = Query(self.verbatim).no_content().paging(0, 1000000)
results = self.client.search(q)
ids = [res.id for res in results.docs]
return ids
def handle_input_dict_key(self, name: str, item: dict):
""" Figures out where to put the input dictionary for the query """
if self.is_sub(name):
# If this is a subkey we'll run the same operation again
# Check to see if the subkey is empty and has information that is reducible to "type"
self.use_sub_query = True
self.search_sub = True
reqs = self.loaded_dict_to_requirements(item)
# logger.debug(reqs)
self.subs[name].requirements = reqs
for k, v in item.items():
self.subs[name][k] = v
else:
# If it's not queryable don't try adding anything
if not is_queryable_dict(item):
return
self.insert_builder.from_dict(name, item)
self.query_builder.from_dict(name, item)
def normal_find(self, limit_ids=None):
built = self.query_builder.build()
q = Query(built).paging(0, 1000000)
if limit_ids is not None and len(limit_ids) > 0:
q.limit_ids(*limit_ids)
results = self.client.search(q)
result_docs = results.docs
return result_docs
def normal_find_ids(self, limit_ids=None):
_query = self.query_builder.build()
q = Query(_query).no_content().paging(0, 1000000)
if limit_ids is not None and len(limit_ids) > 0:
q.limit_ids(*limit_ids)
results = self.client.search(q)
result_docs = results.docs
return [res.id for res in result_docs]
def sub_find(self):
sup_ids, sub_ids = self.verbatim_sub_ids()
if len(sub_ids) == 0:
return []
results = self.normal_find(limit_ids=sup_ids)
results_dicts = []
for result in results:
_id, idict = split_doc(result)
idict.pop("payload", None)
subitems = self.keystore.get(_id)
idict.update(subitems)
results_dicts.append(idict)
return results_dicts
def normal_insert(self, allow_duplicates=False):
if allow_duplicates == False:
verbatim_docs = self.verbatim_docs()
if len(verbatim_docs) > 0 and allow_duplicates == False:
# Not adding docs because we're not allowing duplicates
return verbatim_docs[0].id, False
insert_variables = self.insert_builder.build()
_doc_id = self.insert_builder.doc_id
index_name = self.client.index_name
fields = [i.redis_args()[0] for i in self.indexable]
with logger.catch(message=f"{index_name} - {fields}", reraise=True):
self.client.add_document(_doc_id,
payload=_doc_id,
**insert_variables)
return _doc_id, True
def sub_insert(self, allow_duplicates=False):
_super_id, _did_insert = self.normal_insert(
allow_duplicates=allow_duplicates)
# logger.info(f'Did insert: {_did_insert}')
if _did_insert:
for key, sub in self.subs.items():
if len(sub.insert_builder._insert_dict) > 0:
sub.insert_builder.super_id = _super_id
sub.normal_insert(allow_duplicates=True)
return _super_id
def find_sub_dictionaries(self, super_id):
""" Finds a subdictionary by superid inside of the database. """
# Should use the find within function for every subkey
mega_dict = ADict()
for key, sub in self.subs.items():
key_dict = ADict()
try:
res = sub.client.search(f'"{super_id}"')
if res.total == 0:
continue
dd = [dictify(doc, False) for doc in res.docs]
key_dict[key] = dd[0]
except ResponseError:
pass
mega_dict.update(key_dict)
return mega_dict
def find(self):
"""Given the items we've set, find all matching items"""
self.set_entity()
self.keystore.reset()
if self.use_sub_query and self.search_sub:
return self.sub_find()
normal = self.normal_find()
if len(self.subs) == 0:
if len(normal) > 0:
return [doc_convert(x) for x in normal]
return normal
ndicts = []
for i in normal:
_i = dictify(i)
mega = self.find_sub_dictionaries(_i.id)
if len(mega) > 0:
_i.update(mega.to_dict())
ndicts.append(_i)
return ndicts
def pick(self, _id: str):
"""
Given an id find the element with the top level id. We aren't searching lower level_ids.
After we pull all of the
"""
self.set_entity()
self.keystore.reset()
doc = self.client.load_document(_id)
dd = doc.__dict__
doc = ADict(**dd)
_id = doc.pop("id", None)
doc.pop("payload", None)
doc_z = len(doc) > 0
if len(self.subs) == 0:
if not doc_z:
return None
doc.update({"id": _id})
return doc
if doc_z:
sub_dicts = self.find_sub_dictionaries(_id)
# if len(sub_dicts) > 0:
doc.update(sub_dicts)
doc.update({"id": _id})
return doc
return None
def update(self):
"""
# UPDATE
Given the items or ID we've set, partial update every matching document.
If we have the document_ids already, replace those items
"""
self.set_entity()
self.keystore.reset()
replacement_variables = self.replacement.insert_builder.build()
if self.use_sub_query == False:
doc_ids = self.verbatim_doc_ids()
batcher = self.client.batch_indexer(chunk_size=len(doc_ids))
for doc_id in doc_ids:
batcher.add_document(doc_id,
replace=True,
partial=True,
**replacement_variables)
batcher.commit()
else:
sup_ids, sub_ids = self.verbatim_sub_ids()
norm_ids = self.normal_find_ids(limit_ids=sup_ids)
batcher = self.client.batch_indexer(chunk_size=len(norm_ids))
for doc_id in norm_ids:
batcher.add_document(doc_id,
replace=True,
partial=True,
**replacement_variables)
batcher.commit()
for sub in self.subs.values():
subreplacement = sub.insert_builder.build()
if len(subreplacement) > 0:
subbatcher = sub.client.batch_indexer(
chunk_size=len(sub_ids))
for _id in sub_ids:
self.client.add_document(_id,
replace=True,
partial=True,
**subreplacement)
subbatcher.commit()
def update_id(self, _id):
self.set_entity()
self.keystore.reset()
doc = self.client.load_document(_id)
doc_dict, is_exist = single_doc_check_convert(doc)
if not is_exist:
return
replacement_variables = self.replacement.insert_builder.build()
self.client.add_document(_id,
replace=True,
partial=True,
**replacement_variables)
doc = self.client.load_document(_id)
# if len(self.subs) > 0:
# subreplacement = sub.insert_builder.build()
# def insert_many(self, list_of_items):
# self.client.ba
# pass
def insert(self, allow_duplicates=False):
"""
# INSERT
Given all of the items we've set, add documents
"""
self.set_entity()
self.keystore.reset()
previous_id = None
if self.use_sub_query:
previous_id = self.sub_insert(allow_duplicates=allow_duplicates)
else:
previous_id, _ = self.normal_insert(
allow_duplicates=allow_duplicates)
return previous_id
def remove(self):
"""Remove all documents that match a query.
Given a query, remove every document that matches the results of that query.
::
>>> search['name'] = 'sample_name'
>>> search['category'] = 'sample_query'
>>> search.remove()
"""
self.set_entity()
self.keystore.reset()
if self.use_sub_query and self.search_sub:
removable = set()
sup_ids, sub_ids = self.verbatim_sub_ids()
norm_ids = self.normal_find_ids(limit_ids=sup_ids)
removable = removable.intersection(sup_ids)
removable = removable.intersection(norm_ids)
[self.client.delete_document(_id) for _id in removable]
for sub in self.subs.values():
for _id in sub_ids:
sub.client.delete_document(_id)
else:
norm_ids = self.normal_find_ids()
[self.client.delete_document(_id) for _id in norm_ids]
def reset(self):
"""Reset all local variables"""
self.reset_builders()
self.is_set_entity = True
self.is_replacement = False
self.current_replacement = None
self.current_client = None
self.use_sub_query = False
def load_data(redis_server, redis_port, redis_password):
load_client = Client(
'fortune500-v1',
host=redis_server,
password=redis_password,
port=redis_port
)
load_ac = AutoCompleter(
'ac',
conn = load_client.redis
)
definition = IndexDefinition(
prefix=['fortune500:'],
language='English',
score_field='title',
score=0.5
)
load_client.create_index(
(
TextField("title", weight=5.0),
TextField('website'),
TextField('company'),
NumericField('employees', sortable=True),
TextField('industry', sortable=True),
TextField('sector', sortable=True),
TextField('hqcity', sortable=True),
TextField('hqstate', sortable=True),
TextField('ceo'),
TextField('ceoTitle'),
NumericField('rank', sortable=True),
NumericField('assets', sortable=True),
NumericField('revenues', sortable=True),
NumericField('profits', sortable=True),
NumericField('equity', sortable=True),
TagField('tags'),
TextField('ticker')
),
definition=definition)
with open('./fortune500.csv', encoding='utf-8') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
for row in csv_reader:
if line_count > 0:
load_ac.add_suggestions(Suggestion(row[1].replace('"', ''), 1.0))
load_client.redis.hset(
"fortune500:%s" %(row[1].replace(" ", '')),
mapping = {
'title': row[1],
'company': row[1],
'rank': row[0],
'website': row[2],
'employees': row[3],
'sector': row[4],
'tags': ",".join(row[4].replace('&', '').replace(',', '').replace(' ', ' ').split()).lower(),
'industry': row[5],
'hqcity': row[8],
'hqstate': row[9],
'ceo': row[12],
'ceoTitle': row[13],
'ticker': row[15],
'revenues': row[17],
'profits': row[19],
'assets': row[21],
'equity': row[22]
})
line_count += 1
# Finally Create the alias
load_client.aliasadd("fortune500")