def index_document(doc, **kwargs):
key = document_key(doc['id'])
pipe = DB.pipeline()
tokens = {}
for indexer in config.INDEXERS:
try:
indexer(pipe, key, doc, tokens, **kwargs)
except ValueError as e:
print(e)
return # Do not index.
pipe.execute()
def index_document(doc, **kwargs):
key = keys.document_key(doc['id'])
pipe = DB.pipeline()
tokens = {}
for indexer in config.INDEXERS:
try:
indexer(pipe, key, doc, tokens, **kwargs)
except ValueError as e:
print(e)
return # Do not index.
pipe.execute()
def index_ngram_keys(*keys):
pipe = DB.pipeline(transaction=False)
for key in keys:
key = key.decode()
_, token = key.split('|')
if token.isdigit():
continue
index_edge_ngrams(pipe, token)
try:
pipe.execute()
except redis.RedisError as e:
msg = 'Error while generating ngrams:\n{}'.format(str(e))
raise ValueError(msg)
return keys
def index_document(doc, **kwargs):
key = keys.document_key(doc['id'])
pipe = DB.pipeline()
tokens = {}
for indexer in config.INDEXERS:
try:
indexer(pipe, key, doc, tokens, **kwargs)
except ValueError as e:
print(e)
return # Do not index.
try:
pipe.execute()
except redis.RedisError as e:
msg = 'Error while importing document:\n{}\n{}'.format(doc, str(e))
raise ValueError(msg)
def index_documents(docs):
pipe = DB.pipeline(transaction=False)
for doc in docs:
if not doc:
continue
if doc.get('_action') in ['delete', 'update']:
key = keys.document_key(doc['_id']).encode()
known_doc = get_document(key)
if known_doc:
deindex_document(known_doc)
if doc.get('_action') in ['index', 'update', None]:
index_document(pipe, doc)
yield doc
try:
pipe.execute()
except redis.RedisError as e:
msg = 'Error while importing document:\n{}\n{}'.format(doc, str(e))
raise ValueError(msg)
