def _generate_l5_verification_indexes() -> None:
client = _get_redisearch_index_client(Indexes.verification.value)
client.drop_index()
try:
client.create_index(
[
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
redisearch.TagField("dc_id"),
]
)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/([0-9]+)-([Ll])5(.*)$")
for block_path in block_paths:
if LEVEL == "1" and BROADCAST_ENABLED and re.search(pattern, block_path):
if not client.redis.sismember(L5_BLOCK_MIGRATION_KEY, block_path):
raw_block = storage.get_json_from_object(block_path)
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.verification.value, block_path.split("/")[1], block.export_as_search_index())
client.redis.sadd(L5_NODES, block.dc_id)
client.redis.sadd(L5_BLOCK_MIGRATION_KEY, block_path)
else:
_log.info(f"Skipping already indexed L5 block {block_path}")
def _get_custom_field_from_input(custom_index_input: "custom_index") -> redisearch.client.Field:
input_type = custom_index_input["type"]
field_name = custom_index_input["field_name"]
options = custom_index_input.get("options")
if input_type == "text":
weight = 1.0
sortable = False
no_stem = False
no_index = False
if options:
sortable = bool(options.get("sortable"))
no_stem = bool(options.get("no_stem"))
no_index = bool(options.get("no_index"))
cust_weight = options.get("weight")
if isinstance(cust_weight, (int, float)) and cust_weight >= 0 and cust_weight <= 1:
weight = float(cust_weight)
return redisearch.TextField(field_name, weight=weight, sortable=sortable, no_stem=no_stem, no_index=no_index)
elif input_type == "tag":
separator = ","
no_index = False
if options:
separator = options.get("separator") or ","
no_index = bool(options.get("no_index"))
return redisearch.TagField(field_name, separator=separator, no_index=no_index)
elif input_type == "number":
sortable = False
no_index = False
if options:
sortable = bool(options.get("sortable"))
no_index = bool(options.get("no_index"))
return redisearch.NumericField(field_name, sortable=sortable, no_index=no_index)
else:
raise RuntimeError(f"Index type {input_type} is not supported")
def _generate_smart_contract_indexes() -> None:
delete_index(Indexes.smartcontract.value) # Always generate smart contract indexes from scratch by dropping existing ones
client = _get_redisearch_index_client(Indexes.smartcontract.value)
client.create_index([redisearch.TagField("sc_name")])
# Find what smart contracts exist in storage
_log.info("Listing all smart contracts in storage")
sc_object_paths = storage.list_objects("SMARTCONTRACT/")
pattern = re.compile(r"SMARTCONTRACT\/.{36}\/metadata\.json$")
for sc in sc_object_paths:
if re.search(pattern, sc):
sc_model = smart_contract_model.new_from_at_rest(storage.get_json_from_object(sc))
_log.info(f"Adding index for smart contract {sc_model.id} ({sc_model.txn_type})")
put_document(Indexes.smartcontract.value, sc_model.id, sc_model.export_as_search_index())
def _generate_transaction_indexes() -> None: # noqa: C901
# -- CREATE INDEXES FOR TRANSACTIONS --
client = _get_redisearch_index_client(Indexes.transaction.value)
try:
client.create_index([redisearch.TagField("block_id")]) # Used for reverse-lookup of transactions by id (with no txn_type)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
try:
create_transaction_index(namespace.Namespaces.Contract.value, force=False) # Create the reserved txn type index
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
txn_types_to_watch = {namespace.Namespaces.Contract.value: 1} # Will be use when going through all stored transactions
txn_type_models = {
namespace.Namespaces.Contract.value: transaction_type_model.TransactionTypeModel(namespace.Namespaces.Contract.value, active_since_block="1")
}
for txn_type in transaction_type_dao.list_registered_transaction_types():
txn_type_model = transaction_type_model.new_from_at_rest(txn_type)
txn_type_models[txn_type_model.txn_type] = txn_type_model
_log.info(f"Adding index for {txn_type_model.txn_type}")
try:
create_transaction_index(txn_type_model.txn_type, txn_type_model.custom_indexes, force=False)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
txn_types_to_watch[txn_type_model.txn_type] = int(txn_type_model.active_since_block)
# -- LIST AND INDEX ACTUAL TRANSACTIONS FROM STORAGE
_log.info("Listing all full transactions")
transaction_blocks = storage.list_objects("TRANSACTION/")
for txn_path in transaction_blocks:
# do a check to see if this block's transactions were already marked as indexed
if not client.redis.sismember(TXN_MIGRATION_KEY, txn_path):
_log.info(f"Indexing transactions for {txn_path}")
for txn in storage.get(txn_path).split(b"\n"):
if txn:
txn_model = transaction_model.new_from_at_rest_full(json.loads(txn)["txn"])
# Add general transaction index
put_document(Indexes.transaction.value, f"txn-{txn_model.txn_id}", {"block_id": txn_model.block_id}, upsert=True)
watch_block = txn_types_to_watch.get(txn_model.txn_type)
# Extract custom indexes if necessary
if watch_block and int(txn_model.block_id) >= watch_block:
txn_model.extract_custom_indexes(txn_type_models[txn_model.txn_type])
put_document(txn_model.txn_type, txn_model.txn_id, txn_model.export_as_search_index(), upsert=True)
client.redis.sadd(TXN_MIGRATION_KEY, txn_path)
else:
_log.info(f"Skipping already indexed transaction {txn_path}")
def create_transaction_index(index: str, custom_indexes: Optional[Iterable["custom_index"]] = None, force: bool = True) -> None:
"""Create (and overwrite if necessary) index for a transaction type with optional custom_indexes"""
# Delete the index with this name if necessary
if force:
delete_index(index)
client = _get_redisearch_index_client(index)
# Set standard transaction indexes
index_fields = [
redisearch.TextField("tag"),
redisearch.NumericField("timestamp", sortable=True),
redisearch.NumericField("block_id", sortable=True),
redisearch.TagField("invoker"),
]
# Add custom indexes if they exist
if custom_indexes:
for idx in custom_indexes:
index_fields.append(_get_custom_field_from_input(idx))
# Create the actual index
client.create_index(index_fields)
def main():
print("hello!")
r = redis.Redis(host=redis_host, port=redis_port)
rs = redisearch.Client('recordIndex', redis_host, redis_port)
# flush to get a fresh db
# TODO - remove when dockerized
r.flushall()
record_collection = [{
'title': 'Brothers and Sisters',
'artist': 'Allman Brothers',
'year': 1973,
'genre': ['rock', 'southern rock', 'blues rock']
}, {
'title': 'Aja',
'artist': 'Steely Dan',
'year': 1977,
'genre': ['rock', 'pop']
}, {
'title': 'Can\'t Buy a Thrill',
'artist': 'Steely Dan',
'year': 1972,
'genre': ['rock', 'pop']
}, {
'title': 'Deguello',
'artist': 'ZZ Top',
'year': 1979,
'genre': ['rock']
}, {
'title': 'American Beauty',
'artist': 'Grateful Dead',
'year': 1970,
'genre': ['rock', 'psychedelic rock']
}, {
'title': 'Second Helping',
'artist': 'Lynard Skynard',
'year': 1974,
'genre': ['rock', 'southern rock']
}, {
'title': 'The Joker',
'artist': 'Steve Biller Band',
'year': 1973,
'genre': ['rock', 'blues rock']
}, {
'title': 'Book of Dreams',
'artist': 'Steve Biller Band',
'year': 1977,
'genre': ['rock']
}, {
'title': 'Rumours',
'artist': 'Fleetwood Mac',
'year': 1977,
'genre': ['rock', 'pop']
}, {
'title': 'Where We All Belong',
'artist': 'Marshall Tucker Band',
'year': 1974,
'genre': ['rock', 'southern rock']
}]
try:
rs.create_index((redisearch.TextField('title', sortable=True),
redisearch.TextField('artist', sortable=True),
redisearch.NumericField('year', sortable=True),
redisearch.TagField('genre', separator=',')))
except Exception:
print(f'Error creating index: {sys.exc_info()}')
print(f'index info: {rs.info()}')
run = True
load_data(rs, record_collection)
while run:
txt = input("enter a search term: ")
if (txt == "quit"):
run = False
break
txt_arr = txt.split(' ', 1)
print(f'searching {txt_arr}')
if (txt_arr[0] == 'title'):
res = rs.search(f'@title:{txt_arr[1]}')
print(res)
elif (txt_arr[0] == 'artist'):
res = rs.search(f'@artist:{txt_arr[1]}')
print(res)
elif (txt_arr[0] == 'year'):
full_txt_arr = txt.split(' ')
former = full_txt_arr[1]
latter = full_txt_arr[1]
if (len(full_txt_arr) == 3):
latter = full_txt_arr[2]
res = rs.search(f'@year:[{former} {latter}]')
print(res)
elif (txt_arr[0] == 'genre'):
pass
else:
print("invalid query")
