def _generate_l5_verification_indexes() -> None: client = _get_redisearch_index_client(Indexes.verification.value) client.drop_index() try: client.create_index( [ redisearch.NumericField("block_id", sortable=True), redisearch.NumericField("prev_id", sortable=True), redisearch.NumericField("timestamp", sortable=True), redisearch.TagField("dc_id"), ] ) except redis.exceptions.ResponseError as e: if not str(e).startswith("Index already exists"): # We don't care if index already exists raise _log.info("Listing all blocks in storage") block_paths = storage.list_objects("BLOCK/") pattern = re.compile(r"BLOCK\/([0-9]+)-([Ll])5(.*)$") for block_path in block_paths: if LEVEL == "1" and BROADCAST_ENABLED and re.search(pattern, block_path): if not client.redis.sismember(L5_BLOCK_MIGRATION_KEY, block_path): raw_block = storage.get_json_from_object(block_path) block = l5_block_model.new_from_at_rest(raw_block) put_document(Indexes.verification.value, block_path.split("/")[1], block.export_as_search_index()) client.redis.sadd(L5_NODES, block.dc_id) client.redis.sadd(L5_BLOCK_MIGRATION_KEY, block_path) else: _log.info(f"Skipping already indexed L5 block {block_path}")
def _get_custom_field_from_input(custom_index_input: "custom_index") -> redisearch.client.Field: input_type = custom_index_input["type"] field_name = custom_index_input["field_name"] options = custom_index_input.get("options") if input_type == "text": weight = 1.0 sortable = False no_stem = False no_index = False if options: sortable = bool(options.get("sortable")) no_stem = bool(options.get("no_stem")) no_index = bool(options.get("no_index")) cust_weight = options.get("weight") if isinstance(cust_weight, (int, float)) and cust_weight >= 0 and cust_weight <= 1: weight = float(cust_weight) return redisearch.TextField(field_name, weight=weight, sortable=sortable, no_stem=no_stem, no_index=no_index) elif input_type == "tag": separator = "," no_index = False if options: separator = options.get("separator") or "," no_index = bool(options.get("no_index")) return redisearch.TagField(field_name, separator=separator, no_index=no_index) elif input_type == "number": sortable = False no_index = False if options: sortable = bool(options.get("sortable")) no_index = bool(options.get("no_index")) return redisearch.NumericField(field_name, sortable=sortable, no_index=no_index) else: raise RuntimeError(f"Index type {input_type} is not supported")
def _generate_smart_contract_indexes() -> None: delete_index(Indexes.smartcontract.value) # Always generate smart contract indexes from scratch by dropping existing ones client = _get_redisearch_index_client(Indexes.smartcontract.value) client.create_index([redisearch.TagField("sc_name")]) # Find what smart contracts exist in storage _log.info("Listing all smart contracts in storage") sc_object_paths = storage.list_objects("SMARTCONTRACT/") pattern = re.compile(r"SMARTCONTRACT\/.{36}\/metadata\.json$") for sc in sc_object_paths: if re.search(pattern, sc): sc_model = smart_contract_model.new_from_at_rest(storage.get_json_from_object(sc)) _log.info(f"Adding index for smart contract {sc_model.id} ({sc_model.txn_type})") put_document(Indexes.smartcontract.value, sc_model.id, sc_model.export_as_search_index())
def _generate_transaction_indexes() -> None: # noqa: C901 # -- CREATE INDEXES FOR TRANSACTIONS -- client = _get_redisearch_index_client(Indexes.transaction.value) try: client.create_index([redisearch.TagField("block_id")]) # Used for reverse-lookup of transactions by id (with no txn_type) except redis.exceptions.ResponseError as e: if not str(e).startswith("Index already exists"): # We don't care if index already exists raise try: create_transaction_index(namespace.Namespaces.Contract.value, force=False) # Create the reserved txn type index except redis.exceptions.ResponseError as e: if not str(e).startswith("Index already exists"): # We don't care if index already exists raise txn_types_to_watch = {namespace.Namespaces.Contract.value: 1} # Will be use when going through all stored transactions txn_type_models = { namespace.Namespaces.Contract.value: transaction_type_model.TransactionTypeModel(namespace.Namespaces.Contract.value, active_since_block="1") } for txn_type in transaction_type_dao.list_registered_transaction_types(): txn_type_model = transaction_type_model.new_from_at_rest(txn_type) txn_type_models[txn_type_model.txn_type] = txn_type_model _log.info(f"Adding index for {txn_type_model.txn_type}") try: create_transaction_index(txn_type_model.txn_type, txn_type_model.custom_indexes, force=False) except redis.exceptions.ResponseError as e: if not str(e).startswith("Index already exists"): # We don't care if index already exists raise txn_types_to_watch[txn_type_model.txn_type] = int(txn_type_model.active_since_block) # -- LIST AND INDEX ACTUAL TRANSACTIONS FROM STORAGE _log.info("Listing all full transactions") transaction_blocks = storage.list_objects("TRANSACTION/") for txn_path in transaction_blocks: # do a check to see if this block's transactions were already marked as indexed if not client.redis.sismember(TXN_MIGRATION_KEY, txn_path): _log.info(f"Indexing transactions for {txn_path}") for txn in storage.get(txn_path).split(b"\n"): if txn: txn_model = transaction_model.new_from_at_rest_full(json.loads(txn)["txn"]) # Add general transaction index put_document(Indexes.transaction.value, f"txn-{txn_model.txn_id}", {"block_id": txn_model.block_id}, upsert=True) watch_block = txn_types_to_watch.get(txn_model.txn_type) # Extract custom indexes if necessary if watch_block and int(txn_model.block_id) >= watch_block: txn_model.extract_custom_indexes(txn_type_models[txn_model.txn_type]) put_document(txn_model.txn_type, txn_model.txn_id, txn_model.export_as_search_index(), upsert=True) client.redis.sadd(TXN_MIGRATION_KEY, txn_path) else: _log.info(f"Skipping already indexed transaction {txn_path}")
def create_transaction_index(index: str, custom_indexes: Optional[Iterable["custom_index"]] = None, force: bool = True) -> None: """Create (and overwrite if necessary) index for a transaction type with optional custom_indexes""" # Delete the index with this name if necessary if force: delete_index(index) client = _get_redisearch_index_client(index) # Set standard transaction indexes index_fields = [ redisearch.TextField("tag"), redisearch.NumericField("timestamp", sortable=True), redisearch.NumericField("block_id", sortable=True), redisearch.TagField("invoker"), ] # Add custom indexes if they exist if custom_indexes: for idx in custom_indexes: index_fields.append(_get_custom_field_from_input(idx)) # Create the actual index client.create_index(index_fields)
def main(): print("hello!") r = redis.Redis(host=redis_host, port=redis_port) rs = redisearch.Client('recordIndex', redis_host, redis_port) # flush to get a fresh db # TODO - remove when dockerized r.flushall() record_collection = [{ 'title': 'Brothers and Sisters', 'artist': 'Allman Brothers', 'year': 1973, 'genre': ['rock', 'southern rock', 'blues rock'] }, { 'title': 'Aja', 'artist': 'Steely Dan', 'year': 1977, 'genre': ['rock', 'pop'] }, { 'title': 'Can\'t Buy a Thrill', 'artist': 'Steely Dan', 'year': 1972, 'genre': ['rock', 'pop'] }, { 'title': 'Deguello', 'artist': 'ZZ Top', 'year': 1979, 'genre': ['rock'] }, { 'title': 'American Beauty', 'artist': 'Grateful Dead', 'year': 1970, 'genre': ['rock', 'psychedelic rock'] }, { 'title': 'Second Helping', 'artist': 'Lynard Skynard', 'year': 1974, 'genre': ['rock', 'southern rock'] }, { 'title': 'The Joker', 'artist': 'Steve Biller Band', 'year': 1973, 'genre': ['rock', 'blues rock'] }, { 'title': 'Book of Dreams', 'artist': 'Steve Biller Band', 'year': 1977, 'genre': ['rock'] }, { 'title': 'Rumours', 'artist': 'Fleetwood Mac', 'year': 1977, 'genre': ['rock', 'pop'] }, { 'title': 'Where We All Belong', 'artist': 'Marshall Tucker Band', 'year': 1974, 'genre': ['rock', 'southern rock'] }] try: rs.create_index((redisearch.TextField('title', sortable=True), redisearch.TextField('artist', sortable=True), redisearch.NumericField('year', sortable=True), redisearch.TagField('genre', separator=','))) except Exception: print(f'Error creating index: {sys.exc_info()}') print(f'index info: {rs.info()}') run = True load_data(rs, record_collection) while run: txt = input("enter a search term: ") if (txt == "quit"): run = False break txt_arr = txt.split(' ', 1) print(f'searching {txt_arr}') if (txt_arr[0] == 'title'): res = rs.search(f'@title:{txt_arr[1]}') print(res) elif (txt_arr[0] == 'artist'): res = rs.search(f'@artist:{txt_arr[1]}') print(res) elif (txt_arr[0] == 'year'): full_txt_arr = txt.split(' ') former = full_txt_arr[1] latter = full_txt_arr[1] if (len(full_txt_arr) == 3): latter = full_txt_arr[2] res = rs.search(f'@year:[{former} {latter}]') print(res) elif (txt_arr[0] == 'genre'): pass else: print("invalid query")