def _generate_l5_verification_indexes() -> None:
client = _get_redisearch_index_client(Indexes.verification.value)
client.drop_index()
try:
client.create_index(
[
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
redisearch.TagField("dc_id"),
]
)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/([0-9]+)-([Ll])5(.*)$")
for block_path in block_paths:
if LEVEL == "1" and BROADCAST_ENABLED and re.search(pattern, block_path):
if not client.redis.sismember(L5_BLOCK_MIGRATION_KEY, block_path):
raw_block = storage.get_json_from_object(block_path)
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.verification.value, block_path.split("/")[1], block.export_as_search_index())
client.redis.sadd(L5_NODES, block.dc_id)
client.redis.sadd(L5_BLOCK_MIGRATION_KEY, block_path)
else:
_log.info(f"Skipping already indexed L5 block {block_path}")
def _generate_block_indexes_from_scratch() -> None:
client = _get_redisearch_index_client(Indexes.block.value)
client.create_index([
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
])
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/[0-9]+$")
for block_path in block_paths:
if re.search(pattern, block_path):
_log.info(f"Adding index for {block_path}")
raw_block = storage.get_json_from_object(block_path)
block = cast("model.BlockModel", None)
if LEVEL == "1":
block = l1_block_model.new_from_stripped_block(raw_block)
elif LEVEL == "2":
block = l2_block_model.new_from_at_rest(raw_block)
elif LEVEL == "3":
block = l3_block_model.new_from_at_rest(raw_block)
elif LEVEL == "4":
block = l4_block_model.new_from_at_rest(raw_block)
elif LEVEL == "5":
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.block.value, block.block_id,
block.export_as_search_index())
def perform_api_key_migration_v1_if_necessary() -> None:
"""Checks if an api key migration needs to be performed, and does so if necessary"""
try:
if storage.get(f"{FOLDER}/{MIGRATION_V1}") == b"1":
# Migration was previously performed. No action necessary
return
except exceptions.NotFound:
pass
_log.info("Api key migration required. Performing now")
valid_keys = storage.list_objects(prefix=FOLDER)
regular_keys = list(
filter(lambda x: not x.startswith("KEYS/INTERCHAIN/"), valid_keys))
interchain_keys = list(
filter(lambda x: x.startswith("KEYS/INTERCHAIN/"), valid_keys))
for key in regular_keys:
_log.info(f"Migrating {key}")
api_key = api_key_model.new_from_legacy(
storage.get_json_from_object(key), interchain_dcid="")
save_api_key(api_key)
for key in interchain_keys:
_log.info(f"Migrating interchain key {key}")
interchain_dcid = key[key.find("KEYS/INTERCHAIN/") +
16:] # Get the interchain dcid from the key
api_key = api_key_model.new_from_legacy(
storage.get_json_from_object(key), interchain_dcid=interchain_dcid)
save_api_key(api_key)
# Save migration marker once complete
storage.put(f"{FOLDER}/{MIGRATION_V1}", b"1")
_log.info("Api key migration v1 complete")
def heap_list_v1(contract_id: str, path: str) -> List[str]:
sub_folder = f"{contract_id}/HEAP"
storage_key = f"{smart_contract_dao.FOLDER}/{sub_folder}{path}"
listed_keys = storage.list_objects(storage_key)
key_response = []
for key in listed_keys:
key_response.append(key[key.index(sub_folder) + len(sub_folder):])
return key_response
def _generate_smart_contract_indexes() -> None:
delete_index(Indexes.smartcontract.value) # Always generate smart contract indexes from scratch by dropping existing ones
client = _get_redisearch_index_client(Indexes.smartcontract.value)
client.create_index([redisearch.TagField("sc_name")])
# Find what smart contracts exist in storage
_log.info("Listing all smart contracts in storage")
sc_object_paths = storage.list_objects("SMARTCONTRACT/")
pattern = re.compile(r"SMARTCONTRACT\/.{36}\/metadata\.json$")
for sc in sc_object_paths:
if re.search(pattern, sc):
sc_model = smart_contract_model.new_from_at_rest(storage.get_json_from_object(sc))
_log.info(f"Adding index for smart contract {sc_model.id} ({sc_model.txn_type})")
put_document(Indexes.smartcontract.value, sc_model.id, sc_model.export_as_search_index())
def _generate_transaction_indexes() -> None: # noqa: C901
# -- CREATE INDEXES FOR TRANSACTIONS --
client = _get_redisearch_index_client(Indexes.transaction.value)
try:
client.create_index([redisearch.TagField("block_id")]) # Used for reverse-lookup of transactions by id (with no txn_type)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
try:
create_transaction_index(namespace.Namespaces.Contract.value, force=False) # Create the reserved txn type index
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
txn_types_to_watch = {namespace.Namespaces.Contract.value: 1} # Will be use when going through all stored transactions
txn_type_models = {
namespace.Namespaces.Contract.value: transaction_type_model.TransactionTypeModel(namespace.Namespaces.Contract.value, active_since_block="1")
}
for txn_type in transaction_type_dao.list_registered_transaction_types():
txn_type_model = transaction_type_model.new_from_at_rest(txn_type)
txn_type_models[txn_type_model.txn_type] = txn_type_model
_log.info(f"Adding index for {txn_type_model.txn_type}")
try:
create_transaction_index(txn_type_model.txn_type, txn_type_model.custom_indexes, force=False)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
txn_types_to_watch[txn_type_model.txn_type] = int(txn_type_model.active_since_block)
# -- LIST AND INDEX ACTUAL TRANSACTIONS FROM STORAGE
_log.info("Listing all full transactions")
transaction_blocks = storage.list_objects("TRANSACTION/")
for txn_path in transaction_blocks:
# do a check to see if this block's transactions were already marked as indexed
if not client.redis.sismember(TXN_MIGRATION_KEY, txn_path):
_log.info(f"Indexing transactions for {txn_path}")
for txn in storage.get(txn_path).split(b"\n"):
if txn:
txn_model = transaction_model.new_from_at_rest_full(json.loads(txn)["txn"])
# Add general transaction index
put_document(Indexes.transaction.value, f"txn-{txn_model.txn_id}", {"block_id": txn_model.block_id}, upsert=True)
watch_block = txn_types_to_watch.get(txn_model.txn_type)
# Extract custom indexes if necessary
if watch_block and int(txn_model.block_id) >= watch_block:
txn_model.extract_custom_indexes(txn_type_models[txn_model.txn_type])
put_document(txn_model.txn_type, txn_model.txn_id, txn_model.export_as_search_index(), upsert=True)
client.redis.sadd(TXN_MIGRATION_KEY, txn_path)
else:
_log.info(f"Skipping already indexed transaction {txn_path}")
def list_api_keys(include_interchain: bool) -> List[api_key_model.APIKeyModel]:
"""Retrieve a list of api keys
Args:
include_interchain: whether or not to include interchain api keys
Returns:
List of api key models
"""
# Get keys from storage, excluding migration marker and interchain keys
return_list = []
for key in storage.list_objects(prefix=FOLDER):
if (MIGRATION_V1 in key) or (key.startswith("KEYS/INTERCHAIN")
and not include_interchain):
continue
return_list.append(
api_key_model.new_from_at_rest(storage.get_json_from_object(key)))
return return_list
def rehydrate_transaction_types() -> None:
existing_list = redis.smembers_sync("type_list_key")
if len(existing_list) > 0:
_log.info("redis is already populated")
return
transaction_types = filter(lambda x: len(x.split("/")) == 3,
storage.list_objects("TRANSACTION_TYPES"))
txn_types = list(map(lambda x: x.split("/")[2], transaction_types))
_log.info("Inserting new list into redis")
if len(txn_types) > 0:
response_number = redis.sadd_sync("type_list_key", *txn_types)
if response_number > 0:
_log.info("Succeeded in updating redis")
_log.info(f"response number --> {response_number}")
else:
_log.info("No transaction types found to be updated...")
def _generate_transaction_indexes_from_scratch() -> None:
client = _get_redisearch_index_client(Indexes.transaction.value)
# TODO: replace after redisearch is fixed
client.create_index([
TagField("block_id")
]) # Used for reverse-lookup of transactions by id (with no txn_type)
force_create_transaction_index(namespace.Namespaces.Contract.value
) # Create the reserved txn type index
txn_types_to_watch = {
namespace.Namespaces.Contract.value: 1
} # Will be use when going through all stored transactions
txn_type_models = {
namespace.Namespaces.Contract.value:
transaction_type_model.TransactionTypeModel(
namespace.Namespaces.Contract.value, active_since_block="1")
}
for txn_type in transaction_type_dao.list_registered_transaction_types():
txn_type_model = transaction_type_model.new_from_at_rest(txn_type)
txn_type_models[txn_type_model.txn_type] = txn_type_model
_log.info(f"Adding index for {txn_type_model.txn_type}")
force_create_transaction_index(txn_type_model.txn_type,
txn_type_model.custom_indexes)
txn_types_to_watch[txn_type_model.txn_type] = int(
txn_type_model.active_since_block)
_log.info("Listing all full transactions")
transaction_blocks = storage.list_objects("TRANSACTION/")
for txn_path in transaction_blocks:
_log.info(f"Indexing transactions for {txn_path}")
for txn in storage.get(txn_path).split(b"\n"):
if txn:
txn_model = transaction_model.new_from_at_rest_full(
json.loads(txn)["txn"])
# Add general transaction index
put_document(Indexes.transaction.value,
f"txn-{txn_model.txn_id}",
{"block_id": txn_model.block_id},
upsert=True)
watch_block = txn_types_to_watch.get(txn_model.txn_type)
# Extract custom indexes if necessary
if watch_block and int(txn_model.block_id) >= watch_block:
txn_model.extract_custom_indexes(
txn_type_models[txn_model.txn_type])
put_document(txn_model.txn_type,
txn_model.txn_id,
txn_model.export_as_search_index(),
upsert=True)
def get_pending_l4_blocks(block_id: str) -> List[str]:
all_waiting_verification_keys = storage.list_objects(f"BROADCAST/TO_BROADCAST/{block_id}")
l4_blocks = []
for key in all_waiting_verification_keys:
record_list = storage.get_json_from_object(key)
for record in record_list:
item = {
"l1_dc_id": record["header"]["l1_dc_id"],
"l1_block_id": record["header"]["l1_block_id"],
"l4_dc_id": record["header"]["dc_id"],
"l4_block_id": record["header"]["block_id"],
"l4_proof": record["proof"]["proof"],
}
if record.get("is_invalid"):
item["is_invalid"] = record.get("is_invalid")
l4_blocks.append(json.dumps(item, separators=(",", ":")))
return l4_blocks
def list_interchain_clients(blockchain: str) -> List["model.InterchainModel"]:
"""Get all of the interchain clients for a specific blockchain type
Args:
blockchain: The blockchain of the desired clients to get
Returns:
List of instantiated interchain clients for the specified blockchain
"""
from_rest_function: Any = None
if blockchain == "bitcoin":
from_rest_function = btc.new_from_at_rest
elif blockchain == "ethereum":
from_rest_function = eth.new_from_at_rest
else:
raise exceptions.NotFound(
f"Blockchain network {blockchain} is not supported")
return [
from_rest_function(storage.get_json_from_object(x))
for x in storage.list_objects(f"{FOLDER}/{blockchain}/")
]
def get_verifications_for_l1_block(block_id: str,
level: int) -> List[Dict[str, Any]]:
try:
keys = list(
broadcast_functions.
get_receieved_verifications_for_block_and_level_sync(
block_id, level))
if len(keys) != 0:
for i in range(len(keys)):
keys[
i] = f"{FOLDER}/{block_id}-l{level}-{keys[i].decode('utf8')}"
return list(
map(lambda key: storage.get_json_from_object(key), keys))
except Exception:
pass
# Only fall back to listing from storage if we don't have verifications already saved in redis
prefix = f"{FOLDER}/{block_id}-l{level}"
keys = storage.list_objects(prefix)
_log.info(f"Verification keys by prefix {prefix}: {keys}")
return [] if len(keys) == 0 else list(
map(lambda key: storage.get_json_from_object(key), keys))
def get_verifications_for_l1_block(block_id: str,
level: int) -> List[Dict[str, Any]]:
try:
keys = list(
broadcast_functions.
get_receieved_verifications_for_block_and_level_sync(
block_id, level))
if len(keys) != 0:
for i in range(len(keys)):
keys[i] = f"{FOLDER}/{block_id}-l{level}-{keys[i]}"
return [storage.get_json_from_object(x) for x in keys]
except Exception:
_log.exception(
"Error getting verifications from cached list. Falling back to direct storage list"
)
# Only fall back to listing from storage if we don't have verifications already saved in redis
prefix = f"{FOLDER}/{block_id}-l{level}"
keys = storage.list_objects(prefix)
_log.info(f"Verification keys by prefix {prefix}: {keys}")
return [] if len(keys) == 0 else [
storage.get_json_from_object(key) for key in keys
]
def increment_storage_error_sync(block_id: str, current_level: int) -> None:
"""When getting a storage error/inconsistency between redis/storage, this should be called
This will roll-back a block to a previous level for verifications if FAULT_TOLERATION is surpassed for a block
Basically, the state in redis can be a mis-representation of what's in actual storage, and if this occurs, we need to roll back
the block verifications state and remove any bogus verifications from redis that aren't truly saved in storage, which happens on occasion
Args:
block_id: the block_id to increment a storage error
current_level: the current block verification level state (should be in broadcast:block:state)
"""
# Don't do anything if at or below level two because no verifications are required yet
if current_level <= 2:
return
error_key = storage_error_key(block_id)
current_count = int(redis.get_sync(error_key, decode=False) or 0)
if current_count < FAULT_TOLERATION:
redis.set_sync(error_key, str(current_count + 1))
return
# Beyond fault toleration, we must rollback this block
# First find all verifications actually in storage
prefix = f"BLOCK/{block_id}-l{current_level - 1}"
good_verifications = set()
for key in storage.list_objects(prefix):
good_verifications.add(re.search(f"^{prefix}-(.*)",
key).group(1)) # noqa: T484
# Now find all verifications the system thinks we have in redis
redis_verifications_key = verifications_key(block_id, current_level - 1)
all_verifications = redis.smembers_sync(redis_verifications_key)
# Remove all bad verifications recorded in redis that aren't in storage, and demote block to previous level
p = redis.pipeline_sync()
p.srem(redis_verifications_key,
*all_verifications.difference(good_verifications))
p.delete(error_key)
p.set(state_key(block_id), str(current_level - 1))
p.execute()
def get_api_key_list_v1() -> Dict[str, List[Dict[str, Any]]]:
"""
Gets the list of api key IDs
Returns:
List of API keys
"""
keys = storage.list_objects(prefix=FOLDER)
valid_keys = list(
filter(
lambda x: not x.startswith("KEYS/WEB_") and not x.startswith(
"KEYS/SC_") and not x.startswith("KEYS/INTERCHAIN"), keys))
returned_keys = []
for key in valid_keys:
resp = storage.get_json_from_object(key)
returned_keys.append({
"id":
str(resp["id"]),
"registration_time":
int(resp["registration_time"]),
"nickname":
str(resp.get("nickname") or "")
})
return {"keys": returned_keys}
def _generate_block_indexes() -> None:
client = _get_redisearch_index_client(Indexes.block.value)
try:
client.create_index([
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
])
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"
): # We don't care if index already exists
raise
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/[0-9]+$")
for block_path in block_paths:
if re.search(pattern, block_path):
# do a check to see if this block was already marked as indexed
if not client.redis.sismember(BLOCK_MIGRATION_KEY, block_path):
_log.info(f"Adding index for {block_path}")
raw_block = storage.get_json_from_object(block_path)
block = cast("model.BlockModel", None)
if LEVEL == "1":
block = l1_block_model.new_from_stripped_block(raw_block)
elif LEVEL == "2":
block = l2_block_model.new_from_at_rest(raw_block)
elif LEVEL == "3":
block = l3_block_model.new_from_at_rest(raw_block)
elif LEVEL == "4":
block = l4_block_model.new_from_at_rest(raw_block)
elif LEVEL == "5":
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.block.value, block.block_id,
block.export_as_search_index())
client.redis.sadd(BLOCK_MIGRATION_KEY, block_path)
else:
_log.info(f"Skipping already indexed block {block_path}")
def _level_records(block_id: str, level: int) -> List[Any]:
return [
storage.get_json_from_object(key)
for key in storage.list_objects(f"BLOCK/{block_id}-l{level}")
]
def list_registered_transaction_types() -> List[Dict[str, Any]]:
return [
storage.get_json_from_object(txn_type)
for txn_type in storage.list_objects(f"{FOLDER}/")
]
def test_list_objects_calls_storage_list_objects_with_params(self):
storage.storage.list_objects = MagicMock()
storage.list_objects("prefix")
storage.storage.list_objects.assert_called_once_with("test", "prefix")
