def _generate_block_indexes_from_scratch() -> None:
client = _get_redisearch_index_client(Indexes.block.value)
client.create_index([
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
])
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/[0-9]+$")
for block_path in block_paths:
if re.search(pattern, block_path):
_log.info(f"Adding index for {block_path}")
raw_block = storage.get_json_from_object(block_path)
block = cast("model.BlockModel", None)
if LEVEL == "1":
block = l1_block_model.new_from_stripped_block(raw_block)
elif LEVEL == "2":
block = l2_block_model.new_from_at_rest(raw_block)
elif LEVEL == "3":
block = l3_block_model.new_from_at_rest(raw_block)
elif LEVEL == "4":
block = l4_block_model.new_from_at_rest(raw_block)
elif LEVEL == "5":
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.block.value, block.block_id,
block.export_as_search_index())
def _generate_l5_verification_indexes() -> None:
client = _get_redisearch_index_client(Indexes.verification.value)
client.drop_index()
try:
client.create_index(
[
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
redisearch.TagField("dc_id"),
]
)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/([0-9]+)-([Ll])5(.*)$")
for block_path in block_paths:
if LEVEL == "1" and BROADCAST_ENABLED and re.search(pattern, block_path):
if not client.redis.sismember(L5_BLOCK_MIGRATION_KEY, block_path):
raw_block = storage.get_json_from_object(block_path)
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.verification.value, block_path.split("/")[1], block.export_as_search_index())
client.redis.sadd(L5_NODES, block.dc_id)
client.redis.sadd(L5_BLOCK_MIGRATION_KEY, block_path)
else:
_log.info(f"Skipping already indexed L5 block {block_path}")
def check_confirmations() -> None:
last_confirmed_block = get_last_confirmed_block()
last_confirmed_block_number = last_confirmed_block["block_id"]
last_created_block = get_last_block_number()
_log.info(
f"[L5] Last confirmed block is {last_confirmed_block_number}, last created block is {last_created_block}"
)
if int(last_confirmed_block_number) < int(last_created_block):
# Check for confirmations
next_block_to_confirm = int(last_confirmed_block_number) + 1
block_key = f"BLOCK/{next_block_to_confirm}"
block = l5_block_model.new_from_at_rest(
storage.get_json_from_object(block_key))
for txn_hash in block.transaction_hash:
try:
if _interchain_client.is_transaction_confirmed(txn_hash):
finalize_block(block, last_confirmed_block, txn_hash)
# Stop execution here!
return
except exceptions.RPCTransactionNotFound:
# If transaction not found, it may have been dropped, so we remove it from the block
block.transaction_hash.remove(txn_hash)
# If execution did not stop, the block is not confirmed.
if _interchain_client.should_retry_broadcast(block.block_last_sent_at):
broadcast_to_public_chain(block)
def process_receipt_v1(block_dto: Dict[str, Any]) -> None:
if not block_dto:
raise exceptions.ValidationException("block_dto missing")
_log.info(
f"[RECEIPT] Got receipt from L{block_dto['header']['level']}: {block_dto}"
)
block_model = cast("model.BlockModel",
None) # This will always get defined, or it will raise
level_received_from: int = block_dto["header"]["level"]
if level_received_from == 2:
block_model = l2_block_model.new_from_at_rest(block_dto)
elif level_received_from == 3:
block_model = l3_block_model.new_from_at_rest(block_dto)
elif level_received_from == 4:
block_model = l4_block_model.new_from_at_rest(block_dto)
elif level_received_from == 5:
block_model = l5_block_model.new_from_at_rest(block_dto)
else:
raise exceptions.InvalidNodeLevel("Unsupported level receipt")
_log.info(f"Block model {block_model.__dict__}")
l1_block_id_set = block_model.get_associated_l1_block_id()
_log.info(
f"Processing receipt for blocks {l1_block_id_set} from L{level_received_from}"
)
for l1_block_id in l1_block_id_set:
# Check that the chain which sent this receipt is in our claims, and that this L1 block is accepting receipts for this level
validations = matchmaking.get_claim_check(
l1_block_id)["validations"][f"l{level_received_from}"]
if (
block_model.dc_id in validations
) and broadcast_functions.is_block_accepting_verifications_from_level(
l1_block_id, level_received_from):
_log.info(
f"Verified that block {l1_block_id} was sent. Inserting receipt"
)
storage_location = broadcast_functions.verification_storage_location(
l1_block_id, level_received_from, block_model.dc_id)
storage.put_object_as_json(storage_location,
block_model.export_as_at_rest())
# Set new receipt for matchmaking claim check
try:
block_id = block_model.block_id
proof = block_model.proof
dc_id = block_model.dc_id
matchmaking.add_receipt(l1_block_id, level_received_from,
dc_id, block_id, proof)
except Exception:
_log.exception("matchmaking add_receipt failed!")
# Update the broadcast system about this receipt
broadcast_functions.set_receieved_verification_for_block_from_chain_sync(
l1_block_id, level_received_from, block_model.dc_id)
else:
_log.warning(
f"Chain {block_model.dc_id} (level {level_received_from}) returned a receipt that wasn't expected (possibly expired?) for block {l1_block_id}. Rejecting receipt" # noqa: B950
)
raise exceptions.NotAcceptingVerifications(
f"Not accepting verifications for block {l1_block_id} from {block_model.dc_id}"
)
def check_confirmations() -> None:
last_confirmed_block = get_last_confirmed_block()
last_confirmed_block_number = last_confirmed_block["block_id"]
last_created_block = get_last_block_number()
_log.info(f"[L5] Last confirmed block is {last_confirmed_block_number}, last created block is {last_created_block}")
if int(last_confirmed_block_number) < int(last_created_block):
# Check for confirmations
next_block_to_confirm = int(last_confirmed_block_number) + 1
block_key = f"BLOCK/{next_block_to_confirm}"
block = l5_block_model.new_from_at_rest(storage.get_json_from_object(block_key))
for txn_hash in block.transaction_hash:
confirmed = INTERCHAIN.is_transaction_confirmed(txn_hash)
# If this function returned the transaction hash, that means it was dropped so we
# remove it from the block, otherwise if it returned true that means it was confirmed.
# When broadcast retry occurs, the removed hashes will be removed in storage.
if isinstance(confirmed, str):
block.transaction_hash.remove(txn_hash)
elif confirmed:
finalize_block(block, last_confirmed_block, txn_hash)
# Stop execution here!
return
# If execution did not stop in the above for loop, we know that the block is not confirmed.
retry_broadcast_if_necessary(block)
def _generate_block_indexes() -> None:
client = _get_redisearch_index_client(Indexes.block.value)
try:
client.create_index([
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
])
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"
): # We don't care if index already exists
raise
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/[0-9]+$")
for block_path in block_paths:
if re.search(pattern, block_path):
# do a check to see if this block was already marked as indexed
if not client.redis.sismember(BLOCK_MIGRATION_KEY, block_path):
_log.info(f"Adding index for {block_path}")
raw_block = storage.get_json_from_object(block_path)
block = cast("model.BlockModel", None)
if LEVEL == "1":
block = l1_block_model.new_from_stripped_block(raw_block)
elif LEVEL == "2":
block = l2_block_model.new_from_at_rest(raw_block)
elif LEVEL == "3":
block = l3_block_model.new_from_at_rest(raw_block)
elif LEVEL == "4":
block = l4_block_model.new_from_at_rest(raw_block)
elif LEVEL == "5":
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.block.value, block.block_id,
block.export_as_search_index())
client.redis.sadd(BLOCK_MIGRATION_KEY, block_path)
else:
_log.info(f"Skipping already indexed block {block_path}")
