def perform_api_key_migration_v1_if_necessary() -> None:
"""Checks if an api key migration needs to be performed, and does so if necessary"""
try:
if storage.get(f"{FOLDER}/{MIGRATION_V1}") == b"1":
# Migration was previously performed. No action necessary
return
except exceptions.NotFound:
pass
_log.info("Api key migration required. Performing now")
valid_keys = storage.list_objects(prefix=FOLDER)
regular_keys = list(
filter(lambda x: not x.startswith("KEYS/INTERCHAIN/"), valid_keys))
interchain_keys = list(
filter(lambda x: x.startswith("KEYS/INTERCHAIN/"), valid_keys))
for key in regular_keys:
_log.info(f"Migrating {key}")
api_key = api_key_model.new_from_legacy(
storage.get_json_from_object(key), interchain_dcid="")
save_api_key(api_key)
for key in interchain_keys:
_log.info(f"Migrating interchain key {key}")
interchain_dcid = key[key.find("KEYS/INTERCHAIN/") +
16:] # Get the interchain dcid from the key
api_key = api_key_model.new_from_legacy(
storage.get_json_from_object(key), interchain_dcid=interchain_dcid)
save_api_key(api_key)
# Save migration marker once complete
storage.put(f"{FOLDER}/{MIGRATION_V1}", b"1")
_log.info("Api key migration v1 complete")
def search(
folder: str,
query: Optional[dict] = None,
q: Optional[str] = None,
get_all: bool = False,
sort: Optional[str] = None,
offset: Optional[int] = None,
limit: Optional[int] = None,
should_parse: bool = True,
) -> "ESSearch":
"""invoke queries on elastic search indexes built with #set. Return the full storage stored object
Args:
query: Elastic search query. The search definition using the ES Query DSL.
q: Query in the Lucene query string syntax
Returns:
storage objects matching search query
"""
hits_pages = get_index_only(folder, query, q, get_all, sort, offset, limit)
_log.info(f"Pages: {hits_pages}")
storage_objects = []
for hit in hits_pages["hits"]:
storage_id = hit["_source"][S3_OBJECT_ID] # get the id
# get the folder
storage_folder = hit["_source"][S3_OBJECT_FOLDER]
storage_object = storage.get_json_from_object(f"{storage_folder}/{storage_id}") # pull the object from storage
storage_objects.append(storage_object) # add to the result set
# Parse transactions if should_parse is flagged, and the transactions array exists
if storage_object["dcrn"] == schema.DCRN.Block_L1_At_Rest.value and should_parse:
for index, transaction in enumerate(storage_object["transactions"]):
storage_object["transactions"][index] = json.loads(transaction)
return {"results": storage_objects, "total": hits_pages["total"]}
def check_confirmations() -> None:
last_confirmed_block = get_last_confirmed_block()
last_confirmed_block_number = last_confirmed_block["block_id"]
last_created_block = get_last_block_number()
_log.info(
f"[L5] Last confirmed block is {last_confirmed_block_number}, last created block is {last_created_block}"
)
if int(last_confirmed_block_number) < int(last_created_block):
# Check for confirmations
next_block_to_confirm = int(last_confirmed_block_number) + 1
block_key = f"BLOCK/{next_block_to_confirm}"
block = l5_block_model.new_from_at_rest(
storage.get_json_from_object(block_key))
for txn_hash in block.transaction_hash:
try:
if _interchain_client.is_transaction_confirmed(txn_hash):
finalize_block(block, last_confirmed_block, txn_hash)
# Stop execution here!
return
except exceptions.RPCTransactionNotFound:
# If transaction not found, it may have been dropped, so we remove it from the block
block.transaction_hash.remove(txn_hash)
# If execution did not stop, the block is not confirmed.
if _interchain_client.should_retry_broadcast(block.block_last_sent_at):
broadcast_to_public_chain(block)
def _generate_l5_verification_indexes() -> None:
client = _get_redisearch_index_client(Indexes.verification.value)
client.drop_index()
try:
client.create_index(
[
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
redisearch.TagField("dc_id"),
]
)
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"): # We don't care if index already exists
raise
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/([0-9]+)-([Ll])5(.*)$")
for block_path in block_paths:
if LEVEL == "1" and BROADCAST_ENABLED and re.search(pattern, block_path):
if not client.redis.sismember(L5_BLOCK_MIGRATION_KEY, block_path):
raw_block = storage.get_json_from_object(block_path)
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.verification.value, block_path.split("/")[1], block.export_as_search_index())
client.redis.sadd(L5_NODES, block.dc_id)
client.redis.sadd(L5_BLOCK_MIGRATION_KEY, block_path)
else:
_log.info(f"Skipping already indexed L5 block {block_path}")
def _generate_block_indexes_from_scratch() -> None:
client = _get_redisearch_index_client(Indexes.block.value)
client.create_index([
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
])
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/[0-9]+$")
for block_path in block_paths:
if re.search(pattern, block_path):
_log.info(f"Adding index for {block_path}")
raw_block = storage.get_json_from_object(block_path)
block = cast("model.BlockModel", None)
if LEVEL == "1":
block = l1_block_model.new_from_stripped_block(raw_block)
elif LEVEL == "2":
block = l2_block_model.new_from_at_rest(raw_block)
elif LEVEL == "3":
block = l3_block_model.new_from_at_rest(raw_block)
elif LEVEL == "4":
block = l4_block_model.new_from_at_rest(raw_block)
elif LEVEL == "5":
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.block.value, block.block_id,
block.export_as_search_index())
def get_broadcast_dto(higher_level: int, block_id: str) -> Dict[str, Any]:
"""Get the broadcast dto for a block to a certain level
Args:
higher_level: (2-5)
block_id: block_id used to locate block in storage
Returns:
Broadcast DTO of requested
"""
l1_block = storage.get_json_from_object(f"{FOLDER}/{block_id}")
if higher_level == 2:
return l1_block_model.export_broadcast_dto(l1_block)
else:
required_verification_count = dragonnet_config.DRAGONNET_CONFIG[
f"l{higher_level - 1}"]["nodesRequired"]
verification_blocks = get_verifications_for_l1_block(
block_id, (higher_level - 1))
if len(verification_blocks) < required_verification_count:
raise exceptions.NotEnoughVerifications(
f"This chain requires {required_verification_count} level {higher_level - 1} verifications, but only {len(verification_blocks)} were found. (Probably just need to wait)" # noqa: B950
)
if higher_level == 3:
return l2_block_model.export_broadcast_dto(verification_blocks,
l1_block)
elif higher_level == 4:
return l3_block_model.export_broadcast_dto(verification_blocks,
l1_block)
elif higher_level == 5:
return l4_block_model.export_broadcast_dto(verification_blocks)
else:
raise exceptions.InvalidNodeLevel(
f"Level {higher_level} is not valid for getting a broadcast DTO (Only allowed 2-5)"
)
def check_confirmations() -> None:
last_confirmed_block = get_last_confirmed_block()
last_confirmed_block_number = last_confirmed_block["block_id"]
last_created_block = get_last_block_number()
_log.info(f"[L5] Last confirmed block is {last_confirmed_block_number}, last created block is {last_created_block}")
if int(last_confirmed_block_number) < int(last_created_block):
# Check for confirmations
next_block_to_confirm = int(last_confirmed_block_number) + 1
block_key = f"BLOCK/{next_block_to_confirm}"
block = l5_block_model.new_from_at_rest(storage.get_json_from_object(block_key))
for txn_hash in block.transaction_hash:
confirmed = INTERCHAIN.is_transaction_confirmed(txn_hash)
# If this function returned the transaction hash, that means it was dropped so we
# remove it from the block, otherwise if it returned true that means it was confirmed.
# When broadcast retry occurs, the removed hashes will be removed in storage.
if isinstance(confirmed, str):
block.transaction_hash.remove(txn_hash)
elif confirmed:
finalize_block(block, last_confirmed_block, txn_hash)
# Stop execution here!
return
# If execution did not stop in the above for loop, we know that the block is not confirmed.
retry_broadcast_if_necessary(block)
def get_registered_transaction_type(transaction_type: str) -> transaction_type_model.TransactionTypeModel:
"""
Searches for a registered transaction type by name
:param transaction_type
"""
_log.info(f"Attempting to get type {transaction_type}")
result = storage.get_json_from_object(f"{FOLDER}/{transaction_type}")
return transaction_type_model.new_from_at_rest(result)
def update_api_key_v1(key_id: str, nickname: str) -> None:
"""Updates the nickname for an existing key
Args:
key_id: ID of api key to update
nickname: new nickname for the given key
"""
key = storage.get_json_from_object(f"KEYS/{key_id}")
key["nickname"] = nickname
storage.put_object_as_json(f"KEYS/{key_id}", key)
def list_registered_transaction_types_v1() -> Dict[str, Any]:
"""
Lists out the current registered transaction types
"""
_log.info("Listing out existing transaction types")
transaction_types = redis.smembers_sync(transaction_type_dao.TYPE_LIST_KEY)
transaction_type_list = [storage.get_json_from_object(f"{transaction_type_dao.FOLDER}/TYPES/{txn_type}") for txn_type in transaction_types]
return {"transaction_types": transaction_type_list}
def update_registration(new_data: dict) -> None:
try:
_log.info(f"[MATCHMAKING] Putting matchmaking config in storage: {new_data}")
matchmaking_config = storage.get_json_from_object("MATCHMAKING_CONFIG.json")
matchmaking_config.update(new_data)
storage.put_object_as_json("MATCHMAKING_CONFIG.json", matchmaking_config)
register()
except Exception:
raise exceptions.MatchmakingError("Failure updating matchmaking data")
def get_interchain_client(blockchain: str,
name: str) -> "model.InterchainModel":
"""Get a specific interchain client
Args:
blockchain: the blockchain of the desired client (i.e. bitcoin, ethereum, etc)
name: the name (id) of the network to get (user defined on the creation of the interchain)
Raises:
exceptions.NotFound: When the requested client can't be found
"""
if blockchain == "bitcoin":
return btc.new_from_at_rest(
storage.get_json_from_object(f"{FOLDER}/bitcoin/{name}"))
elif blockchain == "ethereum":
return eth.new_from_at_rest(
storage.get_json_from_object(f"{FOLDER}/ethereum/{name}"))
else:
raise exceptions.NotFound(
f"Blockchain network {blockchain} is not supported")
def get_last_block_proof() -> Dict[str, str]:
"""Return the last closed block's ID and hash
Returns:
Result of last closed block lookup (empty dictionary if not found)
"""
try:
return storage.get_json_from_object(f"{FOLDER}/{LAST_CLOSED_KEY}")
except exceptions.NotFound:
return {}
def get_block_by_id_v1(block_id: str, parse: bool = False) -> Dict[str, Any]:
"""Searches for a block by a specific block ID
Args:
block_id: The block id to get
parse: whether or not to parse the result automatically
"""
raw_block = storage.get_json_from_object(f"BLOCK/{block_id}")
if parse and raw_block["dcrn"] == schema.DCRN.Block_L1_At_Rest.value:
for index, transaction in enumerate(raw_block["transactions"]):
raw_block["transactions"][index] = json.loads(transaction)
return raw_block
def get_auth_key(auth_key_id: str, interchain: bool) -> Optional[str]:
"""Retrieve the auth key corresponding to a key id
Args:
auth_key_id: The key id to grab (if interchain, this is the interchain dcid)
interchain: boolean whether the key to get is an interchain key or not
Returns:
The base64 encoded auth key string corresponding to the id (None if not found)
"""
response = None
try:
if interchain:
response = storage.get_json_from_object(
f"KEYS/INTERCHAIN/{auth_key_id}")
else:
response = storage.get_json_from_object(f"KEYS/{auth_key_id}")
except exceptions.NotFound:
pass
if response:
return response.get("key")
return None
def get_verifications_for_l1_block(block_id: str,
level: int) -> List[Dict[str, Any]]:
try:
keys = list(
broadcast_functions.
get_receieved_verifications_for_block_and_level_sync(
block_id, level))
if len(keys) != 0:
for i in range(len(keys)):
keys[
i] = f"{FOLDER}/{block_id}-l{level}-{keys[i].decode('utf8')}"
return list(
map(lambda key: storage.get_json_from_object(key), keys))
except Exception:
pass
# Only fall back to listing from storage if we don't have verifications already saved in redis
prefix = f"{FOLDER}/{block_id}-l{level}"
keys = storage.list_objects(prefix)
_log.info(f"Verification keys by prefix {prefix}: {keys}")
return [] if len(keys) == 0 else list(
map(lambda key: storage.get_json_from_object(key), keys))
def get_verifications_for_l1_block(block_id: str,
level: int) -> List[Dict[str, Any]]:
try:
keys = list(
broadcast_functions.
get_receieved_verifications_for_block_and_level_sync(
block_id, level))
if len(keys) != 0:
for i in range(len(keys)):
keys[i] = f"{FOLDER}/{block_id}-l{level}-{keys[i]}"
return [storage.get_json_from_object(x) for x in keys]
except Exception:
_log.exception(
"Error getting verifications from cached list. Falling back to direct storage list"
)
# Only fall back to listing from storage if we don't have verifications already saved in redis
prefix = f"{FOLDER}/{block_id}-l{level}"
keys = storage.list_objects(prefix)
_log.info(f"Verification keys by prefix {prefix}: {keys}")
return [] if len(keys) == 0 else [
storage.get_json_from_object(key) for key in keys
]
def _generate_smart_contract_indexes() -> None:
delete_index(Indexes.smartcontract.value) # Always generate smart contract indexes from scratch by dropping existing ones
client = _get_redisearch_index_client(Indexes.smartcontract.value)
client.create_index([redisearch.TagField("sc_name")])
# Find what smart contracts exist in storage
_log.info("Listing all smart contracts in storage")
sc_object_paths = storage.list_objects("SMARTCONTRACT/")
pattern = re.compile(r"SMARTCONTRACT\/.{36}\/metadata\.json$")
for sc in sc_object_paths:
if re.search(pattern, sc):
sc_model = smart_contract_model.new_from_at_rest(storage.get_json_from_object(sc))
_log.info(f"Adding index for smart contract {sc_model.id} ({sc_model.txn_type})")
put_document(Indexes.smartcontract.value, sc_model.id, sc_model.export_as_search_index())
def list_api_keys(include_interchain: bool) -> List[api_key_model.APIKeyModel]:
"""Retrieve a list of api keys
Args:
include_interchain: whether or not to include interchain api keys
Returns:
List of api key models
"""
# Get keys from storage, excluding migration marker and interchain keys
return_list = []
for key in storage.list_objects(prefix=FOLDER):
if (MIGRATION_V1 in key) or (key.startswith("KEYS/INTERCHAIN")
and not include_interchain):
continue
return_list.append(
api_key_model.new_from_at_rest(storage.get_json_from_object(key)))
return return_list
def get_default_interchain_client() -> "model.InterchainModel":
"""Get the interchain model which has been set as the default for this chain
Returns:
Instantiated InterchainModel
Raises:
exceptions.NotFound: When default has not been set, or set default cannot be found
NotImplementedError: WHen the saved default is a bad version
"""
default_dto = storage.get_json_from_object(f"{FOLDER}/default")
if default_dto.get("version") == "1":
return get_interchain_client(default_dto.get("blockchain"),
default_dto.get("name"))
else:
raise NotImplementedError(
f"Default dto error. Version {default_dto.get('version')} not supported"
)
def list_contracts_v1() -> Dict[str, List[Dict[str, Any]]]:
"""Function used by the smartcontract endpoint with method GET.
Returns a list of all smart contracts.
Returns:
The search results of the query specified.
"""
sc_list = smart_contract_dao.list_all_contract_ids()
sc_metadata = []
for sc_id in sc_list:
try:
sc_metadata.append(
storage.get_json_from_object(
f"{smart_contract_dao.FOLDER}/{sc_id}/metadata.json"))
except exceptions.NotFound: # If smart contract metadata is not found, simply ignore it and don't add it to the list
pass
return {"smart_contracts": sc_metadata}
def get_api_key_v1(key_id: str) -> Dict[str, Any]:
"""Returns the api key information (without the actual key itself) for a key id
Args:
key_id: ID of api key to get
hide_key: remove the api key from the returned key
Returns:
API key ID and registration timestamp (if any)
"""
if key_id.startswith("SC_") or key_id.startswith(
"WEB_") or key_id.startswith("INTERCHAIN"):
raise exceptions.NotFound(f"api key with ID {key_id} not found")
key = storage.get_json_from_object(f"KEYS/{key_id}")
return {
"id": str(key["id"]),
"registration_time": int(key["registration_time"]),
"nickname": str(key.get("nickname") or "")
}
def get_api_key(key_id: str, interchain: bool) -> api_key_model.APIKeyModel:
"""Get an api key from storage
Args:
key_id: The key id to fetch (public chain id if interchain)
interchain: Whether or not this is an interchain key
"""
# Explicitly don't allow permission keys with slashes (may be malicious)
if "/" in key_id:
raise exceptions.NotFound
model = api_key_model.new_from_at_rest(
storage.get_json_from_object(
f"{INTERCHAIN_FOLDER if interchain else FOLDER}/{key_id}"))
if model.interchain != interchain: # Double check the interchain value of the key is what we expect; otherwise panic
raise RuntimeError(
f"Bad interchain key {key_id} found. Expected interchain: {interchain} but got {model.interchain}"
)
return model
def list_interchain_clients(blockchain: str) -> List["model.InterchainModel"]:
"""Get all of the interchain clients for a specific blockchain type
Args:
blockchain: The blockchain of the desired clients to get
Returns:
List of instantiated interchain clients for the specified blockchain
"""
from_rest_function: Any = None
if blockchain == "bitcoin":
from_rest_function = btc.new_from_at_rest
elif blockchain == "ethereum":
from_rest_function = eth.new_from_at_rest
else:
raise exceptions.NotFound(
f"Blockchain network {blockchain} is not supported")
return [
from_rest_function(storage.get_json_from_object(x))
for x in storage.list_objects(f"{FOLDER}/{blockchain}/")
]
def get_pending_l4_blocks(block_id: str) -> List[str]:
all_waiting_verification_keys = storage.list_objects(f"BROADCAST/TO_BROADCAST/{block_id}")
l4_blocks = []
for key in all_waiting_verification_keys:
record_list = storage.get_json_from_object(key)
for record in record_list:
item = {
"l1_dc_id": record["header"]["l1_dc_id"],
"l1_block_id": record["header"]["l1_block_id"],
"l4_dc_id": record["header"]["dc_id"],
"l4_block_id": record["header"]["block_id"],
"l4_proof": record["proof"]["proof"],
}
if record.get("is_invalid"):
item["is_invalid"] = record.get("is_invalid")
l4_blocks.append(json.dumps(item, separators=(",", ":")))
return l4_blocks
def query_interchain_broadcasts_v1(block_id: str) -> List[Any]:
"""Return the subsequent broadcasts to other L5 networks"""
_log.info(f"Getting subsequent L5 verifications for {block_id}")
results = []
l5_block = None
l5_verifications = _get_verification_records(block_id, 5)
if len(l5_verifications) > 0:
l5_block = cast(List[Any], l5_verifications)[0]
timestamp = l5_block["header"]["timestamp"]
dc_id = l5_block["header"]["dc_id"]
l5_nodes = redisearch._get_redisearch_index_client(
redisearch.Indexes.verification.value).redis.smembers(
redisearch.L5_NODES)
results = [
_query_l5_verification(l5_dc_id.decode("utf-8"), timestamp)
for l5_dc_id in l5_nodes if l5_dc_id.decode("utf-8") != dc_id
]
return ([l5_block] if l5_block else []) + [
storage.get_json_from_object(f"BLOCK/{x}")
for x in results if x is not None
]
def get_api_key_list_v1() -> Dict[str, List[Dict[str, Any]]]:
"""
Gets the list of api key IDs
Returns:
List of API keys
"""
keys = storage.list_objects(prefix=FOLDER)
valid_keys = list(
filter(
lambda x: not x.startswith("KEYS/WEB_") and not x.startswith(
"KEYS/SC_") and not x.startswith("KEYS/INTERCHAIN"), keys))
returned_keys = []
for key in valid_keys:
resp = storage.get_json_from_object(key)
returned_keys.append({
"id":
str(resp["id"]),
"registration_time":
int(resp["registration_time"]),
"nickname":
str(resp.get("nickname") or "")
})
return {"keys": returned_keys}
def _generate_block_indexes() -> None:
client = _get_redisearch_index_client(Indexes.block.value)
try:
client.create_index([
redisearch.NumericField("block_id", sortable=True),
redisearch.NumericField("prev_id", sortable=True),
redisearch.NumericField("timestamp", sortable=True),
])
except redis.exceptions.ResponseError as e:
if not str(e).startswith("Index already exists"
): # We don't care if index already exists
raise
_log.info("Listing all blocks in storage")
block_paths = storage.list_objects("BLOCK/")
pattern = re.compile(r"BLOCK\/[0-9]+$")
for block_path in block_paths:
if re.search(pattern, block_path):
# do a check to see if this block was already marked as indexed
if not client.redis.sismember(BLOCK_MIGRATION_KEY, block_path):
_log.info(f"Adding index for {block_path}")
raw_block = storage.get_json_from_object(block_path)
block = cast("model.BlockModel", None)
if LEVEL == "1":
block = l1_block_model.new_from_stripped_block(raw_block)
elif LEVEL == "2":
block = l2_block_model.new_from_at_rest(raw_block)
elif LEVEL == "3":
block = l3_block_model.new_from_at_rest(raw_block)
elif LEVEL == "4":
block = l4_block_model.new_from_at_rest(raw_block)
elif LEVEL == "5":
block = l5_block_model.new_from_at_rest(raw_block)
put_document(Indexes.block.value, block.block_id,
block.export_as_search_index())
client.redis.sadd(BLOCK_MIGRATION_KEY, block_path)
else:
_log.info(f"Skipping already indexed block {block_path}")
def _level_records(block_id: str, level: int) -> List[Any]:
return [
storage.get_json_from_object(key)
for key in storage.list_objects(f"BLOCK/{block_id}-l{level}")
]
def list_registered_transaction_types() -> List[Dict[str, Any]]:
return [
storage.get_json_from_object(txn_type)
for txn_type in storage.list_objects(f"{FOLDER}/")
]
