def start() -> None:
"""
Ran by the webserver before it boots
"""
try:
# Chains are given HMAC keys when created. If found, we write them to storage.
key_id = secrets.get_dc_secret("hmac-id")
_log.info(
"HMAC keys were given to this chain on-boot. Writing them to storage."
)
storage.put_object_as_json(
f"KEYS/{key_id}", {
"id": key_id,
"key": secrets.get_dc_secret("hmac-key"),
"root": True,
"registration_time": 0
})
except exceptions.NotFound:
_log.info(
"No HMAC keys were given to this chain on-boot. Skipping credential storage write."
)
_log.info("Checking if redisearch indexes need to be regenerated")
redisearch.generate_indexes_if_necessary()
_log.info("Finish pre-boot successful")
def put_index_in_storage(folder: str, namespace: str, data_model: "model.Model") -> None:
"""Store data model content in storage, while creating an elastic search index
Args:
folder: the storage folder to use
namespace: the storage namespace to use
data_model: the data model to export
"""
_set_elastic_search_client_if_necessary()
_log.info(f"[DAO] SET -> {namespace}")
indexable_object = data_model.export_as_search_index()
full_object = data_model.export_as_at_rest()
_log.info(f"[DAO] ES => INDEXABLE_OBJECT: {indexable_object}")
_log.info(f"[DAO] ES => INDEXING OBJECT: namespace: {namespace}")
_es_client.index(index=build_index(folder=folder), doc_type="_doc", id=namespace, body=json.dumps(indexable_object, separators=(",", ":")))
_log.info(f"[DAO] storage => UPLOADING OBJECT: {full_object}")
try:
key = f"{folder.upper()}/{namespace}"
try:
# Smart contracts use a seperate key for metadata
if data_model.is_sc_model: # noqa: T484 property might not exist, which is explicitly caught
key += "/metadata.json"
except Exception:
pass
storage.put_object_as_json(key, full_object)
except Exception:
_es_client.delete(index=build_index(folder=folder), doc_type="_doc", id=namespace)
def process_receipt_v1(block_dto: Dict[str, Any]) -> None:
if not block_dto:
raise exceptions.ValidationException("block_dto missing")
_log.info(
f"[RECEIPT] Got receipt from L{block_dto['header']['level']}: {block_dto}"
)
block_model = cast("model.BlockModel",
None) # This will always get defined, or it will raise
level_received_from: int = block_dto["header"]["level"]
if level_received_from == 2:
block_model = l2_block_model.new_from_at_rest(block_dto)
elif level_received_from == 3:
block_model = l3_block_model.new_from_at_rest(block_dto)
elif level_received_from == 4:
block_model = l4_block_model.new_from_at_rest(block_dto)
elif level_received_from == 5:
block_model = l5_block_model.new_from_at_rest(block_dto)
else:
raise exceptions.InvalidNodeLevel("Unsupported level receipt")
_log.info(f"Block model {block_model.__dict__}")
l1_block_id_set = block_model.get_associated_l1_block_id()
_log.info(
f"Processing receipt for blocks {l1_block_id_set} from L{level_received_from}"
)
for l1_block_id in l1_block_id_set:
# Check that the chain which sent this receipt is in our claims, and that this L1 block is accepting receipts for this level
validations = matchmaking.get_claim_check(
l1_block_id)["validations"][f"l{level_received_from}"]
if (
block_model.dc_id in validations
) and broadcast_functions.is_block_accepting_verifications_from_level(
l1_block_id, level_received_from):
_log.info(
f"Verified that block {l1_block_id} was sent. Inserting receipt"
)
storage_location = broadcast_functions.verification_storage_location(
l1_block_id, level_received_from, block_model.dc_id)
storage.put_object_as_json(storage_location,
block_model.export_as_at_rest())
# Set new receipt for matchmaking claim check
try:
block_id = block_model.block_id
proof = block_model.proof
dc_id = block_model.dc_id
matchmaking.add_receipt(l1_block_id, level_received_from,
dc_id, block_id, proof)
except Exception:
_log.exception("matchmaking add_receipt failed!")
# Update the broadcast system about this receipt
broadcast_functions.set_receieved_verification_for_block_from_chain_sync(
l1_block_id, level_received_from, block_model.dc_id)
else:
_log.warning(
f"Chain {block_model.dc_id} (level {level_received_from}) returned a receipt that wasn't expected (possibly expired?) for block {l1_block_id}. Rejecting receipt" # noqa: B950
)
raise exceptions.NotAcceptingVerifications(
f"Not accepting verifications for block {l1_block_id} from {block_model.dc_id}"
)
def deploy_to_openfaas(self) -> None:
"""Deploy this job's smart contract to OpenFaaS and update the faas_spec
Returns:
None, or throws exceptions.InternalServerError
"""
_log.info("Deploying to OpenFaaS cluster")
spec = self.get_openfaas_spec()
requests_method = requests.post if self.model.task_type == "create" else requests.put
response = requests_method(
f"{FAAS_GATEWAY}/system/functions",
headers={"Authorization": faas.get_faas_auth()},
json=spec)
_log.info(f"Deployment status: {response.status_code}")
if response.status_code not in [200, 202]:
_log.info(f"OpenFaaS deploy failure: {response.status_code}")
self.model.set_state(state=self.end_error_state,
msg="Failed message state")
raise exceptions.ContractException(
"Contract function deployment failure")
_log.info("Saving faas_spec.json to storage")
if os.environ["STORAGE_TYPE"].lower() == "disk":
os.setuid(1000)
storage.put_object_as_json(
key=f"SMARTCONTRACT/{self.model.id}/faas_spec.json", value=spec)
def register_new_auth_key(smart_contract: bool = False,
auth_key: str = "",
auth_key_id: str = "",
nickname: str = "") -> Dict[str, Any]:
"""Register a new auth key for use with the chain
Args:
smart_contract: whether it should generate a key for a smart contract
auth_key: (optional) specify an auth_key to use (must be in conjunction with auth_key_id)
auth_key_id: (optional) specify an auth_key_id to use (must be in conjunction with auth_key_id)
Returns:
Dictionary where 'id' is the new auth_key_id and 'key' is the new auth_key
Raises:
ValueError when only one of auth_key or auth_key_id are defined, but not both
"""
if (not auth_key) or (not auth_key_id):
# Check that both are not specified (don't allow only auth_key or auth_key_id to be individually provided)
if auth_key or auth_key_id:
raise ValueError(
"auth_key and auth_key_id must both be specified together if provided"
)
# Python do-while
while True:
auth_key_id = gen_auth_key_id(smart_contract)
# Make sure this randomly generated key id doesn't already exist
if not get_auth_key(auth_key_id, False):
break
auth_key = gen_auth_key()
register = {
"key": auth_key,
"id": auth_key_id,
"registration_time": int(time.time()),
"nickname": nickname
}
storage.put_object_as_json(f"KEYS/{auth_key_id}", register)
return register
def output_to_heap(contract_response: Dict[str, Any],
contract_id: str) -> None:
for key, value in contract_response.items():
_log.info(
f"[SC-RETURN] Heap set request {key} : {value} to {contract_id}")
storage.put_object_as_json(f"SMARTCONTRACT/{contract_id}/HEAP/{key}",
value)
def create_new_transaction_type(txn_type_model: transaction_type_model.TransactionTypeModel) -> None:
"""Save a new transaction type model"""
txn_type_dto = txn_type_model.export_as_at_rest()
_log.info(f"Adding transaction index for {txn_type_model.txn_type}")
redisearch.create_transaction_index(txn_type_model.txn_type, txn_type_model.custom_indexes)
_log.debug(f"Queuing for activation")
redis.lpush_sync(QUEUED_TXN_TYPES, txn_type_model.txn_type)
_log.debug(f"Adding the transaction type to storage")
storage.put_object_as_json(f"{FOLDER}/{txn_type_model.txn_type}", txn_type_dto)
def update_registration(new_data: dict) -> None:
try:
_log.info(f"[MATCHMAKING] Putting matchmaking config in storage: {new_data}")
matchmaking_config = storage.get_json_from_object("MATCHMAKING_CONFIG.json")
matchmaking_config.update(new_data)
storage.put_object_as_json("MATCHMAKING_CONFIG.json", matchmaking_config)
register()
except Exception:
raise exceptions.MatchmakingError("Failure updating matchmaking data")
def update_api_key_v1(key_id: str, nickname: str) -> None:
"""Updates the nickname for an existing key
Args:
key_id: ID of api key to update
nickname: new nickname for the given key
"""
key = storage.get_json_from_object(f"KEYS/{key_id}")
key["nickname"] = nickname
storage.put_object_as_json(f"KEYS/{key_id}", key)
async def save_unfinished_claim(block_id: str) -> None:
"""If a claim no longer exists in Dragon Net, but we don't have all the results,
save its id for a potentially later date.
Args:
block_id: The block_id to save and remove from the broadcasting system
"""
storage.put_object_as_json(f"{STORAGE_FOLDER}/UNFINISHED/{block_id}",
{"time": time.time()})
await remove_block_from_broadcast_system_async(block_id)
def insert_block(block: "model.BlockModel") -> None:
"""
Insert new block into blockchain and ref to block's hash
"""
# Create ref to this block for the next block
last_block_ref = {"block_id": block.block_id, "proof": block.proof}
# Upload stripped block
elasticsearch.put_index_in_storage(FOLDER, block.block_id, block)
# Upload ref
storage.put_object_as_json(f"{FOLDER}/{LAST_CLOSED_KEY}", last_block_ref)
def store_l4_blocks(next_block_id_to_broadcast: str) -> None:
# Gets stringified lists of L4 blocks from different L1s
# Shape: ["{l4 block in transit}", "{l4 block in transit}"]
_log.info("[L5] Storing L4 blocks")
queue.check_and_recover_processing_if_necessary()
l4_blocks = queue.get_new_l4_blocks()
_log.info(f"[L5] Popped {len(l4_blocks)} L4 blocks off of queue")
if l4_blocks:
verified_records = verify_blocks(l4_blocks)
storage.put_object_as_json(f"BROADCAST/TO_BROADCAST/{next_block_id_to_broadcast}/{str(uuid.uuid4())}", verified_records)
# Successfully handled block popped from redis
queue.clear_processing_queue()
def insert_block(block: "model.BlockModel") -> None:
"""
Insert new block into blockchain and ref to block's hash
"""
# Create ref to this block for the next block
last_block_ref = {"block_id": block.block_id, "proof": block.proof}
# Upload stripped block
redisearch.put_document(redisearch.Indexes.block.value, block.block_id,
block.export_as_search_index())
storage.put_object_as_json(f"{FOLDER}/{block.block_id}",
block.export_as_at_rest())
# Upload ref
storage.put_object_as_json(f"{FOLDER}/{LAST_CLOSED_KEY}", last_block_ref)
def store_registered_transaction_type(
transaction_type_model: transaction_type_model.TransactionTypeModel
) -> None:
"""
Stores a new transaction type
"""
_log.info("Uploading to datastore")
storage.put_object_as_json(
f"{FOLDER}/TYPES/{transaction_type_model.txn_type}",
transaction_type_model.export_as_at_rest())
redis.sadd_sync(TYPE_LIST_KEY, cast(
str,
transaction_type_model.txn_type)) # This should defined when passed in
_log.info("Successfully uploaded new transaction type to datastore")
def save_interchain_auth_key(interchain_dcid: str, auth_key: str) -> bool:
"""Register a new interchain auth key. !This will overwrite any existing interchain key for this dcid!
Args:
interchain_dcid: chain id of the interchain sharing this key
auth_key: auth_key to add
Returns:
Boolean if successful
"""
try:
# Add the new key
register = {"key": auth_key, "registration_time": int(time.time())}
storage.put_object_as_json(f"KEYS/INTERCHAIN/{interchain_dcid}",
register)
return True
except Exception:
return False
def activate_transaction_types_if_necessary(block_id: str) -> None:
"""Activate transaction type(s) by setting them to active at a certain block number (for index regeneration purposes)
Args:
block_id: the current block id where the transaction types are being activated (if they exist)
"""
# Get all the queued transaction types
p = redis.pipeline_sync(transaction=True)
p.lrange(QUEUED_TXN_TYPES, 0, -1)
p.delete(QUEUED_TXN_TYPES)
results, _ = p.execute()
for txn_type in results:
try:
txn_type_model = get_registered_transaction_type(txn_type.decode("utf8"))
txn_type_model.active_since_block = block_id
# Save the transaction type state
storage.put_object_as_json(f"{FOLDER}/{txn_type_model.txn_type}", txn_type_model.export_as_at_rest())
except exceptions.NotFound:
pass # txn_type was probably deleted before activating. Simply ignore it
def set_default_interchain_client(blockchain: str,
name: str) -> "model.InterchainModel":
"""Set the default interchain model for this chain
Args:
blockchain: the blockchain of the desired client (i.e. bitcoin, ethereum, etc)
name: the name (id) of the network to set as default (user defined on the creation of the interchain)
Returns:
The client for the interchain which was set as default
Raises:
exceptions.NotFound: When trying to set a default to an interchain that doesn't exist on this chain
"""
# Make sure the specified interchain exists before setting as default
client = get_interchain_client(blockchain, name)
storage.put_object_as_json(f"{FOLDER}/default", {
"version": "1",
"blockchain": blockchain,
"name": name
})
return client
def finalize_block(block: l5_block_model.L5BlockModel,
last_confirmed_block: Dict[str, Any],
confirmed_txn_hash: str) -> None:
_log.info(f"[L5] Block {block.block_id} confirmed")
if last_confirmed_block["proof"].get("proof"):
block.prev_proof = last_confirmed_block["proof"]["proof"]
_log.info("[L5] Signing block")
block.transaction_hash = [confirmed_txn_hash]
block.proof = keys.get_my_keys().sign_block(block)
_log.info("[L5] Storing new block and moving pointers")
storage.put_object_as_json(f"BLOCK/{block.block_id}",
block.export_as_at_rest())
# In the future if we change/add indexes to an L5 block, it may need to be re-indexed here.
# For now, no re-indexing is necessary, only a storage update
set_last_confirmed_block(block)
# Notify L1s that contributed to L5 block
broadcast.dispatch(block)
def broadcast_to_public_chain(l5_block: l5_block_model.L5BlockModel) -> None:
_log.info("[L5] Preparing to broadcast")
# Hash the block and publish the block to a public network
public_hash = keys.get_my_keys().hash_l5_for_public_broadcast(l5_block)
transaction_hash = _interchain_client.publish_l5_hash_to_public_network(
public_hash)
_log.info(
"[L5] After Publish to public network, setting new broadcast time")
_log.info(f"[L5] transaction_hash {transaction_hash}")
# Append transaction hash to list, add network and last block sent at
l5_block.transaction_hash += [transaction_hash]
l5_block.block_last_sent_at = _interchain_client.get_current_block()
l5_block.network = INTERCHAIN_NETWORK
storage_key = f"BLOCK/{l5_block.block_id}"
_log.info(f"[L5] Adding to storage at {storage_key} and creating index")
storage.put_object_as_json(storage_key, l5_block.export_as_at_rest())
redisearch.put_document(redisearch.Indexes.block.value, l5_block.block_id,
l5_block.export_as_search_index())
def set_receipt_data(key: str, data_model: "model.BlockModel", l1_block_id: str, level: int) -> None:
"""Store receipt as JSON content in storage, while updating ES metadata to reflect relevant verifications
Args:
key: String of the storage key for the receipt block
data_model: block data model which you want to store as JSON in storage.
l1_block_id: the level 1 block id that this receipt corresponds to
level: the level of the received block
"""
_set_elastic_search_client_if_necessary()
full_object = data_model.export_as_at_rest()
# then get block from elastic and then send with incremented verification count.
upscript = {"script": f"ctx._source.l{level}_verifications += 1"}
count = 0
elastic_success = False
while count < ES_RETRY_COUNT:
try:
elastic_response = _es_client.update(index=f"{INTERNAL_ID}-block", doc_type="_doc", id=l1_block_id, body=upscript)
if int(elastic_response["_shards"]["successful"]) > 0:
_log.info("[DAO] Successfully indexed block verifications")
elastic_success = True
break
except Exception:
count += 1
if not elastic_success:
_log.error("[DAO] ES Indexing Failed, abort storage upload.")
raise exceptions.ElasticSearchFailure("Elasticsearch Index Failure")
# map through each transaction in l2 receipts to update metadata rejections if relevant
if level == 2:
bulk_rejections = {}
upscript = {"script": "ctx._source.l2_rejections += 1"}
data_model = cast("l2_block_model.L2BlockModel", data_model) # if L2, we know this is an L2 block model
for txn in data_model.validations_dict:
if not data_model.validations_dict[txn]:
bulk_rejections.update({txn: upscript})
if len(bulk_rejections.keys()) != 0:
_log.info("[DAO] Level 2 block had rejections, bulk updating rejections for transactions")
bulk_index = prepare_bulk(bulk_rejections, index=f"{data_model.get_associated_l1_dcid()}-transaction", bulk_op="update")
elasticsearch.helpers.bulk(_es_client, bulk_index)
_log.info("[DAO] STORAGE => Uploading receipt to storage")
storage.put_object_as_json(key, full_object)
def save_interchain_client(interchain_client: "model.InterchainModel") -> None:
"""Save an interchain model to storage"""
storage.put_object_as_json(
f"{FOLDER}/{interchain_client.blockchain}/{interchain_client.name}",
interchain_client.export_as_at_rest())
def save_api_key(api_key: api_key_model.APIKeyModel) -> None:
"""Save an api key model to storage"""
storage.put_object_as_json(
f"{INTERCHAIN_FOLDER if api_key.interchain else FOLDER}/{api_key.key_id}",
api_key.export_as_at_rest())
def set_last_confirmed_block(l5_block: l5_block_model.L5BlockModel) -> None:
storage.put_object_as_json(
"BROADCAST/LAST_CONFIRMED_BLOCK", {
"block_id": l5_block.block_id,
"proof": l5_block.export_as_at_rest()["proof"]
})
def test_put_object_as_json_calls_put_with_correct_params(self):
storage.put = MagicMock()
storage.put_object_as_json("key", {})
storage.put.assert_called_once_with("key", b"{}", None, True)
def save(self) -> None:
"""Active record-style save for smart contract state"""
storage.put_object_as_json(f"SMARTCONTRACT/{self.id}/metadata.json",
self.export_as_at_rest())
