def _wait_epoch_chores(cluster_obj: clusterlib.ClusterLib, temp_template: str, this_epoch: int): if cluster_obj.get_epoch() == this_epoch: LOGGER.info(f"{datetime.datetime.now()}: Waiting for next epoch.") cluster_obj.wait_for_new_epoch() LOGGER.info(f"{datetime.datetime.now()}: Waiting for the end of current epoch.") clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster_obj, start=-19, stop=-15) # save ledger state clusterlib_utils.save_ledger_state( cluster_obj=cluster_obj, state_name=f"{temp_template}_{cluster_obj.get_epoch()}", )
def _wait_epoch_chores(this_epoch: int): # wait for next epoch if cluster.get_last_block_epoch() == this_epoch: cluster.wait_for_new_epoch() # wait for the end of the epoch time.sleep(clusterlib_utils.time_to_next_epoch_start(cluster) - 5) # save ledger state clusterlib_utils.save_ledger_state( cluster_obj=cluster, name_template=f"{temp_template}_{cluster.get_last_block_epoch()}", )
def _wait_epoch_chores(this_epoch: int): # wait for next epoch if cluster.get_epoch() == this_epoch: cluster.wait_for_new_epoch() # wait for the end of the epoch clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=-19, stop=-9) # save ledger state clusterlib_utils.save_ledger_state( cluster_obj=cluster, state_name=f"{temp_template}_{cluster.get_epoch()}", )
def test_update_valid_opcert( self, cluster_lock_pool2: clusterlib.ClusterLib, cluster_manager: cluster_management.ClusterManager, ): """Update a valid operational certificate with another valid operational certificate. * generate new operational certificate with valid `--kes-period` * restart the node with the new operational certificate * check that the pool is still producing blocks """ pool_name = "node-pool2" node_name = "pool2" cluster = cluster_lock_pool2 temp_template = helpers.get_func_name() pool_rec = cluster_manager.cache.addrs_data[pool_name] node_cold = pool_rec["cold_key_pair"] stake_pool_id = cluster.get_stake_pool_id(node_cold.vkey_file) stake_pool_id_dec = helpers.decode_bech32(stake_pool_id) opcert_file = pool_rec["pool_operational_cert"] with cluster_manager.restart_on_failure(): # generate new operational certificate with valid `--kes-period` new_opcert_file = cluster.gen_node_operational_cert( node_name=node_name, kes_vkey_file=pool_rec["kes_key_pair"].vkey_file, cold_skey_file=pool_rec["cold_key_pair"].skey_file, cold_counter_file=pool_rec["cold_key_pair"].counter_file, kes_period=cluster.get_kes_period(), ) # restart the node with the new operational certificate logfiles.add_ignore_rule("*.stdout", "MuxBearerClosed") shutil.copy(new_opcert_file, opcert_file) cluster_nodes.restart_node(node_name) LOGGER.info("Checking blocks production for 5 epochs.") blocks_made_db = [] this_epoch = -1 updated_epoch = cluster.get_epoch() for __ in range(5): # wait for next epoch if cluster.get_epoch() == this_epoch: cluster.wait_for_new_epoch() # wait for the end of the epoch clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=-19, stop=-9) this_epoch = cluster.get_epoch() ledger_state = clusterlib_utils.get_ledger_state( cluster_obj=cluster) # save ledger state clusterlib_utils.save_ledger_state( cluster_obj=cluster, state_name=f"{temp_template}_{this_epoch}", ledger_state=ledger_state, ) # check that the pool is still producing blocks blocks_made = ledger_state["blocksCurrent"] blocks_made_db.append(stake_pool_id_dec in blocks_made) assert any(blocks_made_db), ( f"The pool '{pool_name}' has not produced any blocks " f"since epoch {updated_epoch}")
def test_update_valid_opcert( self, cluster_lock_pool2: clusterlib.ClusterLib, cluster_manager: cluster_management.ClusterManager, ): """Update a valid operational certificate with another valid operational certificate. * generate new operational certificate with valid `--kes-period` * copy new operational certificate to the node * stop the node so the corresponding pool is not minting new blocks * check `kes-period-info` while the pool is not minting blocks * start the node with the new operational certificate * check that the pool is minting blocks again * check that metrics reported by `kes-period-info` got updated once the pool started minting blocks again * check `kes-period-info` with the old (replaced) operational certificate """ # pylint: disable=too-many-statements pool_name = cluster_management.Resources.POOL2 node_name = "pool2" cluster = cluster_lock_pool2 temp_template = common.get_test_id(cluster) pool_rec = cluster_manager.cache.addrs_data[pool_name] node_cold = pool_rec["cold_key_pair"] stake_pool_id = cluster.get_stake_pool_id(node_cold.vkey_file) stake_pool_id_dec = helpers.decode_bech32(stake_pool_id) opcert_file = pool_rec["pool_operational_cert"] opcert_file_old = shutil.copy(opcert_file, f"{opcert_file}_old") with cluster_manager.restart_on_failure(): # generate new operational certificate with valid `--kes-period` new_opcert_file = cluster.gen_node_operational_cert( node_name=f"{node_name}_new_opcert_file", kes_vkey_file=pool_rec["kes_key_pair"].vkey_file, cold_skey_file=pool_rec["cold_key_pair"].skey_file, cold_counter_file=pool_rec["cold_key_pair"].counter_file, kes_period=cluster.get_kes_period(), ) # copy new operational certificate to the node logfiles.add_ignore_rule( files_glob="*.stdout", regex="MuxBearerClosed", ignore_file_id=cluster_manager.worker_id, ) shutil.copy(new_opcert_file, opcert_file) # stop the node so the corresponding pool is not minting new blocks cluster_nodes.stop_nodes([node_name]) time.sleep(10) # check kes-period-info while the pool is not minting blocks # TODO: the query is currently broken kes_query_currently_broken = False try: kes_period_info_new = cluster.get_kes_period_info(opcert_file) except clusterlib.CLIError as err: if "currentlyBroken" not in str(err): raise kes_query_currently_broken = True if not kes_query_currently_broken: kes.check_kes_period_info_result( kes_output=kes_period_info_new, expected_scenario=kes.KesScenarios.ALL_VALID ) kes_period_info_old = cluster.get_kes_period_info(opcert_file_old) kes.check_kes_period_info_result( kes_output=kes_period_info_old, expected_scenario=kes.KesScenarios.ALL_VALID ) assert ( kes_period_info_new["metrics"]["qKesNodeStateOperationalCertificateNumber"] == kes_period_info_old["metrics"]["qKesNodeStateOperationalCertificateNumber"] ) # start the node with the new operational certificate cluster_nodes.start_nodes([node_name]) # make sure we are not at the very end of an epoch so we still have time for # the first block production check clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=5, stop=-18) LOGGER.info("Checking blocks production for 5 epochs.") blocks_made_db = [] this_epoch = -1 updated_epoch = cluster.get_epoch() for __ in range(5): # wait for next epoch if cluster.get_epoch() == this_epoch: cluster.wait_for_new_epoch() # wait for the end of the epoch clusterlib_utils.wait_for_epoch_interval( cluster_obj=cluster, start=-19, stop=-15, force_epoch=True ) this_epoch = cluster.get_epoch() ledger_state = clusterlib_utils.get_ledger_state(cluster_obj=cluster) # save ledger state clusterlib_utils.save_ledger_state( cluster_obj=cluster, state_name=f"{temp_template}_{this_epoch}", ledger_state=ledger_state, ) # check that the pool is minting blocks blocks_made = ledger_state["blocksCurrent"] blocks_made_db.append(stake_pool_id_dec in blocks_made) assert any( blocks_made_db ), f"The pool '{pool_name}' has not minted any blocks since epoch {updated_epoch}" if kes_query_currently_broken: pytest.xfail("`query kes-period-info` is currently broken") else: # check that metrics reported by kes-period-info got updated once the pool started # minting blocks again kes_period_info_updated = cluster.get_kes_period_info(opcert_file) kes.check_kes_period_info_result( kes_output=kes_period_info_updated, expected_scenario=kes.KesScenarios.ALL_VALID ) assert ( kes_period_info_updated["metrics"]["qKesNodeStateOperationalCertificateNumber"] != kes_period_info_old["metrics"]["qKesNodeStateOperationalCertificateNumber"] ) # check kes-period-info with operational certificate with a wrong counter kes_period_info_invalid = cluster.get_kes_period_info(opcert_file_old) kes.check_kes_period_info_result( kes_output=kes_period_info_invalid, expected_scenario=kes.KesScenarios.INVALID_COUNTERS, )
def test_oversaturated( # noqa: C901 self, cluster_manager: cluster_management.ClusterManager, cluster_lock_pools: clusterlib.ClusterLib, ): """Check diminished rewards when stake pool is oversaturated. The stake pool continues to operate normally and those who delegate to that pool receive rewards, but the rewards are proportionally lower than those received from stake pool that is not oversaturated. * register and delegate stake address in "init epoch", for all available pools * in "init epoch" + 2, saturate all available pools (block distribution remains balanced among pools) * in "init epoch" + 3, oversaturate one pool * in "init epoch" + 5, for all available pools, withdraw rewards and transfer funds from delegated addresses so pools are no longer (over)saturated * while doing the steps above, collect rewards data for 9 epochs * compare proportionality of rewards in epochs where pools were non-saturated, saturated and oversaturated """ # pylint: disable=too-many-statements,too-many-locals,too-many-branches epoch_saturate = 2 epoch_oversaturate = 4 epoch_withdrawal = 6 cluster = cluster_lock_pools temp_template = common.get_test_id(cluster) initial_balance = 1_000_000_000 faucet_rec = cluster_manager.cache.addrs_data["byron000"] pool_records: Dict[int, PoolRecord] = {} # make sure we have enough time to finish the delegation in one epoch clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=5, stop=-40) init_epoch = cluster.get_epoch() # submit registration certificates and delegate to pools for idx, res in enumerate( [ cluster_management.Resources.POOL1, cluster_management.Resources.POOL2, cluster_management.Resources.POOL3, ], start=1, ): pool_addrs_data = cluster_manager.cache.addrs_data[res] reward_addr = clusterlib.PoolUser( payment=pool_addrs_data["payment"], stake=pool_addrs_data["reward"]) pool_id = delegation.get_pool_id( cluster_obj=cluster, addrs_data=cluster_manager.cache.addrs_data, pool_name=res, ) pool_id_dec = helpers.decode_bech32(bech32=pool_id) delegation_out = delegation.delegate_stake_addr( cluster_obj=cluster, addrs_data=cluster_manager.cache.addrs_data, temp_template=f"{temp_template}_pool{idx}", pool_id=pool_id, amount=initial_balance, ) pool_records[idx] = PoolRecord( name=res, id=pool_id, id_dec=pool_id_dec, reward_addr=reward_addr, delegation_out=delegation_out, user_rewards=[], owner_rewards=[], blocks_minted={}, saturation_amounts={}, ) # record initial reward balance for each pool for pool_rec in pool_records.values(): user_payment_balance = cluster.get_address_balance( pool_rec.delegation_out.pool_user.payment.address) owner_payment_balance = cluster.get_address_balance( pool_rec.reward_addr.payment.address) pool_rec.user_rewards.append( RewardRecord( epoch_no=init_epoch, reward_total=0, reward_per_epoch=0, stake_total=user_payment_balance, )) pool_rec.owner_rewards.append( RewardRecord( epoch_no=init_epoch, reward_total=cluster.get_stake_addr_info( pool_rec.reward_addr.stake.address). reward_account_balance, reward_per_epoch=0, stake_total=owner_payment_balance, )) assert ( cluster.get_epoch() == init_epoch ), "Delegation took longer than expected and would affect other checks" LOGGER.info("Checking rewards for 10 epochs.") for __ in range(10): # wait for new epoch if cluster.get_epoch( ) == pool_records[2].owner_rewards[-1].epoch_no: cluster.wait_for_new_epoch() # sleep till the end of epoch clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=-50, stop=-40, force_epoch=True) this_epoch = cluster.get_epoch() ledger_state = clusterlib_utils.get_ledger_state( cluster_obj=cluster) clusterlib_utils.save_ledger_state( cluster_obj=cluster, state_name=f"{temp_template}_{this_epoch}", ledger_state=ledger_state, ) for pool_rec in pool_records.values(): # reward balance in previous epoch prev_user_reward = pool_rec.user_rewards[-1].reward_total prev_owner_reward = pool_rec.owner_rewards[-1].reward_total pool_rec.blocks_minted[this_epoch - 1] = (ledger_state["blocksBefore"].get( pool_rec.id_dec) or 0) # current reward balance user_reward = cluster.get_stake_addr_info( pool_rec.delegation_out.pool_user.stake.address ).reward_account_balance owner_reward = cluster.get_stake_addr_info( pool_rec.reward_addr.stake.address).reward_account_balance # total reward amounts received this epoch owner_reward_epoch = owner_reward - prev_owner_reward # We cannot compare with previous rewards in epochs where # `this_epoch >= init_epoch + epoch_withdrawal`. # There's a withdrawal of rewards at the end of these epochs. if this_epoch > init_epoch + epoch_withdrawal: user_reward_epoch = user_reward else: user_reward_epoch = user_reward - prev_user_reward # store collected rewards info user_payment_balance = cluster.get_address_balance( pool_rec.delegation_out.pool_user.payment.address) owner_payment_balance = cluster.get_address_balance( pool_rec.reward_addr.payment.address) pool_rec.user_rewards.append( RewardRecord( epoch_no=this_epoch, reward_total=user_reward, reward_per_epoch=user_reward_epoch, stake_total=user_payment_balance + user_reward, )) pool_rec.owner_rewards.append( RewardRecord( epoch_no=this_epoch, reward_total=owner_reward, reward_per_epoch=owner_reward_epoch, stake_total=owner_payment_balance, )) pool_rec.saturation_amounts[ this_epoch] = _get_saturation_threshold( cluster_obj=cluster, ledger_state=ledger_state, pool_id=pool_rec.id) # fund the delegated addresses - saturate all pools if this_epoch == init_epoch + epoch_saturate: clusterlib_utils.fund_from_faucet( *[ p.delegation_out.pool_user.payment for p in pool_records.values() ], cluster_obj=cluster, faucet_data=faucet_rec, amount=[ p.saturation_amounts[this_epoch] - 100_000_000_000 for p in pool_records.values() ], tx_name=f"{temp_template}_saturate_pools_ep{this_epoch}", force=True, ) with cluster_manager.restart_on_failure(): # Fund the address delegated to "pool2" to oversaturate the pool. # New stake amount will be current (saturated) stake * 2. if this_epoch == init_epoch + epoch_oversaturate: assert (pool_records[2].saturation_amounts[this_epoch] > 0), "Pool is already saturated" current_stake = int( cluster.get_stake_snapshot( pool_records[2].id)["poolStakeMark"]) overstaturate_amount = current_stake * 2 saturation_threshold = pool_records[2].saturation_amounts[ this_epoch] assert overstaturate_amount > saturation_threshold, ( f"{overstaturate_amount} Lovelace is not enough to oversature the pool " f"({saturation_threshold} is needed)") clusterlib_utils.fund_from_faucet( pool_records[2].delegation_out.pool_user.payment, cluster_obj=cluster, faucet_data=faucet_rec, amount=overstaturate_amount, tx_name=f"{temp_template}_oversaturate_pool2", force=True, ) # wait 4 epochs for first rewards if this_epoch >= init_epoch + 4: assert (owner_reward > prev_owner_reward ), "New reward was not received by pool owner" # transfer funds back to faucet so the pools are no longer (over)saturated # and staked amount is +- same as the `initial_balance` if this_epoch >= init_epoch + epoch_withdrawal: _withdraw_rewards( *[ p.delegation_out.pool_user for p in pool_records.values() ], cluster_obj=cluster, tx_name=f"{temp_template}_ep{this_epoch}", ) return_to_addrs = [] return_amounts = [] for idx, pool_rec in pool_records.items(): deleg_payment_balance = cluster.get_address_balance( pool_rec.delegation_out.pool_user.payment.address) if deleg_payment_balance > initial_balance + 10_000_000: return_to_addrs.append( pool_rec.delegation_out.pool_user.payment) return_amounts.append(deleg_payment_balance - initial_balance) clusterlib_utils.return_funds_to_faucet( *return_to_addrs, cluster_obj=cluster, faucet_addr=faucet_rec["payment"].address, amount=return_amounts, tx_name=f"{temp_template}_ep{this_epoch}", ) for return_addr in return_to_addrs: deleg_payment_balance = cluster.get_address_balance( return_addr.address) assert ( deleg_payment_balance <= initial_balance ), "Unexpected funds in payment address '{return_addr}'" assert ( cluster.get_epoch() == this_epoch ), "Failed to finish actions in single epoch, it would affect other checks" pool1_user_rewards_per_block = _get_reward_per_block(pool_records[1]) pool2_user_rewards_per_block = _get_reward_per_block(pool_records[2]) pool3_user_rewards_per_block = _get_reward_per_block(pool_records[3]) pool1_owner_rewards_per_block = _get_reward_per_block( pool_records[1], owner_rewards=True) pool2_owner_rewards_per_block = _get_reward_per_block( pool_records[2], owner_rewards=True) pool3_owner_rewards_per_block = _get_reward_per_block( pool_records[3], owner_rewards=True) oversaturated_epoch = max( e for e, r in pool_records[2].saturation_amounts.items() if r < 0) saturated_epoch = oversaturated_epoch - 2 nonsaturated_epoch = oversaturated_epoch - 4 try: # check that rewards per block per stake for "pool2" in the epoch where the pool is # oversaturated is lower than in epochs where pools are not oversaturated assert (pool1_user_rewards_per_block[nonsaturated_epoch] > pool2_user_rewards_per_block[oversaturated_epoch]) assert (pool2_user_rewards_per_block[nonsaturated_epoch] > pool2_user_rewards_per_block[oversaturated_epoch]) assert (pool3_user_rewards_per_block[nonsaturated_epoch] > pool2_user_rewards_per_block[oversaturated_epoch]) assert (pool1_user_rewards_per_block[saturated_epoch] > pool2_user_rewards_per_block[oversaturated_epoch]) assert (pool2_user_rewards_per_block[saturated_epoch] > pool2_user_rewards_per_block[oversaturated_epoch]) assert (pool3_user_rewards_per_block[saturated_epoch] > pool2_user_rewards_per_block[oversaturated_epoch]) # check that oversaturated pool doesn't lead to increased rewards for pool owner # when compared to saturated pool, i.e. total pool margin amount is not increased pool1_rew_fraction_sat = pool1_owner_rewards_per_block[ saturated_epoch] pool2_rew_fraction_sat = pool2_owner_rewards_per_block[ saturated_epoch] pool3_rew_fraction_sat = pool3_owner_rewards_per_block[ saturated_epoch] pool2_rew_fraction_over = pool2_owner_rewards_per_block[ oversaturated_epoch] assert pool2_rew_fraction_sat > pool2_rew_fraction_over or helpers.is_in_interval( pool2_rew_fraction_sat, pool2_rew_fraction_over, frac=0.4, ) assert pool1_rew_fraction_sat > pool2_rew_fraction_over or helpers.is_in_interval( pool1_rew_fraction_sat, pool2_rew_fraction_over, frac=0.4, ) assert pool3_rew_fraction_sat > pool2_rew_fraction_over or helpers.is_in_interval( pool3_rew_fraction_sat, pool2_rew_fraction_over, frac=0.4, ) # Compare rewards in last (non-saturated) epoch to rewards in next-to-last # (saturated / over-saturated) epoch. # This way check that staked amount for each pool was restored to `initial_balance` # and that rewards correspond to the restored amounts. for pool_rec in pool_records.values(): assert (pool_rec.user_rewards[-1].reward_per_epoch * 100 < pool_rec.user_rewards[-2].reward_per_epoch) except Exception: # save debugging data in case of test failure with open(f"{temp_template}_pool_records.pickle", "wb") as out_data: pickle.dump(pool_records, out_data) raise
def test_update_valid_opcert( self, cluster_lock_pool2: clusterlib.ClusterLib, cluster_manager: parallel_run.ClusterManager, ): """Update a valid operational certificate with another valid operational certificate. * generate new operational certificate with valid `--kes-period` * restart the node with the new operational certificate * check that the pool is still producing blocks """ pool_name = "node-pool2" node_name = "pool2" cluster = cluster_lock_pool2 temp_template = helpers.get_func_name() pool_rec = cluster_manager.cache.addrs_data[pool_name] node_cold = pool_rec["cold_key_pair"] stake_pool_id = cluster.get_stake_pool_id(node_cold.vkey_file) stake_pool_id_dec = helpers.decode_bech32(stake_pool_id) opcert_file = pool_rec["pool_operational_cert"] with cluster_manager.restart_on_failure(): # generate new operational certificate with valid `--kes-period` new_opcert_file = cluster.gen_node_operational_cert( node_name=node_name, node_kes_vkey_file=pool_rec["kes_key_pair"].vkey_file, node_cold_skey_file=pool_rec["cold_key_pair"].skey_file, node_cold_counter_file=pool_rec["cold_key_pair"].counter_file, kes_period=cluster.get_last_block_kes_period(), ) # restart the node with the new operational certificate shutil.copy(new_opcert_file, opcert_file) devops_cluster.restart_node(node_name) LOGGER.info("Checking blocks production for 5 epochs.") this_epoch = -1 for __ in range(5): # wait for next epoch if cluster.get_last_block_epoch() == this_epoch: cluster.wait_for_new_epoch() # wait for the end of the epoch time.sleep(clusterlib_utils.time_to_next_epoch_start(cluster) - 5) this_epoch = cluster.get_last_block_epoch() # save ledger state clusterlib_utils.save_ledger_state( cluster_obj=cluster, name_template=f"{temp_template}_{this_epoch}", ) # check that the pool is still producing blocks blocks_made = cluster.get_ledger_state()["nesBcur"]["unBlocksMade"] if blocks_made: assert ( stake_pool_id_dec in blocks_made ), f"The pool '{pool_name}' has not produced blocks in epoch {this_epoch}"
def test_stake_snapshot(self, cluster: clusterlib.ClusterLib): # noqa: C901 """Test the `stake-snapshot` and `ledger-state` commands and ledger state values.""" # pylint: disable=too-many-statements,too-many-locals,too-many-branches temp_template = common.get_test_id(cluster) # make sure the queries can be finished in single epoch stop = ( 20 if cluster_nodes.get_cluster_type().type == cluster_nodes.ClusterType.LOCAL else 200 ) clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=5, stop=-stop) stake_pool_ids = cluster.get_stake_pools() if not stake_pool_ids: pytest.skip("No stake pools are available.") if len(stake_pool_ids) > 200: pytest.skip("Skipping on this testnet, there's too many pools.") ledger_state = clusterlib_utils.get_ledger_state(cluster_obj=cluster) clusterlib_utils.save_ledger_state( cluster_obj=cluster, state_name=temp_template, ledger_state=ledger_state, ) es_snapshot: dict = ledger_state["stateBefore"]["esSnapshots"] def _get_hashes(snapshot: str) -> Dict[str, int]: hashes: Dict[str, int] = {} for r in es_snapshot[snapshot]["stake"]: r_hash_rec = r[0] r_hash = r_hash_rec.get("script hash") or r_hash_rec.get("key hash") if r_hash in hashes: hashes[r_hash] += r[1] else: hashes[r_hash] = r[1] return hashes def _get_delegations(snapshot: str) -> Dict[str, List[str]]: delegations: Dict[str, List[str]] = {} for r in es_snapshot[snapshot]["delegations"]: r_hash_rec = r[0] r_hash = r_hash_rec.get("script hash") or r_hash_rec.get("key hash") r_pool_id = r[1] if r_pool_id in delegations: delegations[r_pool_id].append(r_hash) else: delegations[r_pool_id] = [r_hash] return delegations errors = [] ledger_state_keys = set(ledger_state) if ledger_state_keys != LEDGER_STATE_KEYS: errors.append( "unexpected ledger state keys: " f"{ledger_state_keys.difference(LEDGER_STATE_KEYS)} and " f"{LEDGER_STATE_KEYS.difference(ledger_state_keys)}" ) # stake addresses (hashes) and corresponding amounts stake_mark = _get_hashes("pstakeMark") stake_set = _get_hashes("pstakeSet") stake_go = _get_hashes("pstakeGo") # pools (hashes) and stake addresses (hashes) delegated to corresponding pool delegations_mark = _get_delegations("pstakeMark") delegations_set = _get_delegations("pstakeSet") delegations_go = _get_delegations("pstakeGo") # all delegated stake addresses (hashes) delegated_hashes_mark = set(itertools.chain.from_iterable(delegations_mark.values())) delegated_hashes_set = set(itertools.chain.from_iterable(delegations_set.values())) delegated_hashes_go = set(itertools.chain.from_iterable(delegations_go.values())) # check if all delegated addresses are listed among stake addresses stake_hashes_mark = set(stake_mark) if not delegated_hashes_mark.issubset(stake_hashes_mark): errors.append( "for 'mark', some delegations are not listed in 'stake': " f"{delegated_hashes_mark.difference(stake_hashes_mark)}" ) stake_hashes_set = set(stake_set) if not delegated_hashes_set.issubset(stake_hashes_set): errors.append( "for 'set', some delegations are not listed in 'stake': " f"{delegated_hashes_set.difference(stake_hashes_set)}" ) stake_hashes_go = set(stake_go) if not delegated_hashes_go.issubset(stake_hashes_go): errors.append( "for 'go', some delegations are not listed in 'stake': " f"{delegated_hashes_go.difference(stake_hashes_go)}" ) sum_mark = sum_set = sum_go = 0 seen_hashes_mark: Set[str] = set() seen_hashes_set: Set[str] = set() seen_hashes_go: Set[str] = set() delegation_pool_ids = {*delegations_mark, *delegations_set, *delegations_go} for pool_id_dec in delegation_pool_ids: pool_id = helpers.encode_bech32(prefix="pool", data=pool_id_dec) # get stake info from ledger state pstake_hashes_mark = delegations_mark.get(pool_id_dec) or () seen_hashes_mark.update(pstake_hashes_mark) pstake_amounts_mark = [stake_mark[h] for h in pstake_hashes_mark] pstake_sum_mark = functools.reduce(lambda x, y: x + y, pstake_amounts_mark, 0) pstake_hashes_set = delegations_set.get(pool_id_dec) or () seen_hashes_set.update(pstake_hashes_set) pstake_amounts_set = [stake_set[h] for h in pstake_hashes_set] pstake_sum_set = functools.reduce(lambda x, y: x + y, pstake_amounts_set, 0) pstake_hashes_go = delegations_go.get(pool_id_dec) or () seen_hashes_go.update(pstake_hashes_go) pstake_amounts_go = [stake_go[h] for h in pstake_hashes_go] pstake_sum_go = functools.reduce(lambda x, y: x + y, pstake_amounts_go, 0) # get stake info from `stake-snapshot` command stake_snapshot = cluster.get_stake_snapshot(stake_pool_id=pool_id) pstake_mark_cmd = stake_snapshot["poolStakeMark"] pstake_set_cmd = stake_snapshot["poolStakeSet"] pstake_go_cmd = stake_snapshot["poolStakeGo"] if pstake_sum_mark != pstake_mark_cmd: errors.append(f"pool: {pool_id}, mark:\n {pstake_sum_mark} != {pstake_mark_cmd}") if pstake_sum_set != pstake_set_cmd: errors.append(f"pool: {pool_id}, set:\n {pstake_sum_set} != {pstake_set_cmd}") if pstake_sum_go != pstake_go_cmd: errors.append(f"pool: {pool_id}, go:\n {pstake_sum_go} != {pstake_go_cmd}") sum_mark += pstake_mark_cmd sum_set += pstake_set_cmd sum_go += pstake_go_cmd if seen_hashes_mark != delegated_hashes_mark: errors.append( "seen hashes and existing hashes differ for 'mark': " f"{seen_hashes_mark.difference(delegated_hashes_mark)} and " f"{delegated_hashes_mark.difference(seen_hashes_mark)}" ) if seen_hashes_set != delegated_hashes_set: errors.append( "seen hashes and existing hashes differ for 'set': " f"{seen_hashes_set.difference(delegated_hashes_set)} and " f"{delegated_hashes_set.difference(seen_hashes_set)}" ) if seen_hashes_go != delegated_hashes_go: errors.append( "seen hashes and existing hashes differ for 'go': " f"{seen_hashes_go.difference(delegated_hashes_go)} and " f"{delegated_hashes_go.difference(seen_hashes_go)}" ) # active stake can be lower than sum of stakes, as some pools may not be running # and minting blocks if sum_mark < stake_snapshot["activeStakeMark"]: errors.append(f"active_mark: {sum_mark} < {stake_snapshot['activeStakeMark']}") if sum_set < stake_snapshot["activeStakeSet"]: errors.append(f"active_set: {sum_set} < {stake_snapshot['activeStakeSet']}") if sum_go < stake_snapshot["activeStakeGo"]: errors.append(f"active_go: {sum_go} < {stake_snapshot['activeStakeGo']}") if errors: err_joined = "\n".join(errors) pytest.fail(f"Errors:\n{err_joined}")
def test_pool_blocks( self, skip_leadership_schedule: None, cluster_manager: cluster_management.ClusterManager, cluster_use_pool3: clusterlib.ClusterLib, for_epoch: str, ): """Check that blocks were minted according to leadership schedule. * query leadership schedule for selected pool for current epoch or next epoch * wait for epoch that comes after the queried epoch * get info about minted blocks in queried epoch for the selected pool * compare leadership schedule with blocks that were actually minted * compare db-sync records with ledger state dump """ # pylint: disable=unused-argument cluster = cluster_use_pool3 temp_template = common.get_test_id(cluster) pool_name = cluster_management.Resources.POOL3 pool_rec = cluster_manager.cache.addrs_data[pool_name] pool_id = cluster.get_stake_pool_id( pool_rec["cold_key_pair"].vkey_file) if for_epoch == "current": # wait for beginning of an epoch queried_epoch = cluster.wait_for_new_epoch(padding_seconds=5) else: # wait for stable stake distribution for next epoch, that is last 300 slots of # current epoch clusterlib_utils.wait_for_epoch_interval( cluster_obj=cluster, start=-int(300 * cluster.slot_length), stop=-10, check_slot=True, ) queried_epoch = cluster.get_epoch() + 1 # query leadership schedule for selected pool # TODO: the query is currently broken query_currently_broken = False try: leadership_schedule = cluster.get_leadership_schedule( vrf_skey_file=pool_rec["vrf_key_pair"].skey_file, cold_vkey_file=pool_rec["cold_key_pair"].vkey_file, for_next=for_epoch != "current", ) except clusterlib.CLIError as err: if "currently broken" not in str(err): raise query_currently_broken = True if query_currently_broken: pytest.xfail("`query leadership-schedule` is currently broken") # wait for epoch that comes after the queried epoch cluster.wait_for_new_epoch( new_epochs=1 if for_epoch == "current" else 2) # get info about minted blocks in queried epoch for the selected pool minted_blocks = list( dbsync_queries.query_blocks(pool_id_bech32=pool_id, epoch_from=queried_epoch, epoch_to=queried_epoch)) slots_when_minted = {r.slot_no for r in minted_blocks} errors: List[str] = [] # compare leadership schedule with blocks that were actually minted slots_when_scheduled = {r.slot_no for r in leadership_schedule} difference_scheduled = slots_when_minted.difference( slots_when_scheduled) if difference_scheduled: errors.append( f"Some blocks were minted in other slots than scheduled: {difference_scheduled}" ) difference_minted = slots_when_scheduled.difference(slots_when_minted) if len(difference_minted) > len(leadership_schedule) // 2: errors.append(f"Lot of slots missed: {difference_minted}") # compare db-sync records with ledger state dump ledger_state = clusterlib_utils.get_ledger_state(cluster_obj=cluster) clusterlib_utils.save_ledger_state( cluster_obj=cluster, state_name=temp_template, ledger_state=ledger_state, ) blocks_before: Dict[str, int] = ledger_state["blocksBefore"] pool_id_dec = helpers.decode_bech32(pool_id) minted_blocks_ledger = blocks_before.get(pool_id_dec) or 0 minted_blocks_db = len(slots_when_minted) if minted_blocks_ledger != minted_blocks_db: errors.append( "Numbers of minted blocks reported by ledger state and db-sync don't match: " f"{minted_blocks_ledger} vs {minted_blocks_db}") if errors: err_joined = "\n".join(errors) pytest.fail(f"Errors:\n{err_joined}")