def test_opcert_past_kes_period(
self,
cluster_lock_pool2: clusterlib.ClusterLib,
cluster_manager: cluster_management.ClusterManager,
):
"""Start a stake pool with an operational certificate created with expired `--kes-period`.
* generate new operational certificate with `--kes-period` in the past
* restart the node with the new operational certificate
* check that the pool is not producing any blocks
* generate new operational certificate with valid `--kes-period` and restart the node
* check that the pool is producing blocks again
"""
pool_name = "node-pool2"
node_name = "pool2"
cluster = cluster_lock_pool2
temp_template = helpers.get_func_name()
pool_rec = cluster_manager.cache.addrs_data[pool_name]
node_cold = pool_rec["cold_key_pair"]
stake_pool_id = cluster.get_stake_pool_id(node_cold.vkey_file)
stake_pool_id_dec = helpers.decode_bech32(stake_pool_id)
opcert_file: Path = pool_rec["pool_operational_cert"]
def _wait_epoch_chores(this_epoch: int):
# wait for next epoch
if cluster.get_epoch() == this_epoch:
cluster.wait_for_new_epoch()
# wait for the end of the epoch
clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster,
start=-19,
stop=-9)
# save ledger state
clusterlib_utils.save_ledger_state(
cluster_obj=cluster,
state_name=f"{temp_template}_{cluster.get_epoch()}",
)
with cluster_manager.restart_on_failure():
# generate new operational certificate with `--kes-period` in the past
invalid_opcert_file = cluster.gen_node_operational_cert(
node_name=node_name,
kes_vkey_file=pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=pool_rec["cold_key_pair"].skey_file,
cold_counter_file=pool_rec["cold_key_pair"].counter_file,
kes_period=cluster.get_kes_period() - 5,
)
expected_errors = [
(f"{node_name}.stdout", "TPraosCannotForgeKeyNotUsableYet"),
]
with logfiles.expect_errors(expected_errors):
# restart the node with the new operational certificate
logfiles.add_ignore_rule("*.stdout", "MuxBearerClosed")
shutil.copy(invalid_opcert_file, opcert_file)
cluster_nodes.restart_node(node_name)
cluster.wait_for_new_epoch()
LOGGER.info("Checking blocks production for 5 epochs.")
this_epoch = -1
for __ in range(5):
_wait_epoch_chores(this_epoch)
this_epoch = cluster.get_epoch()
# check that the pool is not producing any blocks
blocks_made = clusterlib_utils.get_ledger_state(
cluster_obj=cluster)["blocksCurrent"]
if blocks_made:
assert (
stake_pool_id_dec not in blocks_made
), f"The pool '{pool_name}' has produced blocks in epoch {this_epoch}"
# generate new operational certificate with valid `--kes-period`
os.remove(opcert_file)
valid_opcert_file = cluster.gen_node_operational_cert(
node_name=node_name,
kes_vkey_file=pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=pool_rec["cold_key_pair"].skey_file,
cold_counter_file=pool_rec["cold_key_pair"].counter_file,
kes_period=cluster.get_kes_period(),
)
# copy the new certificate and restart the node
shutil.move(str(valid_opcert_file), str(opcert_file))
cluster_nodes.restart_node(node_name)
cluster.wait_for_new_epoch()
LOGGER.info("Checking blocks production for another 5 epochs.")
blocks_made_db = []
this_epoch = cluster.get_epoch()
active_again_epoch = this_epoch
for __ in range(5):
_wait_epoch_chores(this_epoch)
this_epoch = cluster.get_epoch()
# check that the pool is producing blocks
blocks_made = clusterlib_utils.get_ledger_state(
cluster_obj=cluster)["blocksCurrent"]
blocks_made_db.append(stake_pool_id_dec in blocks_made)
assert any(blocks_made_db), (
f"The pool '{pool_name}' has not produced any blocks "
f"since epoch {active_again_epoch}")
def test_update_valid_opcert(
self,
cluster_lock_pool2: clusterlib.ClusterLib,
cluster_manager: cluster_management.ClusterManager,
):
"""Update a valid operational certificate with another valid operational certificate.
* generate new operational certificate with valid `--kes-period`
* restart the node with the new operational certificate
* check that the pool is still producing blocks
"""
pool_name = "node-pool2"
node_name = "pool2"
cluster = cluster_lock_pool2
temp_template = helpers.get_func_name()
pool_rec = cluster_manager.cache.addrs_data[pool_name]
node_cold = pool_rec["cold_key_pair"]
stake_pool_id = cluster.get_stake_pool_id(node_cold.vkey_file)
stake_pool_id_dec = helpers.decode_bech32(stake_pool_id)
opcert_file = pool_rec["pool_operational_cert"]
with cluster_manager.restart_on_failure():
# generate new operational certificate with valid `--kes-period`
new_opcert_file = cluster.gen_node_operational_cert(
node_name=node_name,
kes_vkey_file=pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=pool_rec["cold_key_pair"].skey_file,
cold_counter_file=pool_rec["cold_key_pair"].counter_file,
kes_period=cluster.get_kes_period(),
)
# restart the node with the new operational certificate
logfiles.add_ignore_rule("*.stdout", "MuxBearerClosed")
shutil.copy(new_opcert_file, opcert_file)
cluster_nodes.restart_node(node_name)
LOGGER.info("Checking blocks production for 5 epochs.")
blocks_made_db = []
this_epoch = -1
updated_epoch = cluster.get_epoch()
for __ in range(5):
# wait for next epoch
if cluster.get_epoch() == this_epoch:
cluster.wait_for_new_epoch()
# wait for the end of the epoch
clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster,
start=-19,
stop=-9)
this_epoch = cluster.get_epoch()
ledger_state = clusterlib_utils.get_ledger_state(
cluster_obj=cluster)
# save ledger state
clusterlib_utils.save_ledger_state(
cluster_obj=cluster,
state_name=f"{temp_template}_{this_epoch}",
ledger_state=ledger_state,
)
# check that the pool is still producing blocks
blocks_made = ledger_state["blocksCurrent"]
blocks_made_db.append(stake_pool_id_dec in blocks_made)
assert any(blocks_made_db), (
f"The pool '{pool_name}' has not produced any blocks "
f"since epoch {updated_epoch}")
def test_update_valid_opcert(
self,
cluster_lock_pool2: clusterlib.ClusterLib,
cluster_manager: cluster_management.ClusterManager,
):
"""Update a valid operational certificate with another valid operational certificate.
* generate new operational certificate with valid `--kes-period`
* copy new operational certificate to the node
* stop the node so the corresponding pool is not minting new blocks
* check `kes-period-info` while the pool is not minting blocks
* start the node with the new operational certificate
* check that the pool is minting blocks again
* check that metrics reported by `kes-period-info` got updated once the pool started
minting blocks again
* check `kes-period-info` with the old (replaced) operational certificate
"""
# pylint: disable=too-many-statements
pool_name = cluster_management.Resources.POOL2
node_name = "pool2"
cluster = cluster_lock_pool2
temp_template = common.get_test_id(cluster)
pool_rec = cluster_manager.cache.addrs_data[pool_name]
node_cold = pool_rec["cold_key_pair"]
stake_pool_id = cluster.get_stake_pool_id(node_cold.vkey_file)
stake_pool_id_dec = helpers.decode_bech32(stake_pool_id)
opcert_file = pool_rec["pool_operational_cert"]
opcert_file_old = shutil.copy(opcert_file, f"{opcert_file}_old")
with cluster_manager.restart_on_failure():
# generate new operational certificate with valid `--kes-period`
new_opcert_file = cluster.gen_node_operational_cert(
node_name=f"{node_name}_new_opcert_file",
kes_vkey_file=pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=pool_rec["cold_key_pair"].skey_file,
cold_counter_file=pool_rec["cold_key_pair"].counter_file,
kes_period=cluster.get_kes_period(),
)
# copy new operational certificate to the node
logfiles.add_ignore_rule(
files_glob="*.stdout",
regex="MuxBearerClosed",
ignore_file_id=cluster_manager.worker_id,
)
shutil.copy(new_opcert_file, opcert_file)
# stop the node so the corresponding pool is not minting new blocks
cluster_nodes.stop_nodes([node_name])
time.sleep(10)
# check kes-period-info while the pool is not minting blocks
# TODO: the query is currently broken
kes_query_currently_broken = False
try:
kes_period_info_new = cluster.get_kes_period_info(opcert_file)
except clusterlib.CLIError as err:
if "currentlyBroken" not in str(err):
raise
kes_query_currently_broken = True
if not kes_query_currently_broken:
kes.check_kes_period_info_result(
kes_output=kes_period_info_new, expected_scenario=kes.KesScenarios.ALL_VALID
)
kes_period_info_old = cluster.get_kes_period_info(opcert_file_old)
kes.check_kes_period_info_result(
kes_output=kes_period_info_old, expected_scenario=kes.KesScenarios.ALL_VALID
)
assert (
kes_period_info_new["metrics"]["qKesNodeStateOperationalCertificateNumber"]
== kes_period_info_old["metrics"]["qKesNodeStateOperationalCertificateNumber"]
)
# start the node with the new operational certificate
cluster_nodes.start_nodes([node_name])
# make sure we are not at the very end of an epoch so we still have time for
# the first block production check
clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=5, stop=-18)
LOGGER.info("Checking blocks production for 5 epochs.")
blocks_made_db = []
this_epoch = -1
updated_epoch = cluster.get_epoch()
for __ in range(5):
# wait for next epoch
if cluster.get_epoch() == this_epoch:
cluster.wait_for_new_epoch()
# wait for the end of the epoch
clusterlib_utils.wait_for_epoch_interval(
cluster_obj=cluster, start=-19, stop=-15, force_epoch=True
)
this_epoch = cluster.get_epoch()
ledger_state = clusterlib_utils.get_ledger_state(cluster_obj=cluster)
# save ledger state
clusterlib_utils.save_ledger_state(
cluster_obj=cluster,
state_name=f"{temp_template}_{this_epoch}",
ledger_state=ledger_state,
)
# check that the pool is minting blocks
blocks_made = ledger_state["blocksCurrent"]
blocks_made_db.append(stake_pool_id_dec in blocks_made)
assert any(
blocks_made_db
), f"The pool '{pool_name}' has not minted any blocks since epoch {updated_epoch}"
if kes_query_currently_broken:
pytest.xfail("`query kes-period-info` is currently broken")
else:
# check that metrics reported by kes-period-info got updated once the pool started
# minting blocks again
kes_period_info_updated = cluster.get_kes_period_info(opcert_file)
kes.check_kes_period_info_result(
kes_output=kes_period_info_updated, expected_scenario=kes.KesScenarios.ALL_VALID
)
assert (
kes_period_info_updated["metrics"]["qKesNodeStateOperationalCertificateNumber"]
!= kes_period_info_old["metrics"]["qKesNodeStateOperationalCertificateNumber"]
)
# check kes-period-info with operational certificate with a wrong counter
kes_period_info_invalid = cluster.get_kes_period_info(opcert_file_old)
kes.check_kes_period_info_result(
kes_output=kes_period_info_invalid,
expected_scenario=kes.KesScenarios.INVALID_COUNTERS,
)
def test_ledger_state_keys(self, cluster: clusterlib.ClusterLib):
"""Check output of `query ledger-state`."""
ledger_state = clusterlib_utils.get_ledger_state(cluster_obj=cluster)
assert tuple(sorted(ledger_state)) == LEDGER_STATE_KEYS
def test_opcert_future_kes_period( # noqa: C901
self,
cluster_lock_pool2: clusterlib.ClusterLib,
cluster_manager: cluster_management.ClusterManager,
):
"""Start a stake pool with an operational certificate created with invalid `--kes-period`.
* generate new operational certificate with `--kes-period` in the future
* restart the node with the new operational certificate
* check that the pool is not producing any blocks
* if network era > Alonzo
- generate new operational certificate with valid `--kes-period`, but counter value +2
from last used operational ceritificate
- restart the node
- check that the pool is not producing any blocks
* generate new operational certificate with valid `--kes-period` and restart the node
* check that the pool is producing blocks again
"""
# pylint: disable=too-many-statements,too-many-branches
pool_name = cluster_management.Resources.POOL2
node_name = "pool2"
cluster = cluster_lock_pool2
temp_template = common.get_test_id(cluster)
pool_rec = cluster_manager.cache.addrs_data[pool_name]
node_cold = pool_rec["cold_key_pair"]
stake_pool_id = cluster.get_stake_pool_id(node_cold.vkey_file)
stake_pool_id_dec = helpers.decode_bech32(stake_pool_id)
opcert_file: Path = pool_rec["pool_operational_cert"]
cold_counter_file: Path = pool_rec["cold_key_pair"].counter_file
expected_errors = [
(f"{node_name}.stdout", "PraosCannotForgeKeyNotUsableYet"),
]
if VERSIONS.cluster_era > VERSIONS.ALONZO:
expected_errors.append((f"{node_name}.stdout", "CounterOverIncrementedOCERT"))
# In Babbage we get `CounterOverIncrementedOCERT` error if counter for new opcert
# is not exactly +1 from last used opcert. We'll backup the original counter
# file so we can use it for issuing next valid opcert.
cold_counter_file_orig = Path(
f"{cold_counter_file.stem}_orig{cold_counter_file.suffix}"
).resolve()
shutil.copy(cold_counter_file, cold_counter_file_orig)
logfiles.add_ignore_rule(
files_glob="*.stdout",
regex="MuxBearerClosed|CounterOverIncrementedOCERT",
ignore_file_id=cluster_manager.worker_id,
)
# generate new operational certificate with `--kes-period` in the future
invalid_opcert_file = cluster.gen_node_operational_cert(
node_name=f"{node_name}_invalid_opcert_file",
kes_vkey_file=pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=pool_rec["cold_key_pair"].skey_file,
cold_counter_file=cold_counter_file,
kes_period=cluster.get_kes_period() + 100,
)
kes_query_currently_broken = False
with cluster_manager.restart_on_failure():
with logfiles.expect_errors(expected_errors, ignore_file_id=cluster_manager.worker_id):
# restart the node with the new operational certificate
shutil.copy(invalid_opcert_file, opcert_file)
cluster_nodes.restart_nodes([node_name])
cluster.wait_for_new_epoch()
LOGGER.info("Checking blocks production for 4 epochs.")
this_epoch = -1
for invalid_opcert_epoch in range(4):
_wait_epoch_chores(
cluster_obj=cluster, temp_template=temp_template, this_epoch=this_epoch
)
this_epoch = cluster.get_epoch()
# check that the pool is not producing any blocks
blocks_made = clusterlib_utils.get_ledger_state(cluster_obj=cluster)[
"blocksCurrent"
]
if blocks_made:
assert (
stake_pool_id_dec not in blocks_made
), f"The pool '{pool_name}' has produced blocks in epoch {this_epoch}"
if invalid_opcert_epoch == 1:
# check kes-period-info with operational certificate with
# invalid `--kes-period`
# TODO: the query is currently broken
try:
kes_period_info = cluster.get_kes_period_info(invalid_opcert_file)
except clusterlib.CLIError as err:
if "currentlyBroken" not in str(err):
raise
kes_query_currently_broken = True
if not kes_query_currently_broken:
kes.check_kes_period_info_result(
kes_output=kes_period_info,
expected_scenario=kes.KesScenarios.INVALID_KES_PERIOD,
)
# test the `CounterOverIncrementedOCERT` error - the counter will now be +2 from
# last used opcert counter value
if invalid_opcert_epoch == 2 and VERSIONS.cluster_era > VERSIONS.ALONZO:
overincrement_opcert_file = cluster.gen_node_operational_cert(
node_name=f"{node_name}_overincrement_opcert_file",
kes_vkey_file=pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=pool_rec["cold_key_pair"].skey_file,
cold_counter_file=cold_counter_file,
kes_period=cluster.get_kes_period(),
)
# copy the new certificate and restart the node
shutil.copy(overincrement_opcert_file, opcert_file)
cluster_nodes.restart_nodes([node_name])
if invalid_opcert_epoch == 3:
# check kes-period-info with operational certificate with
# invalid counter
# TODO: the query is currently broken, implement once it is fixed
pass
# in Babbage we'll use the original counter for issuing new valid opcert so the counter
# value of new valid opcert equals to counter value of the original opcert +1
if VERSIONS.cluster_era > VERSIONS.ALONZO:
shutil.copy(cold_counter_file_orig, cold_counter_file)
# generate new operational certificate with valid `--kes-period`
valid_opcert_file = cluster.gen_node_operational_cert(
node_name=f"{node_name}_valid_opcert_file",
kes_vkey_file=pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=pool_rec["cold_key_pair"].skey_file,
cold_counter_file=cold_counter_file,
kes_period=cluster.get_kes_period(),
)
# copy the new certificate and restart the node
shutil.copy(valid_opcert_file, opcert_file)
cluster_nodes.restart_nodes([node_name])
this_epoch = cluster.wait_for_new_epoch()
LOGGER.info("Checking blocks production for another 2 epochs.")
blocks_made_db = []
active_again_epoch = this_epoch
for __ in range(2):
_wait_epoch_chores(
cluster_obj=cluster, temp_template=temp_template, this_epoch=this_epoch
)
this_epoch = cluster.get_epoch()
# check that the pool is producing blocks
blocks_made = clusterlib_utils.get_ledger_state(cluster_obj=cluster)[
"blocksCurrent"
]
blocks_made_db.append(stake_pool_id_dec in blocks_made)
assert any(blocks_made_db), (
f"The pool '{pool_name}' has not produced any blocks "
f"since epoch {active_again_epoch}"
)
if kes_query_currently_broken:
pytest.xfail("`query kes-period-info` is currently broken")
else:
# check kes-period-info with valid operational certificate
kes_period_info = cluster.get_kes_period_info(valid_opcert_file)
kes.check_kes_period_info_result(
kes_output=kes_period_info, expected_scenario=kes.KesScenarios.ALL_VALID
)
# check kes-period-info with invalid operational certificate, wrong counter and period
kes_period_info = cluster.get_kes_period_info(invalid_opcert_file)
kes.check_kes_period_info_result(
kes_output=kes_period_info,
expected_scenario=kes.KesScenarios.INVALID_KES_PERIOD
if VERSIONS.cluster_era > VERSIONS.ALONZO
else kes.KesScenarios.ALL_INVALID,
)
def test_expired_kes(
self,
cluster_kes: clusterlib.ClusterLib,
cluster_manager: cluster_management.ClusterManager,
worker_id: str,
):
"""Test expired KES.
* start local cluster instance configured with short KES period and low number of key
evolutions, so KES expires soon on all pools
* refresh opcert on 2 of the 3 pools, so KES doesn't expire on those 2 pools and
the pools keep minting blocks
* wait for KES expiration on the selected pool
* check that the pool with expired KES didn't mint blocks in an epoch that followed after
KES expiration
* check KES period info command with an operational certificate with an expired KES
* check KES period info command with operational certificates with a valid KES
"""
cluster = cluster_kes
temp_template = common.get_test_id(cluster)
expire_timeout = 200
expire_node_name = "pool1"
expire_pool_name = f"node-{expire_node_name}"
expire_pool_rec = cluster_manager.cache.addrs_data[expire_pool_name]
expire_pool_id = cluster.get_stake_pool_id(expire_pool_rec["cold_key_pair"].vkey_file)
expire_pool_id_dec = helpers.decode_bech32(expire_pool_id)
# refresh opcert on 2 of the 3 pools, so KES doesn't expire on those 2 pools and
# the pools keep minting blocks
refreshed_nodes = ["pool2", "pool3"]
def _refresh_opcerts():
for n in refreshed_nodes:
refreshed_pool_rec = cluster_manager.cache.addrs_data[f"node-{n}"]
refreshed_opcert_file = cluster.gen_node_operational_cert(
node_name=f"{n}_refreshed_opcert",
kes_vkey_file=refreshed_pool_rec["kes_key_pair"].vkey_file,
cold_skey_file=refreshed_pool_rec["cold_key_pair"].skey_file,
cold_counter_file=refreshed_pool_rec["cold_key_pair"].counter_file,
kes_period=cluster.get_kes_period(),
)
shutil.copy(refreshed_opcert_file, refreshed_pool_rec["pool_operational_cert"])
cluster_nodes.restart_nodes(refreshed_nodes)
_refresh_opcerts()
expected_err_regexes = ["KESKeyAlreadyPoisoned", "KESCouldNotEvolve"]
# ignore expected errors in bft1 node log file, as bft1 opcert will not get refreshed
logfiles.add_ignore_rule(
files_glob="bft1.stdout",
regex="|".join(expected_err_regexes),
ignore_file_id=worker_id,
)
# search for expected errors only in log file corresponding to pool with expired KES
expected_errors = [(f"{expire_node_name}.stdout", err) for err in expected_err_regexes]
this_epoch = -1
with logfiles.expect_errors(expected_errors, ignore_file_id=worker_id):
LOGGER.info(
f"{datetime.datetime.now()}: Waiting for {expire_timeout} sec for KES expiration."
)
time.sleep(expire_timeout)
_wait_epoch_chores(
cluster_obj=cluster, temp_template=temp_template, this_epoch=this_epoch
)
this_epoch = cluster.get_epoch()
# check that the pool is not producing any blocks
blocks_made = clusterlib_utils.get_ledger_state(cluster_obj=cluster)["blocksCurrent"]
if blocks_made:
assert (
expire_pool_id_dec not in blocks_made
), f"The pool '{expire_pool_name}' has minted blocks in epoch {this_epoch}"
# refresh opcerts one more time
_refresh_opcerts()
LOGGER.info(
f"{datetime.datetime.now()}: Waiting 120 secs to make sure the expected errors "
"make it to log files."
)
time.sleep(120)
# check kes-period-info with an operational certificate with KES expired
# TODO: the query is currently broken
kes_query_currently_broken = False
try:
kes_info_expired = cluster.get_kes_period_info(
opcert_file=expire_pool_rec["pool_operational_cert"]
)
except clusterlib.CLIError as err:
if "currentlyBroken" not in str(err):
raise
kes_query_currently_broken = True
if kes_query_currently_broken:
pytest.xfail("`query kes-period-info` is currently broken")
else:
kes.check_kes_period_info_result(
kes_output=kes_info_expired, expected_scenario=kes.KesScenarios.INVALID_KES_PERIOD
)
# check kes-period-info with valid operational certificates
for n in refreshed_nodes:
refreshed_pool_rec = cluster_manager.cache.addrs_data[f"node-{n}"]
kes_info_valid = cluster.get_kes_period_info(
opcert_file=refreshed_pool_rec["pool_operational_cert"]
)
kes.check_kes_period_info_result(
kes_output=kes_info_valid, expected_scenario=kes.KesScenarios.ALL_VALID
)
def test_oversaturated( # noqa: C901
self,
cluster_manager: cluster_management.ClusterManager,
cluster_lock_pools: clusterlib.ClusterLib,
):
"""Check diminished rewards when stake pool is oversaturated.
The stake pool continues to operate normally and those who delegate to that pool receive
rewards, but the rewards are proportionally lower than those received from stake pool
that is not oversaturated.
* register and delegate stake address in "init epoch", for all available pools
* in "init epoch" + 2, saturate all available pools (block distribution remains balanced
among pools)
* in "init epoch" + 3, oversaturate one pool
* in "init epoch" + 5, for all available pools, withdraw rewards and transfer funds
from delegated addresses so pools are no longer (over)saturated
* while doing the steps above, collect rewards data for 9 epochs
* compare proportionality of rewards in epochs where pools were non-saturated,
saturated and oversaturated
"""
# pylint: disable=too-many-statements,too-many-locals,too-many-branches
epoch_saturate = 2
epoch_oversaturate = 4
epoch_withdrawal = 6
cluster = cluster_lock_pools
temp_template = common.get_test_id(cluster)
initial_balance = 1_000_000_000
faucet_rec = cluster_manager.cache.addrs_data["byron000"]
pool_records: Dict[int, PoolRecord] = {}
# make sure we have enough time to finish the delegation in one epoch
clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster,
start=5,
stop=-40)
init_epoch = cluster.get_epoch()
# submit registration certificates and delegate to pools
for idx, res in enumerate(
[
cluster_management.Resources.POOL1,
cluster_management.Resources.POOL2,
cluster_management.Resources.POOL3,
],
start=1,
):
pool_addrs_data = cluster_manager.cache.addrs_data[res]
reward_addr = clusterlib.PoolUser(
payment=pool_addrs_data["payment"],
stake=pool_addrs_data["reward"])
pool_id = delegation.get_pool_id(
cluster_obj=cluster,
addrs_data=cluster_manager.cache.addrs_data,
pool_name=res,
)
pool_id_dec = helpers.decode_bech32(bech32=pool_id)
delegation_out = delegation.delegate_stake_addr(
cluster_obj=cluster,
addrs_data=cluster_manager.cache.addrs_data,
temp_template=f"{temp_template}_pool{idx}",
pool_id=pool_id,
amount=initial_balance,
)
pool_records[idx] = PoolRecord(
name=res,
id=pool_id,
id_dec=pool_id_dec,
reward_addr=reward_addr,
delegation_out=delegation_out,
user_rewards=[],
owner_rewards=[],
blocks_minted={},
saturation_amounts={},
)
# record initial reward balance for each pool
for pool_rec in pool_records.values():
user_payment_balance = cluster.get_address_balance(
pool_rec.delegation_out.pool_user.payment.address)
owner_payment_balance = cluster.get_address_balance(
pool_rec.reward_addr.payment.address)
pool_rec.user_rewards.append(
RewardRecord(
epoch_no=init_epoch,
reward_total=0,
reward_per_epoch=0,
stake_total=user_payment_balance,
))
pool_rec.owner_rewards.append(
RewardRecord(
epoch_no=init_epoch,
reward_total=cluster.get_stake_addr_info(
pool_rec.reward_addr.stake.address).
reward_account_balance,
reward_per_epoch=0,
stake_total=owner_payment_balance,
))
assert (
cluster.get_epoch() == init_epoch
), "Delegation took longer than expected and would affect other checks"
LOGGER.info("Checking rewards for 10 epochs.")
for __ in range(10):
# wait for new epoch
if cluster.get_epoch(
) == pool_records[2].owner_rewards[-1].epoch_no:
cluster.wait_for_new_epoch()
# sleep till the end of epoch
clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster,
start=-50,
stop=-40,
force_epoch=True)
this_epoch = cluster.get_epoch()
ledger_state = clusterlib_utils.get_ledger_state(
cluster_obj=cluster)
clusterlib_utils.save_ledger_state(
cluster_obj=cluster,
state_name=f"{temp_template}_{this_epoch}",
ledger_state=ledger_state,
)
for pool_rec in pool_records.values():
# reward balance in previous epoch
prev_user_reward = pool_rec.user_rewards[-1].reward_total
prev_owner_reward = pool_rec.owner_rewards[-1].reward_total
pool_rec.blocks_minted[this_epoch -
1] = (ledger_state["blocksBefore"].get(
pool_rec.id_dec) or 0)
# current reward balance
user_reward = cluster.get_stake_addr_info(
pool_rec.delegation_out.pool_user.stake.address
).reward_account_balance
owner_reward = cluster.get_stake_addr_info(
pool_rec.reward_addr.stake.address).reward_account_balance
# total reward amounts received this epoch
owner_reward_epoch = owner_reward - prev_owner_reward
# We cannot compare with previous rewards in epochs where
# `this_epoch >= init_epoch + epoch_withdrawal`.
# There's a withdrawal of rewards at the end of these epochs.
if this_epoch > init_epoch + epoch_withdrawal:
user_reward_epoch = user_reward
else:
user_reward_epoch = user_reward - prev_user_reward
# store collected rewards info
user_payment_balance = cluster.get_address_balance(
pool_rec.delegation_out.pool_user.payment.address)
owner_payment_balance = cluster.get_address_balance(
pool_rec.reward_addr.payment.address)
pool_rec.user_rewards.append(
RewardRecord(
epoch_no=this_epoch,
reward_total=user_reward,
reward_per_epoch=user_reward_epoch,
stake_total=user_payment_balance + user_reward,
))
pool_rec.owner_rewards.append(
RewardRecord(
epoch_no=this_epoch,
reward_total=owner_reward,
reward_per_epoch=owner_reward_epoch,
stake_total=owner_payment_balance,
))
pool_rec.saturation_amounts[
this_epoch] = _get_saturation_threshold(
cluster_obj=cluster,
ledger_state=ledger_state,
pool_id=pool_rec.id)
# fund the delegated addresses - saturate all pools
if this_epoch == init_epoch + epoch_saturate:
clusterlib_utils.fund_from_faucet(
*[
p.delegation_out.pool_user.payment
for p in pool_records.values()
],
cluster_obj=cluster,
faucet_data=faucet_rec,
amount=[
p.saturation_amounts[this_epoch] - 100_000_000_000
for p in pool_records.values()
],
tx_name=f"{temp_template}_saturate_pools_ep{this_epoch}",
force=True,
)
with cluster_manager.restart_on_failure():
# Fund the address delegated to "pool2" to oversaturate the pool.
# New stake amount will be current (saturated) stake * 2.
if this_epoch == init_epoch + epoch_oversaturate:
assert (pool_records[2].saturation_amounts[this_epoch] >
0), "Pool is already saturated"
current_stake = int(
cluster.get_stake_snapshot(
pool_records[2].id)["poolStakeMark"])
overstaturate_amount = current_stake * 2
saturation_threshold = pool_records[2].saturation_amounts[
this_epoch]
assert overstaturate_amount > saturation_threshold, (
f"{overstaturate_amount} Lovelace is not enough to oversature the pool "
f"({saturation_threshold} is needed)")
clusterlib_utils.fund_from_faucet(
pool_records[2].delegation_out.pool_user.payment,
cluster_obj=cluster,
faucet_data=faucet_rec,
amount=overstaturate_amount,
tx_name=f"{temp_template}_oversaturate_pool2",
force=True,
)
# wait 4 epochs for first rewards
if this_epoch >= init_epoch + 4:
assert (owner_reward > prev_owner_reward
), "New reward was not received by pool owner"
# transfer funds back to faucet so the pools are no longer (over)saturated
# and staked amount is +- same as the `initial_balance`
if this_epoch >= init_epoch + epoch_withdrawal:
_withdraw_rewards(
*[
p.delegation_out.pool_user
for p in pool_records.values()
],
cluster_obj=cluster,
tx_name=f"{temp_template}_ep{this_epoch}",
)
return_to_addrs = []
return_amounts = []
for idx, pool_rec in pool_records.items():
deleg_payment_balance = cluster.get_address_balance(
pool_rec.delegation_out.pool_user.payment.address)
if deleg_payment_balance > initial_balance + 10_000_000:
return_to_addrs.append(
pool_rec.delegation_out.pool_user.payment)
return_amounts.append(deleg_payment_balance -
initial_balance)
clusterlib_utils.return_funds_to_faucet(
*return_to_addrs,
cluster_obj=cluster,
faucet_addr=faucet_rec["payment"].address,
amount=return_amounts,
tx_name=f"{temp_template}_ep{this_epoch}",
)
for return_addr in return_to_addrs:
deleg_payment_balance = cluster.get_address_balance(
return_addr.address)
assert (
deleg_payment_balance <= initial_balance
), "Unexpected funds in payment address '{return_addr}'"
assert (
cluster.get_epoch() == this_epoch
), "Failed to finish actions in single epoch, it would affect other checks"
pool1_user_rewards_per_block = _get_reward_per_block(pool_records[1])
pool2_user_rewards_per_block = _get_reward_per_block(pool_records[2])
pool3_user_rewards_per_block = _get_reward_per_block(pool_records[3])
pool1_owner_rewards_per_block = _get_reward_per_block(
pool_records[1], owner_rewards=True)
pool2_owner_rewards_per_block = _get_reward_per_block(
pool_records[2], owner_rewards=True)
pool3_owner_rewards_per_block = _get_reward_per_block(
pool_records[3], owner_rewards=True)
oversaturated_epoch = max(
e for e, r in pool_records[2].saturation_amounts.items() if r < 0)
saturated_epoch = oversaturated_epoch - 2
nonsaturated_epoch = oversaturated_epoch - 4
try:
# check that rewards per block per stake for "pool2" in the epoch where the pool is
# oversaturated is lower than in epochs where pools are not oversaturated
assert (pool1_user_rewards_per_block[nonsaturated_epoch] >
pool2_user_rewards_per_block[oversaturated_epoch])
assert (pool2_user_rewards_per_block[nonsaturated_epoch] >
pool2_user_rewards_per_block[oversaturated_epoch])
assert (pool3_user_rewards_per_block[nonsaturated_epoch] >
pool2_user_rewards_per_block[oversaturated_epoch])
assert (pool1_user_rewards_per_block[saturated_epoch] >
pool2_user_rewards_per_block[oversaturated_epoch])
assert (pool2_user_rewards_per_block[saturated_epoch] >
pool2_user_rewards_per_block[oversaturated_epoch])
assert (pool3_user_rewards_per_block[saturated_epoch] >
pool2_user_rewards_per_block[oversaturated_epoch])
# check that oversaturated pool doesn't lead to increased rewards for pool owner
# when compared to saturated pool, i.e. total pool margin amount is not increased
pool1_rew_fraction_sat = pool1_owner_rewards_per_block[
saturated_epoch]
pool2_rew_fraction_sat = pool2_owner_rewards_per_block[
saturated_epoch]
pool3_rew_fraction_sat = pool3_owner_rewards_per_block[
saturated_epoch]
pool2_rew_fraction_over = pool2_owner_rewards_per_block[
oversaturated_epoch]
assert pool2_rew_fraction_sat > pool2_rew_fraction_over or helpers.is_in_interval(
pool2_rew_fraction_sat,
pool2_rew_fraction_over,
frac=0.4,
)
assert pool1_rew_fraction_sat > pool2_rew_fraction_over or helpers.is_in_interval(
pool1_rew_fraction_sat,
pool2_rew_fraction_over,
frac=0.4,
)
assert pool3_rew_fraction_sat > pool2_rew_fraction_over or helpers.is_in_interval(
pool3_rew_fraction_sat,
pool2_rew_fraction_over,
frac=0.4,
)
# Compare rewards in last (non-saturated) epoch to rewards in next-to-last
# (saturated / over-saturated) epoch.
# This way check that staked amount for each pool was restored to `initial_balance`
# and that rewards correspond to the restored amounts.
for pool_rec in pool_records.values():
assert (pool_rec.user_rewards[-1].reward_per_epoch * 100 <
pool_rec.user_rewards[-2].reward_per_epoch)
except Exception:
# save debugging data in case of test failure
with open(f"{temp_template}_pool_records.pickle",
"wb") as out_data:
pickle.dump(pool_records, out_data)
raise
def test_stake_snapshot(self, cluster: clusterlib.ClusterLib): # noqa: C901
"""Test the `stake-snapshot` and `ledger-state` commands and ledger state values."""
# pylint: disable=too-many-statements,too-many-locals,too-many-branches
temp_template = common.get_test_id(cluster)
# make sure the queries can be finished in single epoch
stop = (
20 if cluster_nodes.get_cluster_type().type == cluster_nodes.ClusterType.LOCAL else 200
)
clusterlib_utils.wait_for_epoch_interval(cluster_obj=cluster, start=5, stop=-stop)
stake_pool_ids = cluster.get_stake_pools()
if not stake_pool_ids:
pytest.skip("No stake pools are available.")
if len(stake_pool_ids) > 200:
pytest.skip("Skipping on this testnet, there's too many pools.")
ledger_state = clusterlib_utils.get_ledger_state(cluster_obj=cluster)
clusterlib_utils.save_ledger_state(
cluster_obj=cluster,
state_name=temp_template,
ledger_state=ledger_state,
)
es_snapshot: dict = ledger_state["stateBefore"]["esSnapshots"]
def _get_hashes(snapshot: str) -> Dict[str, int]:
hashes: Dict[str, int] = {}
for r in es_snapshot[snapshot]["stake"]:
r_hash_rec = r[0]
r_hash = r_hash_rec.get("script hash") or r_hash_rec.get("key hash")
if r_hash in hashes:
hashes[r_hash] += r[1]
else:
hashes[r_hash] = r[1]
return hashes
def _get_delegations(snapshot: str) -> Dict[str, List[str]]:
delegations: Dict[str, List[str]] = {}
for r in es_snapshot[snapshot]["delegations"]:
r_hash_rec = r[0]
r_hash = r_hash_rec.get("script hash") or r_hash_rec.get("key hash")
r_pool_id = r[1]
if r_pool_id in delegations:
delegations[r_pool_id].append(r_hash)
else:
delegations[r_pool_id] = [r_hash]
return delegations
errors = []
ledger_state_keys = set(ledger_state)
if ledger_state_keys != LEDGER_STATE_KEYS:
errors.append(
"unexpected ledger state keys: "
f"{ledger_state_keys.difference(LEDGER_STATE_KEYS)} and "
f"{LEDGER_STATE_KEYS.difference(ledger_state_keys)}"
)
# stake addresses (hashes) and corresponding amounts
stake_mark = _get_hashes("pstakeMark")
stake_set = _get_hashes("pstakeSet")
stake_go = _get_hashes("pstakeGo")
# pools (hashes) and stake addresses (hashes) delegated to corresponding pool
delegations_mark = _get_delegations("pstakeMark")
delegations_set = _get_delegations("pstakeSet")
delegations_go = _get_delegations("pstakeGo")
# all delegated stake addresses (hashes)
delegated_hashes_mark = set(itertools.chain.from_iterable(delegations_mark.values()))
delegated_hashes_set = set(itertools.chain.from_iterable(delegations_set.values()))
delegated_hashes_go = set(itertools.chain.from_iterable(delegations_go.values()))
# check if all delegated addresses are listed among stake addresses
stake_hashes_mark = set(stake_mark)
if not delegated_hashes_mark.issubset(stake_hashes_mark):
errors.append(
"for 'mark', some delegations are not listed in 'stake': "
f"{delegated_hashes_mark.difference(stake_hashes_mark)}"
)
stake_hashes_set = set(stake_set)
if not delegated_hashes_set.issubset(stake_hashes_set):
errors.append(
"for 'set', some delegations are not listed in 'stake': "
f"{delegated_hashes_set.difference(stake_hashes_set)}"
)
stake_hashes_go = set(stake_go)
if not delegated_hashes_go.issubset(stake_hashes_go):
errors.append(
"for 'go', some delegations are not listed in 'stake': "
f"{delegated_hashes_go.difference(stake_hashes_go)}"
)
sum_mark = sum_set = sum_go = 0
seen_hashes_mark: Set[str] = set()
seen_hashes_set: Set[str] = set()
seen_hashes_go: Set[str] = set()
delegation_pool_ids = {*delegations_mark, *delegations_set, *delegations_go}
for pool_id_dec in delegation_pool_ids:
pool_id = helpers.encode_bech32(prefix="pool", data=pool_id_dec)
# get stake info from ledger state
pstake_hashes_mark = delegations_mark.get(pool_id_dec) or ()
seen_hashes_mark.update(pstake_hashes_mark)
pstake_amounts_mark = [stake_mark[h] for h in pstake_hashes_mark]
pstake_sum_mark = functools.reduce(lambda x, y: x + y, pstake_amounts_mark, 0)
pstake_hashes_set = delegations_set.get(pool_id_dec) or ()
seen_hashes_set.update(pstake_hashes_set)
pstake_amounts_set = [stake_set[h] for h in pstake_hashes_set]
pstake_sum_set = functools.reduce(lambda x, y: x + y, pstake_amounts_set, 0)
pstake_hashes_go = delegations_go.get(pool_id_dec) or ()
seen_hashes_go.update(pstake_hashes_go)
pstake_amounts_go = [stake_go[h] for h in pstake_hashes_go]
pstake_sum_go = functools.reduce(lambda x, y: x + y, pstake_amounts_go, 0)
# get stake info from `stake-snapshot` command
stake_snapshot = cluster.get_stake_snapshot(stake_pool_id=pool_id)
pstake_mark_cmd = stake_snapshot["poolStakeMark"]
pstake_set_cmd = stake_snapshot["poolStakeSet"]
pstake_go_cmd = stake_snapshot["poolStakeGo"]
if pstake_sum_mark != pstake_mark_cmd:
errors.append(f"pool: {pool_id}, mark:\n {pstake_sum_mark} != {pstake_mark_cmd}")
if pstake_sum_set != pstake_set_cmd:
errors.append(f"pool: {pool_id}, set:\n {pstake_sum_set} != {pstake_set_cmd}")
if pstake_sum_go != pstake_go_cmd:
errors.append(f"pool: {pool_id}, go:\n {pstake_sum_go} != {pstake_go_cmd}")
sum_mark += pstake_mark_cmd
sum_set += pstake_set_cmd
sum_go += pstake_go_cmd
if seen_hashes_mark != delegated_hashes_mark:
errors.append(
"seen hashes and existing hashes differ for 'mark': "
f"{seen_hashes_mark.difference(delegated_hashes_mark)} and "
f"{delegated_hashes_mark.difference(seen_hashes_mark)}"
)
if seen_hashes_set != delegated_hashes_set:
errors.append(
"seen hashes and existing hashes differ for 'set': "
f"{seen_hashes_set.difference(delegated_hashes_set)} and "
f"{delegated_hashes_set.difference(seen_hashes_set)}"
)
if seen_hashes_go != delegated_hashes_go:
errors.append(
"seen hashes and existing hashes differ for 'go': "
f"{seen_hashes_go.difference(delegated_hashes_go)} and "
f"{delegated_hashes_go.difference(seen_hashes_go)}"
)
# active stake can be lower than sum of stakes, as some pools may not be running
# and minting blocks
if sum_mark < stake_snapshot["activeStakeMark"]:
errors.append(f"active_mark: {sum_mark} < {stake_snapshot['activeStakeMark']}")
if sum_set < stake_snapshot["activeStakeSet"]:
errors.append(f"active_set: {sum_set} < {stake_snapshot['activeStakeSet']}")
if sum_go < stake_snapshot["activeStakeGo"]:
errors.append(f"active_go: {sum_go} < {stake_snapshot['activeStakeGo']}")
if errors:
err_joined = "\n".join(errors)
pytest.fail(f"Errors:\n{err_joined}")
def test_pool_blocks(
self,
skip_leadership_schedule: None,
cluster_manager: cluster_management.ClusterManager,
cluster_use_pool3: clusterlib.ClusterLib,
for_epoch: str,
):
"""Check that blocks were minted according to leadership schedule.
* query leadership schedule for selected pool for current epoch or next epoch
* wait for epoch that comes after the queried epoch
* get info about minted blocks in queried epoch for the selected pool
* compare leadership schedule with blocks that were actually minted
* compare db-sync records with ledger state dump
"""
# pylint: disable=unused-argument
cluster = cluster_use_pool3
temp_template = common.get_test_id(cluster)
pool_name = cluster_management.Resources.POOL3
pool_rec = cluster_manager.cache.addrs_data[pool_name]
pool_id = cluster.get_stake_pool_id(
pool_rec["cold_key_pair"].vkey_file)
if for_epoch == "current":
# wait for beginning of an epoch
queried_epoch = cluster.wait_for_new_epoch(padding_seconds=5)
else:
# wait for stable stake distribution for next epoch, that is last 300 slots of
# current epoch
clusterlib_utils.wait_for_epoch_interval(
cluster_obj=cluster,
start=-int(300 * cluster.slot_length),
stop=-10,
check_slot=True,
)
queried_epoch = cluster.get_epoch() + 1
# query leadership schedule for selected pool
# TODO: the query is currently broken
query_currently_broken = False
try:
leadership_schedule = cluster.get_leadership_schedule(
vrf_skey_file=pool_rec["vrf_key_pair"].skey_file,
cold_vkey_file=pool_rec["cold_key_pair"].vkey_file,
for_next=for_epoch != "current",
)
except clusterlib.CLIError as err:
if "currently broken" not in str(err):
raise
query_currently_broken = True
if query_currently_broken:
pytest.xfail("`query leadership-schedule` is currently broken")
# wait for epoch that comes after the queried epoch
cluster.wait_for_new_epoch(
new_epochs=1 if for_epoch == "current" else 2)
# get info about minted blocks in queried epoch for the selected pool
minted_blocks = list(
dbsync_queries.query_blocks(pool_id_bech32=pool_id,
epoch_from=queried_epoch,
epoch_to=queried_epoch))
slots_when_minted = {r.slot_no for r in minted_blocks}
errors: List[str] = []
# compare leadership schedule with blocks that were actually minted
slots_when_scheduled = {r.slot_no for r in leadership_schedule}
difference_scheduled = slots_when_minted.difference(
slots_when_scheduled)
if difference_scheduled:
errors.append(
f"Some blocks were minted in other slots than scheduled: {difference_scheduled}"
)
difference_minted = slots_when_scheduled.difference(slots_when_minted)
if len(difference_minted) > len(leadership_schedule) // 2:
errors.append(f"Lot of slots missed: {difference_minted}")
# compare db-sync records with ledger state dump
ledger_state = clusterlib_utils.get_ledger_state(cluster_obj=cluster)
clusterlib_utils.save_ledger_state(
cluster_obj=cluster,
state_name=temp_template,
ledger_state=ledger_state,
)
blocks_before: Dict[str, int] = ledger_state["blocksBefore"]
pool_id_dec = helpers.decode_bech32(pool_id)
minted_blocks_ledger = blocks_before.get(pool_id_dec) or 0
minted_blocks_db = len(slots_when_minted)
if minted_blocks_ledger != minted_blocks_db:
errors.append(
"Numbers of minted blocks reported by ledger state and db-sync don't match: "
f"{minted_blocks_ledger} vs {minted_blocks_db}")
if errors:
err_joined = "\n".join(errors)
pytest.fail(f"Errors:\n{err_joined}")
