def test_non_primary_accepts_pre_prepare_time(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle):
"""
One of the non-primary has an in-correct clock so it thinks PRE-PREPARE
has incorrect time
"""
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=2)
# send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 2)
# The replica having the bad clock
confused_npr = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1]
make_clock_faulty(confused_npr.node)
old_acceptable_rvs = getAllReturnVals(
confused_npr, confused_npr.is_pre_prepare_time_acceptable)
old_susp_count = get_timestamp_suspicion_count(confused_npr.node)
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=2)
assert get_timestamp_suspicion_count(confused_npr.node) > old_susp_count
new_acceptable_rvs = getAllReturnVals(
confused_npr, confused_npr.is_pre_prepare_time_acceptable)
# `is_pre_prepare_time_acceptable` first returned False then returned True
assert [True, False, *old_acceptable_rvs] == new_acceptable_rvs
def test_non_primary_accepts_pre_prepare_time(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
One of the non-primary has an in-correct clock so it thinks PRE-PREPARE
has incorrect time
"""
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=2)
# send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 2)
# The replica having the bad clock
confused_npr = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1]
make_clock_faulty(confused_npr.node)
old_acceptable_rvs = getAllReturnVals(
confused_npr, confused_npr.is_pre_prepare_time_acceptable)
old_susp_count = get_timestamp_suspicion_count(confused_npr.node)
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=2)
assert get_timestamp_suspicion_count(confused_npr.node) > old_susp_count
new_acceptable_rvs = getAllReturnVals(
confused_npr, confused_npr.is_pre_prepare_time_acceptable)
# `is_pre_prepare_time_acceptable` first returned False then returned True
assert [True, False, *old_acceptable_rvs] == new_acceptable_rvs
def test_nodes_with_bad_clock(tconf, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
All nodes have bad clocks but they eventaully get repaired, an example of
nodes being cut off from NTP server for some time or NTP sync disabled
then without node restart NTP sync enabled
"""
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 3)
ledger_sizes = {node.name: node.domainLedger.size for node in
txnPoolNodeSet}
susp_counts = {node.name: get_timestamp_suspicion_count(node) for node in
txnPoolNodeSet}
for node in txnPoolNodeSet:
make_clock_faulty(
node,
clock_slow_by_sec=node.config.ACCEPTABLE_DEVIATION_PREPREPARE_SECS +
randint(
5,
15),
ppr_always_wrong=False)
for _ in range(5):
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.runFor(.2)
# Let some time pass
looper.runFor(3)
def chk():
for node in txnPoolNodeSet:
# Each node raises suspicion
assert get_timestamp_suspicion_count(node) > susp_counts[node.name]
# Ledger does not change
assert node.domainLedger.size == ledger_sizes[node.name]
looper.run(eventually(chk, retryWait=1))
# Fix clocks
for node in txnPoolNodeSet:
def utc_epoch(self) -> int:
return get_utc_epoch()
node.utc_epoch = types.MethodType(utc_epoch, node)
# Let some more time pass
looper.runFor(3)
# All nodes reply
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 2)
def test_new_primary_has_wrong_clock(tconf, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
One of non-primary has a bad clock, it raises suspicions but orders
requests after getting PREPAREs. Then a view change happens this
non-primary with the bad clock becomes the new primary but is not able to
get any of it's PRE-PREPAREs ordered. Eventually another view change
happens and a new primary is elected the pool is functional again
:return:
"""
# The node having the bad clock, this node will be primary after view
# change
faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[0].node
make_clock_faulty(faulty_node)
assert not faulty_node.master_replica.isPrimary
# faulty_node replies too
ledger_sizes = {
node.name: node.domainLedger.size
for node in txnPoolNodeSet
}
susp_counts = {
node.name: get_timestamp_suspicion_count(node)
for node in txnPoolNodeSet
}
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# After view change, faulty_node is primary.
# But after it sent first pp, new view change happens
assert txnPoolNodeSet[2].master_replica.isPrimary
def chk():
for node in txnPoolNodeSet:
assert node.viewNo == 2
for node in [n for n in txnPoolNodeSet if n != faulty_node]:
# Each non faulty node raises suspicion
assert get_timestamp_suspicion_count(node) > susp_counts[node.name]
# 1 view txn wasn't written
assert any(txn[1]['txn']['data']['viewNo'] == 2
for txn in node.auditLedger.getAllTxn())
assert not any(txn[1]['txn']['data']['viewNo'] == 1
for txn in node.auditLedger.getAllTxn())
assert faulty_node.domainLedger.size == ledger_sizes[faulty_node.name]
looper.run(eventually(chk, retryWait=1))
# All nodes reply
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 2)
def test_new_primary_has_wrong_clock(tconf, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
One of non-primary has a bad clock, it raises suspicions but orders
requests after getting PREPAREs. Then a view change happens this
non-primary with the bad clock becomes the new primary but is not able to
get any of it's PRE-PREPAREs ordered. Eventually another view change
happens and a new primary is elected the pool is functional again
:return:
"""
# The node having the bad clock, this node will be primary after view
# change
faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[0].node
make_clock_faulty(faulty_node)
assert not faulty_node.master_replica.isPrimary
# faulty_node replies too
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 3)
ledger_sizes = {
node.name: node.domainLedger.size for node in txnPoolNodeSet}
susp_counts = {node.name: get_timestamp_suspicion_count(
node) for node in txnPoolNodeSet}
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# After view change, faulty_node is primary
assert faulty_node.master_replica.isPrimary
old_view_no = txnPoolNodeSet[0].viewNo
# Delay instance change so view change doesn't happen in the middle of this test
stashers = (n.nodeIbStasher for n in txnPoolNodeSet)
with delay_rules(stashers, icDelay()):
# Requests are sent
for _ in range(5):
sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
count=2)
looper.runFor(2)
def chk():
for node in txnPoolNodeSet:
assert node.viewNo == old_view_no
for node in [n for n in txnPoolNodeSet if n != faulty_node]:
# Each non faulty node raises suspicion
assert get_timestamp_suspicion_count(node) > susp_counts[node.name]
# Ledger does not change
assert node.domainLedger.size == ledger_sizes[node.name]
assert faulty_node.domainLedger.size == ledger_sizes[faulty_node.name]
looper.run(eventually(chk, retryWait=1))
# Eventually another view change happens
looper.run(eventually(checkViewNoForNodes, txnPoolNodeSet, old_view_no + 1,
retryWait=1, timeout=2 * tconf.PerfCheckFreq))
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# After view change, faulty_node is no more the primary
assert not faulty_node.master_replica.isPrimary
# All nodes reply
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 2)
def test_new_primary_has_wrong_clock(tconf, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
One of non-primary has a bad clock, it raises suspicions but orders
requests after getting PREPAREs. Then a view change happens this
non-primary with the bad clock becomes the new primary but is not able to
get any of it's PRE-PREPAREs ordered. Eventually another view change
happens and a new primary is elected the pool is functional again
:return:
"""
# The node having the bad clock, this node will be primary after view
# change
faulty_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[0].node
make_clock_faulty(faulty_node)
assert not faulty_node.master_replica.isPrimary
# faulty_node replies too
ledger_sizes = {
node.name: node.domainLedger.size for node in txnPoolNodeSet}
susp_counts = {node.name: get_timestamp_suspicion_count(
node) for node in txnPoolNodeSet}
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# After view change, faulty_node is primary
assert faulty_node.master_replica.isPrimary
old_view_no = txnPoolNodeSet[0].viewNo
# Delay instance change so view change doesn't happen in the middle of this test
stashers = (n.nodeIbStasher for n in txnPoolNodeSet)
with delay_rules(stashers, icDelay()):
# Requests are sent
for _ in range(5):
sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
count=2)
looper.runFor(2)
def chk():
for node in txnPoolNodeSet:
assert node.viewNo == old_view_no
for node in [n for n in txnPoolNodeSet if n != faulty_node]:
# Each non faulty node raises suspicion
assert get_timestamp_suspicion_count(node) > susp_counts[node.name]
# Ledger does not change
assert node.domainLedger.size == ledger_sizes[node.name]
assert faulty_node.domainLedger.size == ledger_sizes[faulty_node.name]
looper.run(eventually(chk, retryWait=1))
# Eventually another view change happens
ensure_view_change(looper, txnPoolNodeSet)
looper.run(eventually(checkViewNoForNodes, txnPoolNodeSet, old_view_no + 1,
retryWait=1, timeout=2 * tconf.PerfCheckFreq))
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
# After view change, faulty_node is no more the primary
assert not faulty_node.master_replica.isPrimary
# All nodes reply
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
count=Max3PCBatchSize * 2)