def testLoggingTxnStateForInvalidRequest(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
logsearch):
logsPropagate, _ = logsearch(files=['propagator.py'],
funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsReject, _ = logsearch(
files=['replica.py'],
funcs=['consume_req_queue_for_pre_prepare'],
msgs=['encountered exception.*while processing.*will reject'])
seed = randomString(32)
wh, _ = sdk_wallet_client
nym_request, _ = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_client, seed, "name", STEWARD_STRING))
request_couple = sdk_sign_and_send_prepared_request(
looper, sdk_wallet_client, sdk_pool_handle, nym_request)
with pytest.raises(RequestRejectedException) as e:
sdk_get_and_check_replies(looper, [request_couple])
assert 'Only Steward is allowed to do these transactions' in e._excinfo[
1].args[0]
request = request_couple[0]
req_id = str(request[f.REQ_ID.nm])
digest = get_key_from_req(request)
assert any(digest in record.getMessage() for record in logsPropagate)
assert any(req_id in record.getMessage() for record in logsReject)
def testLoggingTxnStateForInvalidRequest(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, logsearch):
logsPropagate, _ = logsearch(files=['propagator.py'], funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsReject, _ = logsearch(files=['replica.py'], funcs=['consume_req_queue_for_pre_prepare'],
msgs=['encountered exception.*while processing.*will reject'])
seed = randomString(32)
wh, _ = sdk_wallet_client
nym_request, _ = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_client, seed,
"name", STEWARD_STRING))
request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client,
sdk_pool_handle, nym_request)
with pytest.raises(RequestRejectedException) as e:
sdk_get_and_check_replies(looper, [request_couple])
assert 'Only Steward is allowed to do these transactions' in e._excinfo[1].args[0]
request = json.loads(nym_request)
req_id = str(request[f.REQ_ID.nm])
digest = get_key_from_req(request)
assert any(digest in record.getMessage() for record in logsPropagate)
assert any(req_id in record.getMessage() for record in logsReject)
def testLoggingTxnStateForValidRequest(looper, logsearch, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
logsPropagate, _ = logsearch(levels=['INFO'],
files=['propagator.py'],
funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsOrdered, _ = logsearch(levels=['INFO'],
files=['replica.py'],
funcs=['order_3pc_key'],
msgs=['ordered batch request'])
logsCommited, _ = logsearch(levels=['INFO'],
files=['node.py'],
funcs=['executeBatch'],
msgs=['committed batch request'])
reqs = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
req, _ = reqs[0]
key = get_key_from_req(req)
assert any(key in record.getMessage() for record in logsPropagate)
assert any(key in record.getMessage() for record in logsOrdered)
assert any(key in record.getMessage() for record in logsCommited)
def testReqLatencyThreshold(looper, txnPoolNodeSet, requests):
for node in txnPoolNodeSet:
for rq in requests:
key = get_key_from_req(rq)
assert key in node.monitor.masterReqLatenciesTest
assert node.monitor.masterReqLatenciesTest[
key] <= node.monitor.Lambda
def testLoggingTxnStateForValidRequest(
looper, logsearch, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
logsPropagate, _ = logsearch(files=['propagator.py'], funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsOrdered, _ = logsearch(files=['replica.py'], funcs=['order_3pc_key'], msgs=['ordered batch request'])
logsCommited, _ = logsearch(files=['node.py'], funcs=['executeBatch'], msgs=['committed batch request'])
reqs = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
req, _ = reqs[0]
key = get_key_from_req(req)
assert any(key in record.getMessage() for record in logsPropagate)
assert any(key in record.getMessage() for record in logsOrdered)
assert any(key in record.getMessage() for record in logsCommited)
def testLoggingTxnStateWhenCommitFails(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, logsearch):
logsPropagate, _ = logsearch(files=['propagator.py'], funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsOrdered, _ = logsearch(files=['replica.py'], funcs=['order_3pc_key'], msgs=['ordered batch request'])
logsCommitFail, _ = logsearch(files=['node.py'], funcs=['executeBatch'],
msgs=['commit failed for batch request'])
seed = randomString(32)
wh, _ = sdk_wallet_steward
nym_request, _ = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_steward, seed,
"name", None))
sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward,
sdk_pool_handle, nym_request)
class SomeError(Exception):
pass
def commitPatched(node, commitOrig, *args, **kwargs):
req_handler = node.get_req_handler(ledger_id=DOMAIN_LEDGER_ID)
req_handler.commit = commitOrig
raise SomeError(ERORR_MSG)
excCounter = 0
def executeBatchPatched(node, executeBatchOrig, *args, **kwargs):
nonlocal excCounter
try:
executeBatchOrig(*args, **kwargs)
except SomeError:
excCounter += 1
node.executeBatch = executeBatchOrig
pass
def checkSufficientExceptionsHappend():
assert excCounter == len(txnPoolNodeSet)
return
for node in txnPoolNodeSet:
req_handler = node.get_req_handler(ledger_id=DOMAIN_LEDGER_ID)
req_handler.commit = functools.partial(
commitPatched, node, req_handler.commit
)
node.executeBatch = functools.partial(
executeBatchPatched, node, node.executeBatch
)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(
eventually(checkSufficientExceptionsHappend,
retryWait=1, timeout=timeout))
request = json.loads(nym_request)
digest = get_key_from_req(request)
assert any(digest in record.getMessage() for record in logsPropagate)
assert any(digest in record.getMessage() for record in logsOrdered)
assert any(digest in record.getMessage() for record in logsCommitFail)
assert any(ERORR_MSG in record.getMessage() for record in logsCommitFail)
def testLoggingTxnStateWhenCommitFails(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, logsearch):
logsPropagate, _ = logsearch(files=['propagator.py'],
funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsOrdered, _ = logsearch(files=['replica.py'],
funcs=['order_3pc_key'],
msgs=['ordered batch request'])
logsCommitFail, _ = logsearch(files=['node.py'],
funcs=['executeBatch'],
msgs=['commit failed for batch request'])
seed = randomString(32)
wh, _ = sdk_wallet_steward
nym_request, _ = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_steward, seed, "name", None))
req_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward,
sdk_pool_handle,
nym_request)
class SomeError(Exception):
pass
def commitPatched(node, commitOrig, *args, **kwargs):
req_handler = node.get_req_handler(ledger_id=DOMAIN_LEDGER_ID)
req_handler.commit = commitOrig
raise SomeError(ERORR_MSG)
excCounter = 0
def executeBatchPatched(node, executeBatchOrig, *args, **kwargs):
nonlocal excCounter
try:
executeBatchOrig(*args, **kwargs)
except SomeError:
excCounter += 1
node.executeBatch = executeBatchOrig
pass
def checkSufficientExceptionsHappend():
assert excCounter == len(txnPoolNodeSet)
return
for node in txnPoolNodeSet:
req_handler = node.get_req_handler(ledger_id=DOMAIN_LEDGER_ID)
req_handler.commit = functools.partial(commitPatched, node,
req_handler.commit)
node.executeBatch = functools.partial(executeBatchPatched, node,
node.executeBatch)
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.run(
eventually(checkSufficientExceptionsHappend,
retryWait=1,
timeout=timeout))
request = req_couple[0]
digest = get_key_from_req(request)
assert any(digest in record.getMessage() for record in logsPropagate)
assert any(digest in record.getMessage() for record in logsOrdered)
assert any(digest in record.getMessage() for record in logsCommitFail)
assert any(ERORR_MSG in record.getMessage() for record in logsCommitFail)
def test_commit_signature_validation_integration(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
sdk_wallet_client,
tconf,
tdir):
'''
All nodes receive PrePrepare1(txn1 for pool_ledger)
Nodes 1, 2 ordered txn1 and nodes 3, 4 did not.
All nodes receive PrePrepare2(txn2 for domain_ledger)
Nodes 3, 4 receive commits from nodes 1, 2
Nodes 3, 4 ordered txn1
Check that all nodes ordered txn2
'''
fast_nodes = txnPoolNodeSet[:2]
slow_nodes = txnPoolNodeSet[2:]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# create new steward
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="testClientSteward945",
role=STEWARD_STRING)
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \
prepare_new_node_data(tconf, tdir, "new_node")
# create node request to add new demote node
_, steward_did = new_steward_wallet_handle
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name="new_node",
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
services=[],
key_proof=key_proof))
first_ordered = txnPoolNodeSet[0].master_last_ordered_3PC
with ord_delay(slow_nodes):
request1 = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle,
sdk_pool_handle, node_request)
key1 = get_key_from_req(request1[0])
def check_nodes_receive_pp(view_no, seq_no):
for node in txnPoolNodeSet:
assert node.master_replica._ordering_service.get_preprepare(view_no, seq_no)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 1))
def check_fast_nodes_ordered_request():
for n in fast_nodes:
assert key1 not in n.requests or n.requests[key1].executed
for n in slow_nodes:
assert not n.requests[key1].executed
looper.run(eventually(check_fast_nodes_ordered_request))
request2 = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 2))
def check_nodes_receive_commits(view_no, seq_no):
for node in txnPoolNodeSet:
assert len(node.master_replica._ordering_service.commits[view_no, seq_no].voters) >= node.f + 1
looper.run(eventually(check_nodes_receive_commits, first_ordered[0], first_ordered[1] + 2))
sdk_get_and_check_replies(looper, [request1])
sdk_get_and_check_replies(looper, [request2])
def test_commit_signature_validation_integration(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
sdk_wallet_client,
tconf,
tdir):
'''
All nodes receive PrePrepare1(txn1 for pool_ledger)
Nodes 1, 2 ordered txn1 and nodes 3, 4 did not.
All nodes receive PrePrepare2(txn2 for domain_ledger)
Nodes 3, 4 receive commits from nodes 1, 2
Nodes 3, 4 ordered txn1
Check that all nodes ordered txn2
'''
fast_nodes = txnPoolNodeSet[:2]
slow_nodes = txnPoolNodeSet[2:]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# create new steward
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="testClientSteward945",
role=STEWARD_STRING)
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \
prepare_new_node_data(tconf, tdir, "new_node")
# create node request to add new demote node
_, steward_did = new_steward_wallet_handle
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name="new_node",
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
services=[],
key_proof=key_proof))
first_ordered = txnPoolNodeSet[0].master_last_ordered_3PC
with ord_delay(slow_nodes):
request1 = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle,
sdk_pool_handle, node_request)
key1 = get_key_from_req(request1[0])
def check_nodes_receive_pp(view_no, seq_no):
for node in txnPoolNodeSet:
assert node.master_replica.getPrePrepare(view_no, seq_no)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 1))
def check_fast_nodes_ordered_request():
for n in fast_nodes:
assert key1 not in n.requests or n.requests[key1].executed
for n in slow_nodes:
assert not n.requests[key1].executed
looper.run(eventually(check_fast_nodes_ordered_request))
request2 = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 2))
def check_nodes_receive_commits(view_no, seq_no):
for node in txnPoolNodeSet:
assert len(node.master_replica.commits[view_no, seq_no].voters) >= node.f + 1
looper.run(eventually(check_nodes_receive_commits, first_ordered[0], first_ordered[1] + 2))
sdk_get_and_check_replies(looper, [request1])
sdk_get_and_check_replies(looper, [request2])
def testReqLatencyThreshold(looper, txnPoolNodeSet, requests):
for node in txnPoolNodeSet:
for rq in requests:
key = get_key_from_req(rq)
assert key in node.monitor.masterReqLatenciesTest
assert node.monitor.masterReqLatenciesTest[key] <= node.monitor.Lambda
