def test_resend_instance_change_messages(looper,
txnPoolNodeSet,
tconf,
sdk_wallet_steward,
sdk_pool_handle):
primary_node = txnPoolNodeSet[0]
old_view_no = checkViewNoForNodes(txnPoolNodeSet, 0)
assert primary_node.master_replica.isPrimary
for n in txnPoolNodeSet:
n.nodeIbStasher.delay(icDelay(3 * tconf.INSTANCE_CHANGE_TIMEOUT))
assert set([n.view_changer.instance_change_rounds for n in txnPoolNodeSet]) == {0}
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
primary_node,
stopNode=False)
txnPoolNodeSet.remove(primary_node)
looper.run(eventually(partial(check_count_connected_node, txnPoolNodeSet, 4),
timeout=5,
acceptableExceptions=[AssertionError]))
looper.runFor(2*tconf.INSTANCE_CHANGE_TIMEOUT)
assert set([n.view_changer.instance_change_rounds for n in txnPoolNodeSet]) == {1}
looper.runFor(tconf.INSTANCE_CHANGE_TIMEOUT)
looper.run(eventually(partial(checkViewNoForNodes, txnPoolNodeSet, expectedViewNo=old_view_no + 1),
timeout=tconf.VIEW_CHANGE_TIMEOUT))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def do_view_change_with_unaligned_prepare_certificates(
slow_nodes, nodes, looper, sdk_pool_handle, sdk_wallet_client):
"""
Perform view change with some nodes reaching lower last prepared certificate than others.
With current implementation of view change this can result with view change taking a lot of time.
"""
fast_nodes = [n for n in nodes if n not in slow_nodes]
all_stashers = [n.nodeIbStasher for n in nodes]
slow_stashers = [n.nodeIbStasher for n in slow_nodes]
# Delay some PREPAREs and all COMMITs
with delay_rules(slow_stashers, pDelay()):
with delay_rules(all_stashers, cDelay()):
# Send request
request = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
# Wait until this request is prepared on fast nodes
looper.run(eventually(check_last_prepared_certificate, fast_nodes, (0, 1)))
# Make sure its not prepared on slow nodes
looper.run(eventually(check_last_prepared_certificate, slow_nodes, None))
# Trigger view change
for n in nodes:
n.view_changer.on_master_degradation()
# Now commits are processed
# Wait until view change is complete
looper.run(eventually(check_view_change_done, nodes, 1, timeout=60))
# Finish request gracefully
sdk_get_reply(looper, request)
def test_send_proof_works(aliceAgent, aliceAcceptedFaber, aliceAcceptedAcme,
acmeAgent, emptyLooper):
# 1. request Claims from Faber
faberLink = aliceAgent.wallet.getLink('Faber College')
name, version, origin = faberLink.availableClaims[0]
schemaKey = SchemaKey(name, version, origin)
aliceAgent.sendReqClaim(faberLink, schemaKey)
# 2. check that claim is received from Faber
async def chkClaims():
claim = await aliceAgent.prover.wallet.getClaimSignature(ID(schemaKey))
assert claim.primaryClaim
emptyLooper.run(eventually(chkClaims, timeout=waits.expectedClaimsReceived()))
# 3. send Proof Request to Alice
alice_link = acmeAgent.wallet.getLink('Alice')
acmeAgent.sendProofReq(alice_link, 'Job-Application-v0.3')
def chkProofRequest():
assert len(aliceAgent.wallet.getMatchingLinksWithProofReq("Job-Application-2", "Acme Corp")) > 0
emptyLooper.run(eventually(chkProofRequest, timeout=waits.expectedClaimsReceived()))
# 4. send proof to Acme
acme_link, acme_proof_req = aliceAgent.wallet.getMatchingLinksWithProofReq("Job-Application-2", "Acme Corp")[0]
aliceAgent.sendProof(acme_link, acme_proof_req)
# 5. check that proof is verified by Acme
def chkProof():
internalId = acmeAgent.get_internal_id_by_nonce(acme_link.invitationNonce)
link = acmeAgent.wallet.getLinkBy(internalId=internalId)
assert "Job-Application-2" in link.verifiedClaimProofs
emptyLooper.run(eventually(chkProof, timeout=waits.expectedClaimsReceived()))
def test_request_older_than_stable_checkpoint_removed(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, reqs_for_checkpoint):
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
max_batch_size = chkFreqPatched.Max3PCBatchSize
# Send some requests (insufficient for checkpoint),
# wait replies and check that current checkpoint is not stable
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2 * max_batch_size)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size, 2)
# From the steward send a request creating a user with None role
sdk_wallet_user = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 1, 3)
# From the created user send a request creating another user.
# Dynamic validation of this request must fail since a user with None role cannot create users.
# However, the 3PC-batch with the sent request must be ordered.
with pytest.raises(RequestRejectedException):
sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_user)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 2 * max_batch_size + 2, 4)
# Send more requests to cause checkpoint stabilization
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, max_batch_size)
# Check that checkpoint is stable now
# and verify that requests for it were removed
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 1, 0, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 0, 0)
# Send more requests to cause new checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, reqs_for_checkpoint + 1)
looper.run(eventually(chkChkpoints, txnPoolNodeSet, 2, 0, retryWait=1, timeout=timeout))
checkRequestCounts(txnPoolNodeSet, 1, 1)
def test_zstack_non_utf8(tdir, looper, tconf):
"""
ZStack gets a non utf-8 message and does not hand it over to the
processing method
:return:
"""
names = ['Alpha', 'Beta']
genKeys(tdir, names)
(alpha, beta), (alphaP, betaP) = create_and_prep_stacks(names, tdir,
looper, tconf)
# Send a utf-8 message and see its received
for uid in alpha.remotes:
alpha.transmit(b'{"k1": "v1"}', uid, serialized=True)
looper.run(eventually(chkPrinted, betaP, {"k1": "v1"}))
# Send a non utf-8 message and see its not received (by the receiver method)
for uid in alpha.remotes:
alpha.transmit(b'{"k2": "v2\x9c"}', uid, serialized=True)
with pytest.raises(AssertionError):
looper.run(eventually(chkPrinted, betaP, {"k2": "v2\x9c"}))
# TODO: A better test where the output of the parsing method is checked
# requires spyable methods
# Again send a utf-8 message and see its received (checks if stack is
# functional after receiving a bad message)
for uid in alpha.remotes:
alpha.transmit(b'{"k3": "v3"}', uid, serialized=True)
looper.run(eventually(chkPrinted, betaP, {"k3": "v3"}))
def test_propagate_of_ordered_request_doesnt_stash_requests_in_authenticator(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
# Universal delayer
def stopAll(msg):
return 100000
def check_verified_req_list_is_empty():
for node in txnPoolNodeSet:
assert len(node.clientAuthNr._verified_reqs) == 0
# Order one request while cutting off last node
lastNode = txnPoolNodeSet[-1]
with delay_rules(lastNode.nodeIbStasher, stopAll), \
delay_rules(lastNode.clientIbStasher, stopAll):
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, 1)
old_propagates = [n.spylog.count('processPropagate') for n in txnPoolNodeSet]
def check_more_propagates_delivered():
new_propagates = [n.spylog.count('processPropagate') for n in txnPoolNodeSet]
assert all(old < new for old, new in zip(old_propagates, new_propagates))
# Wait until more propagates are delivered to all nodes
looper.run(eventually(check_more_propagates_delivered))
# Make sure that verified req list will be empty eventually
looper.run(eventually(check_verified_req_list_is_empty))
def testOrderingCase1(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle):
"""
Scenario -> PRE-PREPARE not received by the replica, Request not received
for ordering by the replica, but received enough commits to start ordering.
It queues up the request so when a PRE-PREPARE is received or request is
receievd for ordering, an order can be triggered
https://www.pivotaltracker.com/story/show/125239401
Reproducing by - Pick a node with no primary replica, replica ignores
forwarded request to replica and delay reception of PRE-PREPARE sufficiently
so that enough COMMITs reach to trigger ordering.
"""
delay = 10
replica = getNonPrimaryReplicas(txnPoolNodeSet, instId=0)[0]
delaysPrePrepareProcessing(replica.node, delay=delay, instId=0)
def doNotProcessReqDigest(self, _):
pass
patchedMethod = types.MethodType(doNotProcessReqDigest, replica)
replica.processRequest = patchedMethod
def chk(n):
assert replica.spylog.count(replica.doOrder.__name__) == n
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
timeout = delay - 5
looper.run(eventually(chk, 0, retryWait=1, timeout=timeout))
timeout = delay + 5
looper.run(eventually(chk, 1, retryWait=1, timeout=timeout))
def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, txnPoolNodeSet):
maliciousNode = txnPoolNodeSet[0]
for i in range(0, 5):
maliciousNode.send(maliciousNode.view_changer._create_instance_change_msg(i, 0))
def chk(instId):
for node in txnPoolNodeSet:
if node.name != maliciousNode.name:
args = getAllArgs(node, ViewChanger.process_instance_change_msg)
assert len(args) == 5
for arg in args:
assert arg['frm'] == maliciousNode.name
numOfNodes = len(txnPoolNodeSet)
instanceChangeTimeout = waits.expectedPoolViewChangeStartedTimeout(
numOfNodes)
for i in range(0, 5):
looper.run(eventually(chk, i, retryWait=1,
timeout=instanceChangeTimeout))
def g():
for node in txnPoolNodeSet:
if node.name != maliciousNode.name:
frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode)
assert frm == maliciousNode.name
assert isinstance(reason, SuspiciousNode)
suspectingNodes = \
getNodeSuspicions(node,
Suspicions.FREQUENT_INST_CHNG.code)
assert len(suspectingNodes) == 13
timeout = waits.expectedTransactionExecutionTime(numOfNodes)
looper.run(eventually(g, retryWait=1, timeout=timeout))
def test_api():
loop = asyncio.get_event_loop()
with pytest.raises(PlenumValueError):
loop.run_until_complete(eventually(lambda x: True, timeout=0))
with pytest.raises(PlenumValueError):
loop.run_until_complete(eventually(lambda x: True, timeout=250))
loop.close()
def do_view_change_with_delayed_commits_on_all_but_one(nodes, nodes_without_one_stashers,
except_node,
looper,
sdk_pool_handle,
sdk_wallet_client):
new_view_no = except_node.viewNo + 1
old_last_ordered = except_node.master_replica.last_ordered_3pc
# delay commits for all nodes except node X
with delay_rules(nodes_without_one_stashers, cDelay(sys.maxsize)):
# send one request
requests2 = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
def last_ordered(node: Node, last_ordered):
assert node.master_replica.last_ordered_3pc == last_ordered
# wait until except_node ordered txn
looper.run(
eventually(last_ordered, except_node, (except_node.viewNo,
old_last_ordered[1] + 1)))
# trigger view change on all nodes
for node in nodes:
node.view_changer.on_master_degradation()
# wait for view change done on all nodes
looper.run(eventually(view_change_done, nodes, new_view_no))
sdk_get_replies(looper, requests2)
def disconnect(looper, disconnected_stack, connection_timeout):
disconnected_motor, other_stacks = disconnected_stack
looper.run(eventually(
checkStackDisonnected, disconnected_motor.stack, other_stacks,
retryWait=1, timeout=connection_timeout))
looper.run(eventually(
checkStacksConnected, other_stacks, retryWait=1, timeout=connection_timeout))
def testValidatorSuspensionByTrustee(trustee, trusteeWallet, looper, nodeSet):
node = nodeSet[-1]
nodeNym = hexToFriendly(node.nodestack.verhex)
suspendNode(looper, trustee, trusteeWallet, nodeNym, node.name)
for n in nodeSet[:-1]:
looper.run(eventually(checkNodeNotInNodeReg, n, node.name))
looper.run(eventually(checkNodeNotInNodeReg, trustee, node.name))
def nymsAddedInQuickSuccession(nodeSet, addedTrustAnchor, looper,
trustAnchor, trustAnchorWallet):
usigner = DidSigner()
nym = usigner.verkey
idy = Identity(identifier=nym)
trustAnchorWallet.addTrustAnchoredIdentity(idy)
# Creating a NYM request with same nym again
req = idy.ledgerRequest()
trustAnchorWallet._pending.appendleft((req, idy.identifier))
reqs = trustAnchorWallet.preparePending()
trustAnchor.submitReqs(*reqs)
def check():
assert trustAnchorWallet._trustAnchored[nym].seqNo
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(check, timeout=timeout))
timeout = waits.expectedReqNAckQuorumTime()
looper.run(eventually(checkNacks,
trustAnchor,
req.reqId,
"is already added",
retryWait=1, timeout=timeout))
count = 0
for node in nodeSet:
for seq, txn in node.domainLedger.getAllTxn():
if txn[TXN_TYPE] == NYM and txn[TARGET_NYM] == usigner.identifier:
count += 1
assert(count == len(nodeSet))
def testPropagateRecvdBeforeRequest(setup, looper, txnPoolNodeSet, sent1):
A, B, C, D = txnPoolNodeSet
def x():
# A should not have received a request from the client
assert len(recvdRequest(A)) == 0
# A should have received only one PROPAGATE
assert len(recvdPropagate(A)) == 1
# A should have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delaySec - 2
looper.run(eventually(x, retryWait=.5, timeout=timeout))
def y():
# A should have received a request from the client
assert len(recvdRequest(A)) == 1
# A should still have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delaySec + 2
looper.run(eventually(y, retryWait=.5, timeout=timeout))
def chk():
# A should have forwarded the request
assertLength(forwardedRequest(A), 1)
timeout = waits.expectedClientRequestPropagationTime(
len(txnPoolNodeSet)) + delaySec
looper.run(eventually(chk, retryWait=1, timeout=timeout))
auth_obj = A.authNr(0).core_authenticator
auth_calling_count = get_count(auth_obj, auth_obj.authenticate)
assert auth_calling_count == reqCount
def testRescheduleUpgradeToLowerVersionThanPreviouslyScheduled(
looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet):
"""
A node starts at version 1.2 running has scheduled upgrade for version 1.5
but get a txn for upgrade 1.4, it will schedule it and cancel upgrade to 1.5.
"""
upgr1 = deepcopy(validUpgrade)
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(3)
# upgr2[SHA256] = get_valid_code_hash()
upgr2[SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c'
# An upgrade for higher version scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# An upgrade for lower version scheduled, the transaction should pass and
# the upgrade should be scheduled
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def testPrePrepareWithHighSeqNo(looper, txnPoolNodeSet, propagated1):
def chk():
for r in getNonPrimaryReplicas(txnPoolNodeSet, instId):
nodeSuspicions = len(getNodeSuspicions(
r.node, Suspicions.WRONG_PPSEQ_NO.code))
assert nodeSuspicions == 1
def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares):
assert (replica.prePrepares[viewNo, ppSeqNo][0]) == \
(req.identifier, req.reqId, req.digest)
primary = getPrimaryReplica(txnPoolNodeSet, instId)
nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, instId)
req = propagated1.reqDigest
primary.doPrePrepare(req)
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet))
for np in nonPrimaryReplicas:
looper.run(
eventually(checkPreprepare, np, primary.viewNo,
primary.lastPrePrepareSeqNo - 1, req, 1,
retryWait=.5, timeout=timeout))
newReqDigest = (req.identifier, req.reqId + 1, req.digest)
incorrectPrePrepareReq = PrePrepare(instId,
primary.viewNo,
primary.lastPrePrepareSeqNo + 2,
*newReqDigest,
get_utc_epoch())
primary.send(incorrectPrePrepareReq, TPCStat.PrePrepareSent)
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def testPropagateRecvdAfterRequest(setup, looper, txnPoolNodeSet):
A, B, C, D = txnPoolNodeSet # type: TestNode
sent1 = sdk_json_to_request_object(setup[0][0])
def x():
# A should have received a request from the client
assert len(recvdRequest(A)) == 1
# A should not have received a PROPAGATE
assert len(recvdPropagate(A)) == 0
# A should have sent a PROPAGATE
assert len(sentPropagate(A)) == 1
timeout = howlong - 2
looper.run(eventually(x, retryWait=.5, timeout=timeout))
for n in txnPoolNodeSet:
n.nodeIbStasher.resetDelays()
def y():
# A should have received 3 PROPAGATEs
assert len(recvdPropagate(A)) == 3
# A should have total of 4 PROPAGATEs (3 from other nodes and 1 from
# itself)
key = sent1.digest
assert key in A.requests
assert len(A.requests[key].propagates) == 4
# A should still have sent only one PROPAGATE
assert len(sentPropagate(A)) == 1
timeout = howlong + 2
looper.run(eventually(y, retryWait=.5, timeout=timeout))
auth_obj = A.authNr(0).core_authenticator
auth_calling_count = get_count(auth_obj, auth_obj.authenticate)
assert auth_calling_count == reqCount
def test_req_drop_on_propagate_phase_on_non_primary_and_then_ordered(
tconf, setup, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
global initial_ledger_size
A, B, C, D = txnPoolNodeSet # type: TestNode
sent1 = sdk_json_to_request_object(setup[0][0])
lagged_node = C
def check_propagates_and_3pc_delayed():
# Node should have received a request from the client
assert len(recvdRequest(lagged_node)) == 1
# Node should not have received a PROPAGATE
assert len(recvdPropagate(lagged_node)) == 0
# Node should have sent a PROPAGATE
assert len(sentPropagate(lagged_node)) == 1
# Node should have not received PrePrepares for master instance
assert len(recvdPrePrepareForInstId(lagged_node, 0)) == 0
# Node should have not received Prepares for master instance
assert len(recvdPrepareForInstId(lagged_node, 0)) == 0
# Node should have not received Commits for master instance
assert len(recvdCommitForInstId(lagged_node, 0)) == 0
# Node should have 1 request in requests queue
assert len(lagged_node.requests) == 1
timeout = howlong - 2
looper.run(eventually(check_propagates_and_3pc_delayed, retryWait=.5, timeout=timeout))
def check_drop():
assert len(lagged_node.requests) == 0
timeout = tconf.PROPAGATES_PHASE_REQ_TIMEOUT + tconf.OUTDATED_REQS_CHECK_INTERVAL + 1
looper.run(eventually(check_drop, retryWait=.5, timeout=timeout))
for n in txnPoolNodeSet:
n.nodeIbStasher.resetDelays()
def check_propagates_received():
# Node should have received 3 PROPAGATEs
assert len(recvdPropagate(lagged_node)) == 3
# Node should have total of 4 PROPAGATEs (3 from other nodes and 1 from
# itself)
key = sent1.digest
assert key in lagged_node.requests
assert len(lagged_node.requests[key].propagates) == 4
# Node should still have sent two PROPAGATEs since request
# was dropped and re-received over propagate
assert len(sentPropagate(lagged_node)) == 2
timeout = howlong + 2
looper.run(eventually(check_propagates_received, retryWait=.5, timeout=timeout))
def check_ledger_size():
# The request should be eventually ordered
for n in txnPoolNodeSet:
assert n.domainLedger.size - initial_ledger_size == 1
looper.run(eventually(check_ledger_size, retryWait=.5, timeout=timeout))
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def test_all_replicas_hold_request_keys(
perf_chk_patched,
looper,
txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle):
"""
All replicas whether primary or non primary hold request keys of forwarded
requests. Once requests are ordered, they request keys are removed from replica.
"""
tconf = perf_chk_patched
delay_3pc = 2
delay_3pc_messages(txnPoolNodeSet, 0, delay_3pc)
delay_3pc_messages(txnPoolNodeSet, 1, delay_3pc)
def chk(count):
# All replicas have same amount of forwarded request keys and all keys
# are finalised.
for node in txnPoolNodeSet:
for r in node.replicas.values():
if r.isPrimary is False:
assert len(r.requestQueues[DOMAIN_LEDGER_ID]) == count
for i in range(count):
k = r.requestQueues[DOMAIN_LEDGER_ID][i]
assert r.requests[k].finalised
elif r.isPrimary is True:
assert len(r.requestQueues[DOMAIN_LEDGER_ID]) == 0
reqs = sdk_signed_random_requests(looper,
sdk_wallet_client,
tconf.Max3PCBatchSize - 1)
req_resps = sdk_send_signed_requests(sdk_pool_handle, reqs)
# Only non primary replicas should have all request keys with them
looper.run(eventually(chk, tconf.Max3PCBatchSize - 1))
sdk_get_replies(looper, req_resps, timeout=sdk_eval_timeout(
tconf.Max3PCBatchSize - 1, len(txnPoolNodeSet),
add_delay_to_timeout=delay_3pc))
# Replicas should have no request keys with them since they are ordered
looper.run(eventually(chk, 0)) # Need to wait since one node might not
# have processed it.
delay = 1
for node in txnPoolNodeSet:
node.nodeIbStasher.delay(nom_delay(delay))
ensure_view_change(looper, txnPoolNodeSet)
reqs = sdk_signed_random_requests(looper,
sdk_wallet_client,
2 * tconf.Max3PCBatchSize)
req_resps = sdk_send_signed_requests(sdk_pool_handle, reqs)
looper.run(eventually(chk, 2 * tconf.Max3PCBatchSize))
# Since each nomination is delayed and there will be multiple nominations
# so adding some extra time
timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \
len(txnPoolNodeSet) * delay
ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=timeout)
sdk_get_replies(looper, req_resps, timeout=timeout)
looper.run(eventually(chk, 0))
def test_no_propagate_request_on_different_prepares_on_backup_before_vc(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
''' Send random request and do view change then fast_nodes (2,3 - with
primary backup replica) will have prepare or send preprepare on backup
replicas and slow_nodes are have not and transaction will ordered on all
master replicas. Check last ordered after view change and after another
one request.'''
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
slow_instance = 1
slow_nodes = txnPoolNodeSet[1:3]
fast_nodes = [n for n in txnPoolNodeSet if n not in slow_nodes]
nodes_stashers = [n.nodeIbStasher for n in slow_nodes]
old_last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(nodes_stashers, pDelay(instId=slow_instance)):
with delay_rules(nodes_stashers, ppDelay(instId=slow_instance)):
# send one request
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
old_view_no = txnPoolNodeSet[0].viewNo
looper.run(
eventually(is_prepared,
fast_nodes,
2,
slow_instance))
# trigger view change on all nodes
ensure_view_change(looper, txnPoolNodeSet)
# wait for view change done on all nodes
ensureElectionsDone(looper, txnPoolNodeSet)
primary = getPrimaryReplica(txnPoolNodeSet, slow_instance).node
non_primaries = [n for n in txnPoolNodeSet if n is not primary]
check_last_ordered(non_primaries,
slow_instance,
(old_view_no, old_last_ordered[1] + 1))
# Backup primary replica must not advance last_ordered_3pc
# up to the master's value
check_last_ordered([primary],
slow_instance,
(old_view_no, old_last_ordered[1]))
check_last_ordered(txnPoolNodeSet,
txnPoolNodeSet[0].master_replica.instId,
(old_last_ordered[0], old_last_ordered[1] + 1))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
looper.run(
eventually(check_last_ordered,
txnPoolNodeSet,
slow_instance,
(txnPoolNodeSet[0].viewNo, 1)))
assert all(0 == node.spylog.count(node.request_propagates)
for node in txnPoolNodeSet)
def test_claim_from_libindy_works(
aliceAgent,
aliceAcceptedFaber,
aliceAcceptedAcme,
acmeAgent,
emptyLooper,
faberAgent):
faberLink = aliceAgent.wallet.getConnection('Faber College')
name, version, origin = faberLink.availableClaims[0]
schemaKey = SchemaKey(name, version, origin)
timeout = waits.expectedClaimsReceived()
schema = faberAgent.issuer.wallet._schemasByKey[schemaKey]
async def create_claim_and_send_to_prover():
claimReq = await aliceAgent.prover.createClaimRequest(
schemaId=ID(schemaKey),
proverId='b1134a647eb818069c089e7694f63e6d',
reqNonRevoc=False)
assert claimReq
attr = faberAgent.issuer_backend.get_record_by_internal_id(1)
faberAgent.issuer._attrRepo.addAttributes(schemaKey=schemaKey,
userId=claimReq.userId,
attributes=attr)
claim_signature, claim_attributes = await faberAgent.issuer.issueClaim(ID(schemaKey=schemaKey), claimReq)
msg = get_claim_libindy_msg(claim_signature, schema.seqId)
await aliceAgent.handleReqClaimResponse(msg)
emptyLooper.run(eventually(
create_claim_and_send_to_prover, timeout=timeout))
# 2. check that claim is received from Faber
async def chkClaims():
claim = await aliceAgent.prover.wallet.getClaimSignature(ID(schemaKey))
assert claim.primaryClaim
emptyLooper.run(eventually(chkClaims, timeout=timeout))
# 3. send proof to Acme
acme_link, acme_proof_req = aliceAgent.wallet.getMatchingConnectionsWithProofReq(
"Job-Application", "Acme Corp")[0]
aliceAgent.sendProof(acme_link, acme_proof_req)
# 4. check that proof is verified by Acme
def chkProof():
internalId = acmeAgent.get_internal_id_by_nonce(
acme_link.request_nonce)
link = acmeAgent.wallet.getConnectionBy(internalId=internalId)
assert "Job-Application" in link.verifiedClaimProofs
timeout = waits.expectedClaimsReceived()
emptyLooper.run(eventually(chkProof, timeout=timeout))
def test_claim_request_from_libsovrin_works(aliceAgent, aliceAcceptedFaber, aliceAcceptedAcme,
acmeAgent, emptyLooper):
faberLink = aliceAgent.wallet.getLink('Faber College')
name, version, origin = faberLink.availableClaims[0]
schemaKey = SchemaKey(name, version, origin)
timeout = waits.expectedClaimsReceived()
async def create_claim_init_data_and_send_msg():
claimReq = await aliceAgent.prover.createClaimRequest(
schemaId=ID(schemaKey),
proverId='b1134a647eb818069c089e7694f63e6d',
reqNonRevoc=False)
assert claimReq
msg = (
{
'issuer_did': 'FuN98eH2eZybECWkofW6A9BKJxxnTatBCopfUiNxo6ZB',
'claim_def_seq_no': 14,
'blinded_ms': {
'prover_did': 'b1134a647eb818069c089e7694f63e6d',
'u': str(crypto_int_to_str(claimReq.U)),
'ur': None
},
'type': 'CLAIM_REQUEST',
'schema_seq_no': 13,
'nonce': 'b1134a647eb818069c089e7694f63e6d',
}
)
aliceAgent.signAndSendToLink(msg=msg, linkName=faberLink.name)
emptyLooper.run(eventually(create_claim_init_data_and_send_msg, timeout=timeout))
# 2. check that claim is received from Faber
async def chkClaims():
claim = await aliceAgent.prover.wallet.getClaims(ID(schemaKey))
assert claim.primaryClaim
emptyLooper.run(eventually(chkClaims, timeout=timeout))
# 3. send proof to Acme
acme_link, acme_proof_req = aliceAgent.wallet.getMatchingLinksWithProofReq(
"Job-Application", "Acme Corp")[0]
aliceAgent.sendProof(acme_link, acme_proof_req)
# 4. check that proof is verified by Acme
def chkProof():
internalId = acmeAgent.get_internal_id_by_nonce(acme_link.invitationNonce)
link = acmeAgent.wallet.getLinkBy(internalId=internalId)
assert "Job-Application" in link.verifiedClaimProofs
timeout = waits.expectedClaimsReceived()
emptyLooper.run(eventually(chkProof, timeout=timeout))
def test_view_change_done_delayed(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client):
"""
A node is slow so is behind other nodes, after view change, it catches up
but it also gets view change message as delayed, a node should start
participating only when caught up and ViewChangeCone quorum received.
"""
nprs = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)]
slow_node = nprs[-1]
other_nodes = [n for n in txnPoolNodeSet if n != slow_node]
delay_3pc = 10
delay_vcd = 25
delay_3pc_messages([slow_node], 0, delay_3pc)
slow_node.nodeIbStasher.delay(vcd_delay(delay_vcd))
def chk(node):
assert node.view_changer.has_acceptable_view_change_quorum
assert node.view_changer._primary_verified
assert node.isParticipating
assert None not in {r.isPrimary for r in node.replicas.values()}
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 5 * 4, 4)
ensure_view_change(looper, nodes=txnPoolNodeSet)
# After view change, the slow node successfully completes catchup
waitNodeDataEquality(looper, slow_node, *other_nodes)
# Other nodes complete view change, select primary and participate
for node in other_nodes:
looper.run(eventually(chk, node, retryWait=1))
# Since `ViewChangeCone` is delayed, slow_node is not able to select primary
# and participate
assert not slow_node.view_changer.has_acceptable_view_change_quorum
assert not slow_node.view_changer._primary_verified
assert not slow_node.isParticipating
assert {r.isPrimary for r in slow_node.replicas.values()} == {None}
# Send requests to make sure pool is functional
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5)
# Repair network
slow_node.reset_delays_and_process_delayeds()
# `slow_node` selects primary and participate
looper.run(eventually(chk, slow_node, retryWait=1))
# Processes requests received during lack of primary
waitNodeDataEquality(looper, slow_node, *other_nodes)
# Send more requests and compare data of all nodes
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def test_req_drop_on_commit_phase_on_master_primary_and_then_ordered(
tconf, setup, looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
global initial_ledger_size
A, B, C, D = txnPoolNodeSet # type: TestNode
lagged_node = A
def check_propagates():
# Node should have received a request from the client
assert len(recvdRequest(lagged_node)) == 1
# Node should have received a PROPAGATEs
assert len(recvdPropagate(lagged_node)) == 3
# Node should have sent a PROPAGATE
assert len(sentPropagate(lagged_node)) == 1
# Node should have one request in the requests queue
assert len(lagged_node.requests) == 1
timeout = howlong - 2
looper.run(eventually(check_propagates, retryWait=.5, timeout=timeout))
def check_prepares_received():
# Node should have received all Prepares for master instance
assert len(recvdPrepareForInstId(lagged_node, 0)) == 3
assert len(lagged_node.requests) == 1
looper.run(eventually(check_prepares_received, retryWait=.5, timeout=timeout))
def check_drop():
# Node should have not received Commits for master instance
assert len(recvdCommitForInstId(lagged_node, 0)) == 0
# Request object should be dropped by timeout
assert len(lagged_node.requests) == 0
timeout = tconf.ORDERING_PHASE_REQ_TIMEOUT + tconf.OUTDATED_REQS_CHECK_INTERVAL + 1
looper.run(eventually(check_drop, retryWait=.5, timeout=timeout))
for n in txnPoolNodeSet:
n.nodeIbStasher.resetDelays()
def check_commits_received():
# Node should have received all delayed Commits for master instance
assert len(recvdCommitForInstId(lagged_node, 0)) == 3
timeout = howlong * 2
looper.run(eventually(check_commits_received, retryWait=.5, timeout=timeout))
def check_ledger_size():
# The request should be eventually ordered
for n in txnPoolNodeSet:
assert n.domainLedger.size - initial_ledger_size == 1
looper.run(eventually(check_ledger_size, retryWait=.5, timeout=timeout))
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def checkIdentityRequestFailed(looper, client, req, cause):
timeout = waits.expectedReqRejectQuorumTime()
# TODO: Just for now, better to have a generic negative response checker
try:
looper.run(eventually(checkRejects,
client,
req.reqId,
cause, retryWait=1, timeout=timeout))
except AssertionError:
looper.run(eventually(checkNacks,
client,
req.reqId,
cause, retryWait=1, timeout=timeout))
def test_proof_from_libindy_works(
aliceAgent,
aliceAcceptedFaber,
aliceAcceptedAcme,
acmeAgent,
emptyLooper,
faberAgent):
# 1. request Claims from Faber
faberLink = aliceAgent.wallet.getConnection('Faber College')
name, version, origin = faberLink.availableClaims[0]
schemaKey = SchemaKey(name, version, origin)
aliceAgent.sendReqClaim(faberLink, schemaKey)
schema = faberAgent.issuer.wallet._schemasByKey[schemaKey]
# 2. check that claim is received from Faber
async def chkClaims():
claim = await aliceAgent.prover.wallet.getClaimSignature(ID(schemaKey))
assert claim.primaryClaim
timeout = waits.expectedClaimsReceived()
emptyLooper.run(eventually(chkClaims, timeout=timeout))
# 3. send proof to Acme
acme_link, acme_proof_req = aliceAgent.wallet.getMatchingConnectionsWithProofReq(
"Job-Application", "Acme Corp")[0]
async def create_proof():
proofRequest = ProofRequest("proof1",
"1.0",
int(acme_proof_req.nonce),
verifiableAttributes=acme_proof_req.verifiableAttributes,
predicates=acme_proof_req.predicates)
proof = await aliceAgent.prover.presentProof(proofRequest)
msg = get_proof_libindy_msg(
acme_link, acme_proof_req, proof, str(schema.seqId), schema.seqId)
aliceAgent.signAndSendToLink(msg=msg, linkName=acme_link.name)
emptyLooper.run(eventually(create_proof, timeout=timeout))
# 4. check that proof is verified by Acme
def chkProof():
internalId = acmeAgent.get_internal_id_by_nonce(
acme_link.request_nonce)
link = acmeAgent.wallet.getConnectionBy(internalId=internalId)
assert "Job-Application" in link.verifiedClaimProofs
emptyLooper.run(eventually(chkProof, timeout=timeout))
def testOrderingWhenPrePrepareNotReceived(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle):
"""
Send commits but delay pre-prepare and prepares such that enough
commits are received, now the request should not be ordered until
pre-prepare is received and ordering should just happen once,
"""
delay = 10
non_prim_reps = getNonPrimaryReplicas(txnPoolNodeSet, 0)
slow_rep = non_prim_reps[0]
slow_node = slow_rep.node
slow_node.nodeIbStasher.delay(ppDelay(delay, 0))
slow_node.nodeIbStasher.delay(pDelay(delay, 0))
stash_pp = []
stash_p = []
orig_pp_method = slow_rep.processPrePrepare
orig_p_method = slow_rep.processPrepare
def patched_pp(self, msg, sender):
stash_pp.append((msg, sender))
def patched_p(self, msg, sender):
stash_p.append((msg, sender))
slow_rep.processPrePrepare = \
types.MethodType(patched_pp, slow_rep)
slow_rep.processPrepare = \
types.MethodType(patched_p, slow_rep)
def chk1():
assert len(slow_rep.commitsWaitingForPrepare) > 0
sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + delay
looper.run(eventually(chk1, retryWait=1, timeout=timeout))
for m, s in stash_pp:
orig_pp_method(m, s)
for m, s in stash_p:
orig_p_method(m, s)
def chk2():
assert len(slow_rep.commitsWaitingForPrepare) == 0
assert slow_rep.spylog.count(slow_rep.doOrder.__name__) == 1
timeout = waits.expectedOrderingTime(len(non_prim_reps) + 1) + 2 * delay
looper.run(eventually(chk2, retryWait=1, timeout=timeout))
def test_claim_request_from_libindy_works(
aliceAgent,
aliceAcceptedFaber,
aliceAcceptedAcme,
acmeAgent,
emptyLooper,
faberAgent):
faberLink = aliceAgent.wallet.getConnection('Faber College')
name, version, origin = faberLink.availableClaims[0]
schemaKey = SchemaKey(name, version, origin)
timeout = waits.expectedClaimsReceived()
schema = faberAgent.issuer.wallet._schemasByKey[schemaKey]
async def create_claim_init_data_and_send_msg():
claimReq = await aliceAgent.prover.createClaimRequest(
schemaId=ID(schemaKey),
proverId='b1134a647eb818069c089e7694f63e6d',
reqNonRevoc=False)
assert claimReq
msg = get_claim_request_libindy_msg(claimReq, schema.seqId)
aliceAgent.signAndSendToLink(msg=msg, linkName=faberLink.name)
emptyLooper.run(eventually(
create_claim_init_data_and_send_msg, timeout=timeout))
# 2. check that claim is received from Faber
async def chkClaims():
claim = await aliceAgent.prover.wallet.getClaimSignature(ID(schemaKey))
assert claim.primaryClaim
emptyLooper.run(eventually(chkClaims, timeout=timeout))
# 3. send proof to Acme
acme_link, acme_proof_req = aliceAgent.wallet.getMatchingConnectionsWithProofReq(
"Job-Application", "Acme Corp")[0]
aliceAgent.sendProof(acme_link, acme_proof_req)
# 4. check that proof is verified by Acme
def chkProof():
internalId = acmeAgent.get_internal_id_by_nonce(
acme_link.request_nonce)
link = acmeAgent.wallet.getConnectionBy(internalId=internalId)
assert "Job-Application" in link.verifiedClaimProofs
timeout = waits.expectedClaimsReceived()
emptyLooper.run(eventually(chkProof, timeout=timeout))
def some_requests(txn_pool_node_set_post_creation, looper,
sdk_wallet_steward, sdk_pool_handle):
def check_auctions_amount(expected_amount):
assert auctions['pqr'][did] == expected_amount
old_bls_store_size = None
for node in txn_pool_node_set_post_creation:
if old_bls_store_size is None:
old_bls_store_size = node.bls_bft.bls_store._kvs.size
else:
assert node.bls_bft.bls_store._kvs.size == old_bls_store_size
op = {
TXN_TYPE: AUCTION_START,
DATA: {'id': 'pqr'}
}
successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle)
op = {
TXN_TYPE: PLACE_BID,
DATA: {'id': 'pqr', AMOUNT: 20}
}
successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle)
_, did = sdk_wallet_steward
for node in txn_pool_node_set_post_creation:
auctions = node.get_req_handler(AUCTION_LEDGER_ID).auctions
assert 'pqr' in auctions
looper.run(eventually(check_auctions_amount, 20))
op = {
TXN_TYPE: PLACE_BID,
DATA: {'id': 'pqr', AMOUNT: 40}
}
successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle)
for node in txn_pool_node_set_post_creation:
auctions = node.get_req_handler(AUCTION_LEDGER_ID).auctions
assert 'pqr' in auctions
looper.run(eventually(check_auctions_amount, 40))
op = {
TXN_TYPE: AUCTION_END,
DATA: {'id': 'pqr'}
}
successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle)
for node in txn_pool_node_set_post_creation:
# Not all batches might have BLS-sig but at least one of them will have
assert node.bls_bft.bls_store._kvs.size > old_bls_store_size
def testKitZStacksCommunication(registry, tdir, looper):
# TODO: Use a packet capture tool
genKeys(tdir, registry.keys())
stacks = []
names = []
for name, ha in registry.items():
printer = Printer(name)
stackParams = dict(name=name, ha=ha, basedirpath=tdir,
auth_mode=AuthMode.RESTRICTED.value)
reg = copy(registry)
reg.pop(name)
stack = KITZStack(stackParams, printer.print, reg)
stacks.append(stack)
names.append((name, printer))
prepStacks(looper, *stacks, connect=False, useKeys=True)
# TODO: the connection may not be established for the first try because
# some of the stacks may not have had a remote yet (that is they haven't had yet called connect)
timeout = 4 * KITZStack.RETRY_TIMEOUT_RESTRICTED + 1
looper.run(eventually(
checkStacksConnected, stacks, retryWait=1, timeout=timeout))
def send_recv():
for i, j in combinations(range(len(stacks)), 2):
alpha = stacks[i]
beta = stacks[j]
alpha.send({'greetings': 'hi'}, beta.name)
beta.send({'greetings': 'hello'}, alpha.name)
looper.run(
eventually(chkPrinted, names[i][1], {'greetings': 'hello'}, timeout=15))
looper.run(eventually(chkPrinted, names[j][1], {
'greetings': 'hi'}, timeout=15))
names[i][1].reset()
names[j][1].reset()
def pr():
for stack in stacks:
for r in stack.remotes.values():
print(r._lastSocketEvents())
for _ in range(100):
# send_recv()
looper.run(eventually(
checkStacksConnected, stacks, retryWait=1, timeout=timeout))
# pr()
looper.runFor(30)
def upgradeScheduledExpForceTrue(validUpgradeSentExpForceTrue, looper, nodeSet,
validUpgradeExpForceTrue):
looper.run(eventually(checkUpgradeScheduled, nodeSet,
validUpgradeExpForceTrue[VERSION], retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_backup_primary_restores_pp_seq_no_if_view_is_same(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf,
tdir, allPluginsPath, chkFreqPatched, view_no):
# Get a node with a backup primary replica
replica = getPrimaryReplica(txnPoolNodeSet, instId=backup_inst_id)
node = replica.node
# Send some 3PC-batches and wait until the replica orders the 3PC-batches
sdk_send_batches_of_random(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=7,
num_batches=7,
timeout=tconf.Max3PCBatchWait)
seq_no = 7 if view_no == 0 else 8
looper.run(
eventually(lambda: assertExp(replica.last_ordered_3pc ==
(view_no, seq_no)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
# Check view no of the node and lastPrePrepareSeqNo of the replica
assert node.viewNo == view_no
assert replica.lastPrePrepareSeqNo == seq_no
# Ensure that the node has stored the last sent PrePrepare key
assert LAST_SENT_PRE_PREPARE in node.nodeStatusDB
last_sent_pre_prepare_key = \
node_status_db_serializer.deserialize(
node.nodeStatusDB.get(LAST_SENT_PRE_PREPARE))
assert last_sent_pre_prepare_key == {
str(backup_inst_id): [view_no, seq_no]
}
# Restart the node containing the replica
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
node.name,
stopNode=True)
looper.removeProdable(node)
txnPoolNodeSet.remove(node)
node = start_stopped_node(node, looper, tconf, tdir, allPluginsPath)
txnPoolNodeSet.append(node)
looper.run(checkNodesConnected(txnPoolNodeSet))
ensureElectionsDone(looper, txnPoolNodeSet)
replica = node.replicas[backup_inst_id]
# Verify that after the successful propagate primary procedure the replica
# (which must still be the primary in its instance) has restored
# lastPrePrepareSeqNo and adjusted last_ordered_3pc and shifted
# the watermarks correspondingly
assert node.viewNo == view_no
assert replica.isPrimary
assert replica.lastPrePrepareSeqNo == seq_no
assert replica.last_ordered_3pc == (view_no, seq_no)
assert replica.h == seq_no
assert replica.H == seq_no + LOG_SIZE
# Verify also that the stored last sent PrePrepare key has not been erased
assert LAST_SENT_PRE_PREPARE in node.nodeStatusDB
# Send a 3PC-batch and ensure that the replica orders it
sdk_send_batches_of_random(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=1,
num_batches=1,
timeout=tconf.Max3PCBatchWait)
seq_no = 8 if view_no == 0 else 9
looper.run(
eventually(lambda: assertExp(replica.last_ordered_3pc ==
(view_no, seq_no)),
retryWait=1,
timeout=waits.expectedTransactionExecutionTime(nodeCount)))
def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Replicas should not accept PRE-PREPARE for view "v" and prepare sequence
number "n" if it has already accepted a request with view number "v" and
sequence number "n"
"""
numOfNodes = 4
fValue = getMaxFailures(numOfNodes)
primaryRepl = getPrimaryReplica(txnPoolNodeSet, 1)
logger.debug("Primary Replica: {}".format(primaryRepl))
nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, 1)
logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas))
# Delay COMMITs so request is not ordered and checks can be made
c_delay = 10
for node in txnPoolNodeSet:
node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1))
req1 = sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)[0]
request1 = sdk_json_to_request_object(req1[0])
for npr in nonPrimaryReplicas:
looper.run(eventually(checkPrepareReqSent,
npr,
request1.key,
primaryRepl.viewNo,
retryWait=1))
prePrepareReq = primaryRepl._ordering_service.sent_preprepares[primaryRepl.viewNo,
primaryRepl.lastPrePrepareSeqNo]
looper.run(eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1))
# logger.debug("Patching the primary replica's pre-prepare sending method ")
# orig_method = primaryRepl.sendPrePrepare
# def patched(self, ppReq):
# self._ordering_service.sent_preprepares[ppReq.viewNo, ppReq.ppSeqNo] = ppReq
# ppReq = updateNamedTuple(ppReq, **{f.PP_SEQ_NO.nm: 1})
# self.send(ppReq, TPCStat.PrePrepareSent)
#
# primaryRepl.sendPrePrepare = types.MethodType(patched, primaryRepl)
logger.debug(
"Decrementing the primary replica's pre-prepare sequence number by "
"one...")
primaryRepl._ordering_service._lastPrePrepareSeqNo -= 1
view_no = primaryRepl.viewNo
request2 = sdk_json_to_request_object(
sdk_send_random_requests(looper,
sdk_pool_handle,
sdk_wallet_client,
1)[0][0])
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet))
looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2,
retryWait=1, timeout=timeout))
# Since the node is malicious, it will not be able to process requests due
# to conflicts in PRE-PREPARE
primaryRepl.node.stop()
looper.removeProdable(primaryRepl.node)
reqIdr = [request2.digest]
tm = get_utc_epoch()
prePrepareReq = PrePrepare(
primaryRepl.instId,
view_no,
primaryRepl.lastPrePrepareSeqNo,
tm,
reqIdr,
init_discarded(),
primaryRepl._ordering_service.generate_pp_digest([request2.digest], view_no, tm),
DOMAIN_LEDGER_ID,
primaryRepl._ordering_service.get_state_root_hash(DOMAIN_LEDGER_ID),
primaryRepl._ordering_service.get_txn_root_hash(DOMAIN_LEDGER_ID),
0,
True
)
logger.debug("""Checking whether all the non primary replicas have received
the pre-prepare request with same sequence number""")
timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet))
looper.run(eventually(checkPrePrepareReqRecvd,
nonPrimaryReplicas,
prePrepareReq,
retryWait=1,
timeout=timeout))
logger.debug("""Check that none of the non primary replicas didn't send
any prepare message "
in response to the pre-prepare message""")
timeout = waits.expectedPrepareTime(len(txnPoolNodeSet))
looper.runFor(timeout) # expect prepare processing timeout
# check if prepares have not been sent
for npr in nonPrimaryReplicas:
with pytest.raises(AssertionError):
looper.run(eventually(checkPrepareReqSent,
npr,
request2.key,
view_no,
retryWait=1,
timeout=timeout))
timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + c_delay
result1 = sdk_get_replies(looper, [req1])[0][1]
logger.debug("request {} gives result {}".format(request1, result1))
def test_node_requests_missing_preprepares_and_prepares_after_long_disconnection(
looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tconf,
tdirWithPoolTxns, allPluginsPath):
"""
2 of 4 nodes go down, so pool can not process any more incoming requests.
A new request comes in.
Test than waits for some time to ensure that PrePrepare was created
long enough seconds to be dropped by time checker.
Two stopped nodes come back alive.
Another request comes in.
Check that previously disconnected two nodes request missing PREPREPARES
and PREPARES and the pool successfully handles both transactions.
"""
INIT_REQS_CNT = 5
MISSING_REQS_CNT = 4
REQS_AFTER_RECONNECT_CNT = 1
alive_nodes = []
disconnected_nodes = []
for node in txnPoolNodeSet:
if node.hasPrimary:
alive_nodes.append(node)
else:
disconnected_nodes.append(node)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, INIT_REQS_CNT)
waitNodeDataEquality(looper, disconnected_nodes[0], *txnPoolNodeSet)
init_ledger_size = txnPoolNodeSet[0].domainLedger.size
current_node_set = set(txnPoolNodeSet)
for node in disconnected_nodes:
disconnect_node_and_ensure_disconnected(looper,
current_node_set,
node,
stopNode=False)
current_node_set.remove(node)
sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client,
MISSING_REQS_CNT)
looper.run(
eventually(check_pp_out_of_sync,
alive_nodes,
disconnected_nodes,
retryWait=1,
timeout=expectedPoolGetReadyTimeout(len(txnPoolNodeSet))))
preprepare_deviation = 4
tconf.ACCEPTABLE_DEVIATION_PREPREPARE_SECS = preprepare_deviation
time.sleep(preprepare_deviation * 2)
for node in disconnected_nodes:
current_node_set.add(node)
reconnect_node_and_ensure_connected(looper, current_node_set, node)
for node in txnPoolNodeSet:
assert node.domainLedger.size == init_ledger_size
for node in disconnected_nodes:
assert node.master_replica.spylog.count(
Replica._request_pre_prepare) == 0
assert node.master_replica.spylog.count(Replica._request_prepare) == 0
assert node.master_replica.spylog.count(
Replica.process_requested_pre_prepare) == 0
assert node.master_replica.spylog.count(
Replica.process_requested_prepare) == 0
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, REQS_AFTER_RECONNECT_CNT)
waitNodeDataEquality(looper, disconnected_nodes[0], *txnPoolNodeSet)
for node in disconnected_nodes:
assert node.master_replica.spylog.count(
Replica._request_pre_prepare) > 0
assert node.master_replica.spylog.count(Replica._request_prepare) > 0
assert node.master_replica.spylog.count(
Replica.process_requested_pre_prepare) > 0
assert node.master_replica.spylog.count(
Replica.process_requested_prepare) > 0
for node in txnPoolNodeSet:
assert node.domainLedger.size == (init_ledger_size + MISSING_REQS_CNT +
REQS_AFTER_RECONNECT_CNT)
def poolUpgradeScheduled(poolUpgradeSubmitted, poolNodesStarted, validUpgrade):
nodes = poolNodesStarted.nodes.values()
timeout = waits.expectedUpgradeScheduled()
poolNodesStarted.looper.run(
eventually(checkUpgradeScheduled, nodes,
validUpgrade[VERSION], retryWait=1, timeout=timeout))
def testPoolUpgradeCancelled(poolUpgradeCancelled, poolNodesStarted):
nodes = poolNodesStarted.nodes.values()
timeout = waits.expectedNoUpgradeScheduled()
poolNodesStarted.looper.run(
eventually(checkNoUpgradeScheduled,
nodes, retryWait=1, timeout=timeout))
def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func):
nodeNames = genNodeNames(4)
nodeReg = genNodeReg(names=nodeNames)
with TestNodeSet(nodeReg=nodeReg, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
prepareNodeSet(looper, nodeSet)
nodeA, nodeB, nodeC, nodeD = tuple(
addNodeBack(
nodeSet, looper, nodeNames[i]) for i in range(
0, 4))
# Since primary selection is round robin, A and B will be primaries
# Nodes C and D delays self nomination so A and B can become
# primaries
# nodeC.delaySelfNomination(10)
# nodeD.delaySelfNomination(10)
# Node D delays receiving PRIMARY messages from all nodes so it
# will not know whether it is primary or not
# delayD = 5
# nodeD.nodeIbStasher.delay(delayerMsgTuple(delayD, Primary))
checkPoolReady(looper=looper, nodes=nodeSet)
# client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func)
# request = sendRandomRequest(wal, client1)
# TODO Rethink this
instNo = 0
timeout = waits.expectedClientRequestPropagationTime(len(nodeSet))
for i in range(3):
node = nodeSet.getNode(nodeNames[i])
# Nodes A, B and C should have received PROPAGATE request
# from Node D
looper.run(
eventually(checkIfPropagateRecvdFromNode, node, nodeD,
request.identifier,
request.reqId, retryWait=1, timeout=timeout))
def assert_msg_count(typ, count):
assert len(getPendingRequestsForReplica(nodeD.replicas[instNo],
typ)) == count
# Node D should have 1 pending PRE-PREPARE request
timeout = waits.expectedPrePrepareTime(len(nodeSet))
looper.run(eventually(assert_msg_count, PrePrepare, 1,
retryWait=1, timeout=timeout))
# Node D should have 2 pending PREPARE requests(from node B and C)
timeout = waits.expectedPrepareTime(len(nodeSet))
looper.run(eventually(assert_msg_count, Prepare, 2, retryWait=1,
timeout=timeout))
# Its been checked above that replica stashes 3 phase messages in
# lack of primary, now avoid delay (fix the network)
nodeD.nodeIbStasher.reset_delays_and_process_delayeds()
# Node D should have no pending PRE-PREPARE, PREPARE or COMMIT
# requests
for reqType in [PrePrepare, Prepare, Commit]:
looper.run(
eventually(
lambda: assertLength(
getPendingRequestsForReplica(
nodeD.replicas[instNo],
reqType),
0),
retryWait=1,
timeout=delayD)) # wait little more than delay
def ensureConnectedToTestEnv(cli):
if not cli.activeEnv:
cli.enterCmd("connect test")
cli.looper.run(
eventually(checkConnectedToEnv, cli, retryWait=1, timeout=10))
def testPrimaryElectionWithTie(electTieFixture, looper, txnPoolNodeSet):
"""
Primary selection (Rainy Day)
A, B, C, D, E
A, B, C, D startup. E is lagging.
A sees the minimum number of nodes, and then sends Nominate(A)
At the same exact time, B sees the minimum number of nodes, and then sends out Nominate(B)
A sees B sending Nominate(B), but it has already nominated itself, so it does nothing
B sees A sending Nominate(A), but it has already nominated itself, so it does nothing
C sees A sending Nominate(A), and sends Nominate(A)
D sees B sending Nominate(B), and sends Nominate(B)
There's a split. C and A think A is the primary, B and D think B is the primary
All nodes can see that there is a split. Each sends out Reelection([A,B])
A and B both see Reelection([A,B]) from themselves as well as the other 3 (the number from others should be at least f+1),
1. they wait a random amount of time (between 0 and 2 seconds),
2. they each send out a Nominate(self)
Voting is repeated until we have a good election.
"""
# TODO optimize the sending messages in batches, for example, we don't
# send messages more often than 400 milliseconds. Once those 400
# millis have passed, we send the several queued messages in one
# batch.
A, B, C, D = txnPoolNodeSet
checkPoolReady(looper, txnPoolNodeSet)
for node in txnPoolNodeSet:
for instId, replica in enumerate(node.elector.replicas):
logger.debug("replica {} {} with votes {}".
format(replica.name, replica.instId,
node.elector.nominations.get(instId, {})))
nominationTimeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet))
logger.debug("Check nomination")
# Checking whether Node A nominated itself
looper.run(eventually(checkNomination, A, A.name,
retryWait=1, timeout=nominationTimeout))
# Checking whether Node B nominated itself
looper.run(eventually(checkNomination, B, B.name,
retryWait=1, timeout=nominationTimeout))
# Checking whether Node C nominated Node A
looper.run(eventually(checkNomination, C, A.name,
retryWait=1, timeout=nominationTimeout))
# Checking whether Node D nominated Node D
looper.run(eventually(checkNomination, D, B.name,
retryWait=1, timeout=nominationTimeout))
# No node should be primary
for node in txnPoolNodeSet:
assert node.hasPrimary is False
for node in txnPoolNodeSet:
node.resetDelays()
checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1)
def test_catchup_not_triggered_if_another_in_progress(
looper, chkFreqPatched, reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, broken_node_and_others):
"""
A node misses 3pc messages and checkpoints during some period but later it
stashes some amount of checkpoints and starts catchup. When the node is
performing the catchup, it receives more checkpoints enough to start a new
catchup but it does not start it because the first catchup is in progress.
"""
max_batch_size = chkFreqPatched.Max3PCBatchSize
broken_node, other_nodes = broken_node_and_others
logger.info(
"Step 1: The node misses quite a lot of 3PC-messages and checkpoints")
send_reqs_batches_and_get_suff_replies(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
reqs_for_checkpoint + max_batch_size)
waitNodeDataInequality(looper, broken_node, *other_nodes)
logger.info(
"Step 2: The node receives 3PC-messages but cannot process them because of "
"missed ones. But the node eventually stashes some amount of checkpoints "
"and after that starts catchup")
repaired_node = repair_broken_node(broken_node)
initial_do_start_catchup_times = repaired_node.spylog.count(
Node._do_start_catchup)
initial_all_ledgers_caught_up = repaired_node.spylog.count(
Node.allLedgersCaughtUp)
with delay_rules(repaired_node.nodeIbStasher, cr_delay()):
send_reqs_batches_and_get_suff_replies(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
(Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) *
reqs_for_checkpoint - max_batch_size)
ensure_all_nodes_have_same_data(looper, other_nodes)
looper.run(
eventually(
lambda: assertExp(repaired_node.mode == Mode.discovering),
timeout=waits.expectedPoolInterconnectionTime(
len(txnPoolNodeSet)) +
waits.expectedPoolConsistencyProof(len(txnPoolNodeSet))))
assert repaired_node.spylog.count(
Node._do_start_catchup) - initial_do_start_catchup_times == 1
logger.info(
"Step 3: While doing the catchup, the node receives new checkpoints "
"enough to start a new catchup but the node does not start it because "
"the former is in progress")
process_checkpoint_times_before = \
repaired_node.master_replica._checkpointer.spylog.count(CheckpointService.process_checkpoint)
send_reqs_batches_and_get_suff_replies(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client,
(Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) *
reqs_for_checkpoint)
# Wait until the node receives the new checkpoints from all the other nodes
looper.run(
eventually(lambda: assertExp(
repaired_node.master_replica._checkpointer.spylog.count(
CheckpointService.process_checkpoint
) - process_checkpoint_times_before ==
(Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1) *
(len(txnPoolNodeSet) - 1)),
timeout=waits.expectedPoolInterconnectionTime(
len(txnPoolNodeSet))))
# New catchup is not started when another one is in progress
assert repaired_node.spylog.count(
Node._do_start_catchup) - initial_do_start_catchup_times == 1
assert repaired_node.mode == Mode.discovering
logger.info("Step 4: The node completes the catchup. The ledger has been "
"updated to the level determined on its start")
looper.run(
eventually(lambda: assertExp(repaired_node.mode == Mode.participating),
timeout=waits.expectedPoolCatchupTime(len(txnPoolNodeSet))))
assert repaired_node.spylog.count(
Node._do_start_catchup) - initial_do_start_catchup_times == 1
assert repaired_node.spylog.count(
Node.allLedgersCaughtUp) - initial_all_ledgers_caught_up == 1
assert repaired_node.domainLedger.size == other_nodes[0].domainLedger.size
def _expect(expected,
mapper=None,
line_no=None,
within=None,
ignore_extra_lines=None):
cur_cli = current_cli()
def _():
expected_ = expected if not mapper \
else [s.format(**mapper) for s in expected]
assert isinstance(expected_, List)
bm = get_bookmark()
actual = ''.join(cur_cli.cli.output.writes).splitlines()[bm:]
assert isinstance(actual, List)
explanation = ''
expected_index = 0
for i in range(min(len(expected_), len(actual))):
e = expected_[expected_index]
assert isinstance(e, str)
a = actual[i]
assert isinstance(a, str)
is_p = type(e) == P
if (not is_p and a != e) or (is_p and not e.match(a)):
if ignore_extra_lines:
continue
explanation += "line {} doesn't match\n"\
" expected: {}\n"\
" actual: {}\n".format(i, e, a)
expected_index += 1
if len(expected_) > len(actual):
for e in expected_:
try:
p = re.compile(e) if type(e) == P else None
except Exception as err:
explanation += "ERROR COMPILING REGEX for {}: {}\n".\
format(e, err)
for a in actual:
if (p and p.fullmatch(a)) or a == e:
break
else:
explanation += "missing: {}\n".format(e)
if len(expected_) < len(actual) and ignore_extra_lines is None:
for a in actual:
for e in expected_:
p = re.compile(e) if type(e) == P else None
if (p and p.fullmatch(a)) or a == e:
break
else:
explanation += "extra: {}\n".format(a)
if explanation:
explanation += "\nexpected:\n"
for x in expected_:
explanation += " > {}\n".format(x)
explanation += "\nactual:\n"
for x in actual:
explanation += " > {}\n".format(x)
if line_no:
explanation += "section ends line number: {}\n".format(
line_no)
pytest.fail(''.join(explanation))
else:
inc_bookmark(len(actual))
if within:
cur_cli.looper.run(eventually(_, timeout=within))
else:
_()
def testReplyMatchesRequest(looper, nodeSet, tdir, up):
'''
This tests does check following things:
- wallet works correctly when used by multiple clients
- clients do receive responses for exactly the same request they sent
'''
def makeClient(id):
client, wallet = genTestClient(nodeSet,
tmpdir=tdir,
name="client-{}".format(id))
looper.add(client)
looper.run(client.ensureConnectedToNodes())
return client, wallet
# creating clients
numOfClients = 3
numOfRequests = 1
clients = set()
sharedWallet = None
for i in range(numOfClients):
client, wallet = makeClient(i)
if sharedWallet is None:
sharedWallet = wallet
clients.add(client)
for i in range(1, numOfRequests + 1):
# sending requests
requests = {}
for client in clients:
op = randomOperation()
req = sharedWallet.signOp(op)
request = client.submitReqs(req)[0][0]
requests[client] = (request.reqId, request.operation['amount'])
# checking results
responseTimeout = waits.expectedTransactionExecutionTime(nodeCount)
for client, (reqId, sentAmount) in requests.items():
looper.run(eventually(checkResponseRecvdFromNodes,
client,
nodeCount,
reqId,
retryWait=1,
timeout=responseTimeout))
print("Expected amount for request {} is {}".
format(reqId, sentAmount))
# This looks like it fails on some python versions
# replies = [r[0]['result']['amount']
# for r in client.inBox
# if r[0]['op'] == 'REPLY'
# and r[0]['result']['reqId'] == reqId]
replies = []
for r in client.inBox:
if r[0]['op'] == 'REPLY' and r[0]['result']['reqId'] == reqId:
if 'amount' not in r[0]['result']:
logger.debug('{} cannot find amount in {}'.
format(client, r[0]['result']))
replies.append(r[0]['result']['amount'])
assert all(replies[0] == r for r in replies)
assert replies[0] == sentAmount
def test_node_catchup_after_restart_with_txns(
newNodeCaughtUp,
txnPoolNodeSet,
tconf,
nodeSetWithNodeAddedAfterSomeTxns,
tdirWithPoolTxns,
allPluginsPath):
"""
A node that restarts after some transactions should eventually get the
transactions which happened while it was down
:return:
"""
looper, newNode, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
logger.debug("Stopping node {} with pool ledger size {}".
format(newNode, newNode.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, newNode)
looper.removeProdable(newNode)
# for n in txnPoolNodeSet[:4]:
# for r in n.nodestack.remotes.values():
# if r.name == newNode.name:
# r.removeStaleCorrespondents()
# looper.run(eventually(checkNodeDisconnectedFrom, newNode.name,
# txnPoolNodeSet[:4], retryWait=1, timeout=5))
# TODO: Check if the node has really stopped processing requests?
logger.debug("Sending requests")
more_requests = 5
sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, more_requests)
logger.debug("Starting the stopped node, {}".format(newNode))
nodeHa, nodeCHa = HA(*newNode.nodestack.ha), HA(*newNode.clientstack.ha)
newNode = TestNode(
newNode.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
# Delay catchup reply processing so LedgerState does not change
delay_catchup_reply = 5
newNode.nodeIbStasher.delay(cr_delay(delay_catchup_reply))
looper.run(checkNodesConnected(txnPoolNodeSet))
# Make sure ledger starts syncing (sufficient consistency proofs received)
looper.run(eventually(check_ledger_state, newNode, DOMAIN_LEDGER_ID,
LedgerState.syncing, retryWait=.5, timeout=5))
confused_node = txnPoolNodeSet[0]
new_node_ledger = newNode.ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID]
cp = new_node_ledger.catchUpTill
start, end = cp.seqNoStart, cp.seqNoEnd
cons_proof = confused_node.ledgerManager._buildConsistencyProof(
DOMAIN_LEDGER_ID, start, end)
bad_send_time = None
def chk():
nonlocal bad_send_time
entries = newNode.ledgerManager.spylog.getAll(
newNode.ledgerManager.canProcessConsistencyProof.__name__)
for entry in entries:
# `canProcessConsistencyProof` should return False after `syncing_time`
if entry.result == False and entry.starttime > bad_send_time:
return
assert False
def send_and_chk(ledger_state):
nonlocal bad_send_time, cons_proof
bad_send_time = perf_counter()
confused_node.ledgerManager.sendTo(cons_proof, newNode.name)
# Check that the ConsistencyProof messages rejected
looper.run(eventually(chk, retryWait=.5, timeout=5))
check_ledger_state(newNode, DOMAIN_LEDGER_ID, ledger_state)
send_and_chk(LedgerState.syncing)
# Not accurate timeout but a conservative one
timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \
2 * delay_catchup_reply
waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1],
customTimeout=timeout)
assert new_node_ledger.num_txns_caught_up == more_requests
send_and_chk(LedgerState.synced)
def test_discard_3PC_messages_for_already_ordered(looper, txnPoolNodeSet,
sdk_wallet_client,
sdk_pool_handle):
"""
Nodes discard any 3PC messages for already ordered 3PC keys
(view_no, pp_seq_no). Delay all 3PC messages to a node so it cannot respond
to them unless the other nodes order them, now when the slow node will get
them it will respond but other nodes will not process them and discard them
"""
slow_node = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)][-1]
other_nodes = [n for n in txnPoolNodeSet if n != slow_node]
delay = 20
delay_3pc_messages([slow_node], 0, delay)
delay_3pc_messages([slow_node], 1, delay)
sent_batches = 3
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=2 * sent_batches,
num_batches=sent_batches)
# send_reqs_batches_and_get_suff_replies(looper, wallet1, client1,
# 2 * sent_batches, sent_batches)
def chk(node, inst_id, p_count, c_count):
# A node will still record PREPRAREs even if more than n-f-1, till the
# request is not ordered
assert len(
node.replicas[inst_id]._ordering_service.prepares) >= p_count
assert len(node.replicas[inst_id]._ordering_service.commits) == c_count
def count_discarded(inst_id, count):
for node in other_nodes:
assert countDiscarded(node.replicas[inst_id].stasher,
'already ordered 3 phase message') == count
# `slow_node` did not receive any PREPAREs or COMMITs
chk(slow_node, 0, 0, 0)
# `other_nodes` have not discarded any 3PC message
count_discarded(0, 0)
# `other_nodes` have not recorded any PREPAREs or COMMITs from `slow_node`
chk_commits_prepares_recvd(0, other_nodes, slow_node)
slow_node.reset_delays_and_process_delayeds()
waitNodeDataEquality(looper, slow_node, *other_nodes)
# `slow_node` did receive correct number of PREPAREs and COMMITs
looper.run(
eventually(chk,
slow_node,
0,
sent_batches - 1,
sent_batches,
retryWait=1))
# `other_nodes` have not recorded any PREPAREs or COMMITs from `slow_node`
chk_commits_prepares_recvd(0, other_nodes, slow_node)
# `other_nodes` have discarded PREPAREs and COMMITs all batches
count_discarded(0, 2 * sent_batches)
def test_audit_ledger_updated_after_freshness_updated(looper, tconf,
txnPoolNodeSet,
initial_domain_size,
initial_pool_size,
initial_config_size):
# 1. Wait for the first freshness update
looper.run(
eventually(check_freshness_updated_for_all,
txnPoolNodeSet,
timeout=2 * FRESHNESS_TIMEOUT))
audit_size_initial = [node.auditLedger.size for node in txnPoolNodeSet]
view_no = txnPoolNodeSet[0].master_replica.last_ordered_3pc[0]
pp_seq_no = txnPoolNodeSet[0].master_replica.last_ordered_3pc[1]
initial_seq_no = txnPoolNodeSet[0].auditLedger.size
# 2. Wait for the second freshness update
bls_multi_sigs_after_first_update = get_all_multi_sig_values_for_all_nodes(
txnPoolNodeSet)
looper.run(
eventually(check_updated_bls_multi_sig_for_all_ledgers,
txnPoolNodeSet,
bls_multi_sigs_after_first_update,
FRESHNESS_TIMEOUT,
timeout=FRESHNESS_TIMEOUT + 5))
# 3. check that there is audit ledger txn created for each ledger updated as a freshness check
check_audit_ledger_updated(audit_size_initial,
txnPoolNodeSet,
audit_txns_added=3)
for node in txnPoolNodeSet:
check_audit_txn(
txn=node.auditLedger.getBySeqNo(node.auditLedger.size - 2),
view_no=view_no,
pp_seq_no=pp_seq_no + 1,
seq_no=initial_seq_no + 1,
txn_time=node.master_replica.last_accepted_pre_prepare_time,
ledger_id=POOL_LEDGER_ID,
txn_root=node.getLedger(POOL_LEDGER_ID).tree.root_hash,
state_root=node.getState(POOL_LEDGER_ID).committedHeadHash,
pool_size=initial_pool_size,
domain_size=initial_domain_size,
config_size=initial_config_size,
last_pool_seqno=None,
last_domain_seqno=2,
last_config_seqno=3,
primaries=pp_seq_no + 1 - 1)
check_audit_txn(
txn=node.auditLedger.getBySeqNo(node.auditLedger.size - 1),
view_no=view_no,
pp_seq_no=pp_seq_no + 2,
seq_no=initial_seq_no + 2,
txn_time=node.master_replica.last_accepted_pre_prepare_time,
ledger_id=DOMAIN_LEDGER_ID,
txn_root=node.getLedger(DOMAIN_LEDGER_ID).tree.root_hash,
state_root=node.getState(DOMAIN_LEDGER_ID).committedHeadHash,
pool_size=initial_pool_size,
domain_size=initial_domain_size,
config_size=initial_config_size,
last_pool_seqno=4,
last_domain_seqno=None,
last_config_seqno=3,
primaries=pp_seq_no + 2 - 1)
check_audit_txn(
txn=node.auditLedger.getBySeqNo(node.auditLedger.size),
view_no=view_no,
pp_seq_no=pp_seq_no + 3,
seq_no=initial_seq_no + 3,
txn_time=node.master_replica.last_accepted_pre_prepare_time,
ledger_id=CONFIG_LEDGER_ID,
txn_root=node.getLedger(CONFIG_LEDGER_ID).tree.root_hash,
state_root=node.getState(CONFIG_LEDGER_ID).committedHeadHash,
pool_size=initial_pool_size,
domain_size=initial_domain_size,
config_size=initial_config_size,
last_pool_seqno=4,
last_domain_seqno=5,
last_config_seqno=None,
primaries=pp_seq_no + 3 - 1)
def test_second_checkpoint_after_catchup_can_be_stabilized(
chkFreqPatched, looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_wallet_client, sdk_pool_handle, tdir, tconf,
allPluginsPath):
_, new_node = sdk_add_new_steward_and_node(
looper, sdk_pool_handle, sdk_wallet_steward,
'EpsilonSteward', 'Epsilon', tdir, tconf,
allPluginsPath=allPluginsPath)
txnPoolNodeSet.append(new_node)
looper.run(checkNodesConnected(txnPoolNodeSet))
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
# Epsilon did not participate in ordering of the batch with EpsilonSteward
# NYM transaction and the batch with Epsilon NODE transaction.
# Epsilon got these transactions via catch-up.
master_replica = new_node.replicas._master_replica
assert len(master_replica._checkpointer._checkpoint_state) == 0
assert len(master_replica._checkpointer._stashed_recvd_checkpoints) == 0
assert master_replica.h == 2
assert master_replica.H == 17
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 1)
for replica in new_node.replicas.values():
looper.run(eventually(
lambda r: assertExp(len(r._checkpointer._checkpoint_state) == 1), replica))
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 0
assert replica.h == 2
assert replica.H == 17
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 6)
stabilization_timeout = \
waits.expectedTransactionExecutionTime(len(txnPoolNodeSet))
looper.runFor(stabilization_timeout)
for replica in new_node.replicas.values():
assert len(replica._checkpointer._checkpoint_state) == 2
keys_iter = iter(replica._checkpointer._checkpoint_state)
assert next(keys_iter) == (3, 5)
assert replica._checkpointer._checkpoint_state[3, 5].seqNo == 5
assert replica._checkpointer._checkpoint_state[3, 5].digest is None
assert replica._checkpointer._checkpoint_state[3, 5].isStable is False
assert next(keys_iter) == (6, 10)
assert replica._checkpointer._checkpoint_state[6, 10].seqNo == 9
assert replica._checkpointer._checkpoint_state[6, 10].digest is None
assert replica._checkpointer._checkpoint_state[6, 10].isStable is False
# nothing is stashed since it's ordered during catch-up
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 0
assert replica.h == 2
assert replica.H == 17
sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 1)
looper.runFor(stabilization_timeout)
for replica in new_node.replicas.values():
assert len(replica._checkpointer._checkpoint_state) == 1
keys_iter = iter(replica._checkpointer._checkpoint_state)
assert next(keys_iter) == (6, 10)
assert replica._checkpointer._checkpoint_state[6, 10].seqNo == 10
assert replica._checkpointer._checkpoint_state[6, 10].digest is not None
assert replica._checkpointer._checkpoint_state[6, 10].isStable is True
assert len(replica._checkpointer._stashed_recvd_checkpoints) == 0
assert replica.h == 10
assert replica.H == 25
def try_view_change(looper, nodes):
for node in nodes:
looper.run(eventually(node.view_changer.serviceQueues))
def test_view_not_changed_when_primary_disconnected_from_less_than_quorum(
txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client):
"""
Less than quorum nodes lose connection with primary, this should not
trigger view change as the protocol can move ahead
"""
pr_node = get_master_primary_node(txnPoolNodeSet)
npr = getNonPrimaryReplicas(txnPoolNodeSet, 0)
partitioned_rep = npr[0]
partitioned_node = partitioned_rep.node
lost_pr_calls = partitioned_node.spylog.count(
partitioned_node.lost_master_primary.__name__)
recv_inst_chg_calls = {
node.name: node.spylog.count(
node.view_changer.process_instance_change_msg.__name__)
for node in txnPoolNodeSet
if node != partitioned_node and node != pr_node
}
view_no = checkViewNoForNodes(txnPoolNodeSet)
orig_retry_meth = partitioned_node.nodestack.retryDisconnected
def wont_retry(self, exclude=None):
# Do not attempt to retry connection
pass
# simulating a partition here
# Disconnect a node from only the primary of the master and dont retry to
# connect to it
partitioned_node.nodestack.retryDisconnected = types.MethodType(
wont_retry, partitioned_node.nodestack)
r = partitioned_node.nodestack.getRemote(pr_node.nodestack.name)
r.disconnect()
def chk1():
# Check that the partitioned node detects losing connection with
# primary and sends an instance change which is received by other
# nodes except the primary (since its disconnected from primary)
assert partitioned_node.spylog.count(
partitioned_node.lost_master_primary.__name__) > lost_pr_calls
for node in txnPoolNodeSet:
if node != partitioned_node and node != pr_node:
assert node.view_changer.spylog.count(
node.view_changer.process_instance_change_msg.__name__
) > recv_inst_chg_calls[node.name]
looper.run(eventually(chk1, retryWait=1, timeout=10))
def chk2():
# Check the view does not change
with pytest.raises(AssertionError):
assert checkViewNoForNodes(txnPoolNodeSet) == view_no + 1
looper.run(eventually(chk2, retryWait=1, timeout=10))
# Send some requests and make sure the request execute
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 5)
# Repair the connection so the node is no longer partitioned
partitioned_node.nodestack.retryDisconnected = types.MethodType(
orig_retry_meth, partitioned_node.nodestack)
# Send some requests and make sure the request execute
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 5)
# Partitioned node should have the same ledger and state as others
# eventually
waitNodeDataEquality(looper, partitioned_node,
*[n for n in txnPoolNodeSet if n != partitioned_node])
def test_reconnect_primary_and_not_primary(looper, txnPoolNodeSet,
sdk_wallet_steward, sdk_pool_handle,
tconf):
"""
Test steps:
Pool of 7 nodes.
count of instances must be 3
1. Choose node, that is not primary on all replicas (3 index)
2. Disconnect them
3. Ensure, that number of replicas was decreased
4. Choose current primary node (must be 0)
5. Disconnect primary
6. Ensure, that view change complete and primary was selected
7. Add node back from 1 step
8. Add node back from 4 step
9. Check, that count of instance (f+1 = 3)
10. Send some requests and check, that pool works.
"""
restNodes = set(txnPoolNodeSet)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 5)
assert txnPoolNodeSet[0].master_replica.isPrimary
node_after_all_primary = txnPoolNodeSet[3]
# Disconnect node after all primaries (after all backup primaries)
disconnect_node_and_ensure_disconnected(looper,
restNodes,
node_after_all_primary,
stopNode=False)
# -------------------------------------------------------
restNodes.remove(node_after_all_primary)
looper.run(
eventually(partial(check_count_connected_node, restNodes, 6),
timeout=5,
acceptableExceptions=[AssertionError]))
sdk_send_random_and_check(looper, restNodes, sdk_pool_handle,
sdk_wallet_steward, 5)
# Get primary node for backup replica
primary_node = txnPoolNodeSet[0]
assert primary_node.master_replica.isPrimary
old_view_no = checkViewNoForNodes(restNodes, 0)
# disconnect primary node
disconnect_node_and_ensure_disconnected(looper,
restNodes,
primary_node,
stopNode=False)
# -------------------------------------------------------
restNodes.remove(primary_node)
looper.run(
eventually(partial(check_count_connected_node, restNodes, 5),
timeout=5,
acceptableExceptions=[AssertionError]))
looper.run(
eventually(partial(checkViewNoForNodes,
restNodes,
expectedViewNo=old_view_no + 1),
timeout=tconf.VIEW_CHANGE_TIMEOUT))
sdk_send_random_and_check(looper, restNodes, sdk_pool_handle,
sdk_wallet_steward, 5)
logger.debug("restNodes: {}".format(restNodes))
restNodes.add(node_after_all_primary)
# Return back node after all primary
reconnect_node_and_ensure_connected(looper, restNodes,
node_after_all_primary)
looper.run(
checkNodesConnected(restNodes,
customTimeout=5 * tconf.RETRY_TIMEOUT_RESTRICTED))
looper.run(
eventually(partial(check_count_connected_node, restNodes, 6),
timeout=5,
acceptableExceptions=[AssertionError]))
assert len(set([len(n.replicas) for n in restNodes])) == 1
sdk_send_random_and_check(looper, restNodes, sdk_pool_handle,
sdk_wallet_steward, 5)
# Return back primary node
restNodes.add(primary_node)
reconnect_node_and_ensure_connected(looper, restNodes, primary_node)
looper.run(
checkNodesConnected(restNodes,
customTimeout=5 * tconf.RETRY_TIMEOUT_RESTRICTED))
sdk_send_random_and_check(looper, restNodes, sdk_pool_handle,
sdk_wallet_steward, 5)
def test_commit_signature_validation_integration(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward,
sdk_wallet_client,
tconf,
tdir):
'''
All nodes receive PrePrepare1(txn1 for pool_ledger)
Nodes 1, 2 ordered txn1 and nodes 3, 4 did not.
All nodes receive PrePrepare2(txn2 for domain_ledger)
Nodes 3, 4 receive commits from nodes 1, 2
Nodes 3, 4 ordered txn1
Check that all nodes ordered txn2
'''
fast_nodes = txnPoolNodeSet[:2]
slow_nodes = txnPoolNodeSet[2:]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# create new steward
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias="testClientSteward945",
role=STEWARD_STRING)
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \
prepare_new_node_data(tconf, tdir, "new_node")
# create node request to add new demote node
_, steward_did = new_steward_wallet_handle
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name="new_node",
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
services=[],
key_proof=key_proof))
first_ordered = txnPoolNodeSet[0].master_last_ordered_3PC
with ord_delay(slow_nodes):
request1 = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle,
sdk_pool_handle, node_request)
key1 = get_key_from_req(request1[0])
def check_nodes_receive_pp(view_no, seq_no):
for node in txnPoolNodeSet:
assert node.master_replica.getPrePrepare(view_no, seq_no)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 1))
def check_fast_nodes_ordered_request():
for n in fast_nodes:
assert key1 not in n.requests or n.requests[key1].executed
for n in slow_nodes:
assert not n.requests[key1].executed
looper.run(eventually(check_fast_nodes_ordered_request))
request2 = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
looper.run(eventually(check_nodes_receive_pp, first_ordered[0], first_ordered[1] + 2))
def check_nodes_receive_commits(view_no, seq_no):
for node in txnPoolNodeSet:
assert len(node.master_replica.commits[view_no, seq_no].voters) >= node.f + 1
looper.run(eventually(check_nodes_receive_commits, first_ordered[0], first_ordered[1] + 2))
sdk_get_and_check_replies(looper, [request1])
sdk_get_and_check_replies(looper, [request2])
def test_limited_stash_3pc_while_catchup(tdir, tconf, looper, testNodeClass,
txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, allPluginsPath,
chkFreqPatched):
'''
Test that the lagging_node can process messages from catchup stash after catchup
and request lost messages from other nodes.
'''
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
rest_nodes = txnPoolNodeSet[:-1]
# Check that requests executed well
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
# Stop one node
waitNodeDataEquality(looper, lagging_node, *rest_nodes)
disconnect_node_and_ensure_disconnected(looper,
txnPoolNodeSet,
lagging_node,
stopNode=True)
looper.removeProdable(lagging_node)
# Order 2 checkpoints on rest_nodes (2 txns in 2 batches)
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client,
2 * CHK_FREQ, 2)
waitNodeDataEquality(looper, *rest_nodes)
# Restart stopped node and wait for successful catch up
lagging_node = start_stopped_node(
lagging_node,
looper,
tconf,
tdir,
allPluginsPath,
start=False,
)
initial_all_ledgers_caught_up = lagging_node.spylog.count(
Node.allLedgersCaughtUp)
with delay_rules(lagging_node.nodeIbStasher, cs_delay(),
msg_rep_delay(types_to_delay=[PREPARE, PREPREPARE])):
with delay_rules(lagging_node.nodeIbStasher, cr_delay()):
looper.add(lagging_node)
txnPoolNodeSet[-1] = lagging_node
looper.run(checkNodesConnected(txnPoolNodeSet))
# Order 2 checkpoints in the first lagging node catchup (2 txns in 2 batches)
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
2 * CHK_FREQ, 2)
# Check that firs txn was ordered from stash after first catchup
looper.run(
eventually(lambda: assertExp(lagging_node.master_last_ordered_3PC[
1] == txnPoolNodeSet[0].master_last_ordered_3PC[1] - 1),
retryWait=1,
timeout=waits.expectedPoolCatchupTime(
len(txnPoolNodeSet))))
# Order 2 checkpoints in the second lagging node catchup (2 txns in 2 batches)
sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, 2 * CHK_FREQ,
2)
waitNodeDataEquality(looper, *txnPoolNodeSet, customTimeout=5)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 1)
looper.run(
eventually(lambda: assertExp(lagging_node.master_last_ordered_3PC == n.
master_last_ordered_3PC
for n in txnPoolNodeSet)))
# check that catch-up was started only twice
assert lagging_node.spylog.count(
Node.allLedgersCaughtUp) == initial_all_ledgers_caught_up + 2
def test_checkpoints_after_view_change(tconf, looper, chkFreqPatched,
reqs_for_checkpoint, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
'''
Tests that there is no infinite catchups if there is
a quorum of stashed checkpoints received during the view change
'''
# Prepare nodes
lagging_node = txnPoolNodeSet[-1]
rest_nodes = txnPoolNodeSet[:-1]
initial_all_ledgers_caught_up = lagging_node.spylog.count(
Node.allLedgersCaughtUp)
initial_start_catchup = lagging_node.spylog.count(Node.start_catchup)
with delay_rules(lagging_node.nodeIbStasher, lsDelay()):
with delay_rules(lagging_node.nodeIbStasher, vcd_delay()):
ensure_view_change(looper, txnPoolNodeSet)
looper.run(
eventually(lambda: assertExp(lagging_node.
view_change_in_progress is True),
timeout=waits.expectedPoolCatchupTime(
len(txnPoolNodeSet))))
ensureElectionsDone(looper=looper,
nodes=rest_nodes,
instances_list=range(2))
assert all(n.view_change_in_progress is False for n in rest_nodes)
assert lagging_node.view_change_in_progress is True
# make sure that more requests are being ordered while catch-up is in progress on the lagging node
# stash enough stable checkpoints for starting a catch-up
num_checkpoints = Replica.STASHED_CHECKPOINTS_BEFORE_CATCHUP + 1
num_reqs = reqs_for_checkpoint * num_checkpoints + 1
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, num_reqs)
looper.run(
eventually(check_last_ordered_3pc_on_master, rest_nodes,
(1, num_reqs + 1)))
looper.run(
eventually(check_last_ordered_3pc_on_backup, rest_nodes,
(1, num_reqs + 1)))
# all good nodes stabilized checkpoint
looper.run(eventually(chkChkpoints, rest_nodes, 2, 0))
assert get_stashed_checkpoints(
lagging_node) == num_checkpoints * len(rest_nodes)
# lagging node is doing the view change and stashing all checkpoints
assert lagging_node.view_change_in_progress is True
looper.run(
eventually(lambda: assertExp(
get_stashed_checkpoints(lagging_node) == 2 * len(rest_nodes
)),
timeout=waits.expectedPoolCatchupTime(
len(txnPoolNodeSet))))
# check that view change is finished
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
assert lagging_node.view_change_in_progress is False
# check that last_ordered is set
looper.run(
eventually(check_last_ordered_3pc_on_master, [lagging_node],
(1, num_reqs + 1)))
looper.run(
eventually(check_last_ordered_3pc_on_backup, [lagging_node],
(1, num_reqs + 1)))
# check that checkpoint is stabilized for master
looper.run(eventually(chk_chkpoints_for_instance, [lagging_node], 0, 2, 0))
# check that the catch-up is finished
assert lagging_node.mode == Mode.participating
assert lagging_node.spylog.count(
Node.allLedgersCaughtUp) == initial_all_ledgers_caught_up + 1
assert lagging_node.spylog.count(
Node.start_catchup) == initial_start_catchup + 1
waitNodeDataEquality(looper, *txnPoolNodeSet, customTimeout=5)
def test_successive_batch_do_no_change_state(looper, tdirWithPoolTxns,
tdirWithDomainTxnsUpdated, tconf,
nodeSet, trustee, trusteeWallet,
monkeypatch):
"""
Send 2 NYM txns in different batches such that the second batch does not
change state so that state root remains same, but keep the identifier
and reqId different. Make sure the first request is not ordered by the
primary before PRE-PREPARE for the second is sent.
Also check reject and commit
:return:
"""
prim_node = getPrimaryReplica(nodeSet, 0).node
other_nodes = [n for n in nodeSet if n != prim_node]
# Delay only first PRE-PREPARE
pp_seq_no_to_delay = 1
def specific_pre_prepare(wrappedMsg):
nonlocal pp_seq_no_to_delay
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare) and \
msg.instId == 0 and \
msg.ppSeqNo == pp_seq_no_to_delay:
return 5
def delay_commits(wrappedMsg):
msg, sender = wrappedMsg
if isinstance(msg, Commit) and msg.instId == 0:
return 10
def new_identity():
wallet = Wallet(randomString(5))
signer = DidSigner()
new_idr, _ = wallet.addIdentifier(signer=signer)
verkey = wallet.getVerkey(new_idr)
idy = Identity(identifier=new_idr, verkey=verkey, role=None)
return idy
def submit_id_req(idy):
nonlocal all_reqs
trusteeWallet.updateTrustAnchoredIdentity(idy)
reqs = trusteeWallet.preparePending()
all_reqs.extend(reqs)
trustee.submitReqs(*reqs)
def check_verkey(i, vk):
for node in nodeSet:
data = node.reqHandler.idrCache.getNym(i, isCommitted=True)
assert data[VERKEY] == vk
def check_uncommitted(count):
for node in nodeSet:
assert len(node.reqHandler.idrCache.unCommitted) == count
for node in other_nodes:
node.nodeIbStasher.delay(specific_pre_prepare)
idy = new_identity()
new_idr = idy.identifier
verkey = idy.verkey
all_reqs = []
# Setting the same verkey twice but in different batches with different
# request ids
for _ in range(3):
submit_id_req(idy)
looper.runFor(.2)
waitForSufficientRepliesForRequests(looper,
trustee,
requests=all_reqs,
add_delay_to_timeout=5)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_idr, verkey)
pp_seq_no_to_delay = 4
for node in other_nodes:
node.nodeIbStasher.delay(specific_pre_prepare)
# disable requesting missing 3 phase messages. Otherwise PP delay won't work
for rpl in node.replicas:
monkeypatch.setattr(rpl, '_request_missing_three_phase_messages',
lambda *x, **y: None)
# Setting the verkey to `x`, then `y` and then back to `x` but in different
# batches with different request ids. The idea is to change
# state root to `t` then `t'` and then back to `t` and observe that no
# errors are encountered
idy = new_identity()
new_idr = idy.identifier
verkey = idy.verkey
submit_id_req(idy)
looper.runFor(.2)
new_verkey = SimpleSigner().verkey
idy.verkey = new_verkey
submit_id_req(idy)
looper.runFor(.2)
idy.verkey = verkey
submit_id_req(idy)
looper.runFor(.2)
waitForSufficientRepliesForRequests(looper,
trustee,
requests=all_reqs,
add_delay_to_timeout=5)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_idr, verkey)
monkeypatch.undo()
# Delay COMMITs so that IdrCache can be checked for correct
# number of entries
uncommitteds = {}
methods = {}
for node in nodeSet:
node.nodeIbStasher.delay(delay_commits)
cache = node.reqHandler.idrCache
uncommitteds[cache._name] = []
cre = cache.currentBatchCreated
com = cache.onBatchCommitted
methods[cache._name] = (cre, com)
# Patch methods to record and check roots after commit
def patched_cre(self, stateRoot):
uncommitteds[self._name].append(stateRoot)
return methods[self._name][0](stateRoot)
def patched_com(self, stateRoot):
assert uncommitteds[self._name][0] == stateRoot
rv = methods[self._name][1](stateRoot)
uncommitteds[self._name] = uncommitteds[self._name][1:]
return rv
cache.currentBatchCreated = types.MethodType(patched_cre, cache)
cache.onBatchCommitted = types.MethodType(patched_com, cache)
# Set verkey of multiple identities
more = 5
keys = {}
for _ in range(more):
idy = new_identity()
keys[idy.identifier] = idy.verkey
submit_id_req(idy)
looper.runFor(.01)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, more, retryWait=1))
waitForSufficientRepliesForRequests(looper,
trustee,
requests=all_reqs,
add_delay_to_timeout=10)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
# The verkeys are correct
for i, v in keys.items():
check_verkey(i, v)
waitNodeDataEquality(looper, nodeSet[0], *nodeSet[1:])
keys = {}
for _ in range(3):
idy = new_identity()
keys[idy.identifier] = idy.verkey
submit_id_req(idy)
looper.runFor(.01)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, 3, retryWait=1))
# Check batch reject
for node in nodeSet:
cache = node.reqHandler.idrCache
initial = cache.unCommitted
cache.batchRejected()
# After reject, last entry is removed
assert cache.unCommitted == initial[:-1]
root = cache.unCommitted[0][0]
cache.onBatchCommitted(root)
# Calling commit with same root results in Assertion error
with pytest.raises(AssertionError):
cache.onBatchCommitted(root)
def test_unstash_three_phase_msg_after_catchup_in_view_change(
txnPoolNodeSet, looper, tconf, sdk_pool_handle, sdk_wallet_steward):
"""
1. Delay Commit on Node4
2. Order 1 req
3. Delay Commit on all nodes
4. Order 1 req
5. Delay CatchupRep on Node4
6. Delay Ledger Status and ViewChangeDones on Nodes1-3
7. Start View change on all nodes
8. Wait until Node4 got 3 stashed CatchupReps
9. Reset delaying of Commits on all Nodes
10. Reset Ledger Status on Nodes1-3
11. Check that 3 nodes finished VC while Node4 is syncing and not finished
12. Reset CatchupRep on Node4
13. Check that Node4 finished VC, and there was just 1 round of cacth-up (edited)
"""
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
view_no = txnPoolNodeSet[0].viewNo
old_stashed = slow_node.master_replica.stasher.num_stashed_future_view
last_ordered = txnPoolNodeSet[0].master_replica.last_ordered_3pc
with delay_rules(
[n.nodeIbStasher for n in txnPoolNodeSet],
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
# Delay Commit messages for slow_node.
slow_node.nodeIbStasher.delay(cDelay(sys.maxsize))
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
# Delay Commit messages for fast_nodes.
for n in fast_nodes:
n.nodeIbStasher.delay(cDelay(sys.maxsize))
request2 = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
def check_commits(commit_key):
for n in fast_nodes:
for r in n.replicas.values():
assert commit_key in r.commits
assert len(r.commits[commit_key].voters) == 1
looper.run(eventually(check_commits, (view_no, last_ordered[1] + 2)))
# Delay CatchupRep messages for the slow_node.
with delay_rules([slow_node.nodeIbStasher], cr_delay()):
with delay_rules([n.nodeIbStasher for n in fast_nodes],
vcd_delay()):
with delay_rules(
[n.nodeIbStasher for n in fast_nodes],
msg_rep_delay(types_to_delay=[LEDGER_STATUS])):
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
looper.run(
eventually(lambda: assertExp(slow_node.mode == Mode.
discovering)))
# Reset delay Commit messages for all nodes.
for n in txnPoolNodeSet:
n.nodeIbStasher.reset_delays_and_process_delayeds(
COMMIT)
assert slow_node.view_change_in_progress
assert slow_node.mode == Mode.discovering
looper.run(
eventually(_check_nodes_stashed, fast_nodes,
old_stashed,
len(txnPoolNodeSet) - 1))
looper.run(
eventually(_check_nodes_stashed, [slow_node],
old_stashed, (len(txnPoolNodeSet) - 1) * 2))
waitForViewChange(looper,
fast_nodes,
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper,
nodes=fast_nodes,
instances_list=range(
fast_nodes[0].requiredNumberOfInstances),
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
sdk_get_and_check_replies(looper, [request2])
waitForViewChange(looper, [slow_node],
expectedViewNo=view_no + 1,
customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
_check_nodes_stashed(fast_nodes, old_stashed, 0)
assert all(n.master_replica.last_ordered_3pc == (last_ordered[0],
last_ordered[1] + 2)
for n in txnPoolNodeSet)
assert slow_node.catchup_rounds_without_txns == 1
def test_checkpoint_across_views(sent_batches, chkFreqPatched, looper,
txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client):
"""
Test checkpointing across views.
This test checks that checkpointing and garbage collection works correctly
no matter if view change happened before a checkpoint or after a checkpoint
"""
batch_size = 2
send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client,
batch_size * sent_batches, sent_batches)
# Check that correct garbage collection happens
non_gced_batch_count = (
sent_batches - CHK_FREQ) if sent_batches >= CHK_FREQ else sent_batches
looper.run(
eventually(checkRequestCounts,
txnPoolNodeSet,
batch_size * non_gced_batch_count,
non_gced_batch_count,
non_gced_batch_count,
retryWait=1))
ensure_view_change(looper, txnPoolNodeSet)
ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# Check that after view change, proper clean up is done
for node in txnPoolNodeSet:
for r in node.replicas:
assert not r.checkpoints
# No stashed checkpoint for previous view
assert not [
view_no
for view_no in r.stashedRecvdCheckpoints if view_no < r.viewNo
]
assert r._h == 0
assert r._lastPrePrepareSeqNo == 0
assert r.h == 0
assert r.H == r._h + chkFreqPatched.LOG_SIZE
checkRequestCounts(txnPoolNodeSet, 0, 0, 0)
# Even after view change, chekpointing works
send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client,
batch_size * sent_batches, sent_batches)
looper.run(
eventually(checkRequestCounts,
txnPoolNodeSet,
batch_size * non_gced_batch_count,
non_gced_batch_count,
non_gced_batch_count,
retryWait=1))
# Send more batches so one more checkpoint happens. This is done so that
# when this test finishes, all requests are garbage collected and the
# next run of this test (with next param) has the calculations correct
more = CHK_FREQ - non_gced_batch_count
send_batches_of_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, batch_size * more,
more)
looper.run(
eventually(checkRequestCounts, txnPoolNodeSet, 0, 0, 0, retryWait=1))
def test_node_requests_missing_preprepares_prepares_and_commits(
looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, tdir,
allPluginsPath):
"""
1 of 4 nodes goes down ((simulate this by dropping requests)). A new request comes in and is ordered by
the 3 remaining nodes. After a while the previously disconnected node
comes back alive. Another request comes in. Check that the previously
disconnected node requests missing PREPREPARES, PREPARES and COMMITS,
orders the previous request and all the nodes successfully handles
the last request.
"""
INIT_REQS_CNT = 5
MISSING_REQS_CNT = 4
REQS_AFTER_RECONNECT_CNT = 1
disconnected_node = txnPoolNodeSet[3]
alive_nodes = txnPoolNodeSet[:3]
disconnected_node_stashers = disconnected_node.nodeIbStasher
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, INIT_REQS_CNT)
init_ledger_size = txnPoolNodeSet[0].domainLedger.size
with delay_rules_without_processing(disconnected_node_stashers,
delay_3pc()):
last_ordered_key = txnPoolNodeSet[0].master_replica.last_ordered_3pc
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, MISSING_REQS_CNT)
looper.run(
eventually(check_pp_out_of_sync,
alive_nodes, [disconnected_node],
last_ordered_key,
retryWait=1,
timeout=expectedPoolGetReadyTimeout(
len(txnPoolNodeSet))))
for node in alive_nodes:
assert node.domainLedger.size == init_ledger_size + MISSING_REQS_CNT
# Ensure that the reconnected node has not caught up though
assert disconnected_node.domainLedger.size == init_ledger_size
ordering_service = disconnected_node.master_replica._ordering_service
assert ordering_service.spylog.count(
OrderingService._request_pre_prepare) == 0
assert ordering_service.spylog.count(OrderingService._request_prepare) == 0
assert ordering_service.spylog.count(OrderingService._request_commit) == 0
assert disconnected_node.master_replica._message_req_service.spylog.count(
MessageReqService.process_message_rep) == 0
doOrderTimesBefore = ordering_service.spylog.count(
OrderingService._do_order)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, REQS_AFTER_RECONNECT_CNT)
waitNodeDataEquality(looper, disconnected_node, *alive_nodes)
assert ordering_service.spylog.count(
OrderingService._request_pre_prepare) > 0
assert ordering_service.spylog.count(OrderingService._request_prepare) > 0
assert ordering_service.spylog.count(OrderingService._request_commit) > 0
assert disconnected_node.master_replica._message_req_service.spylog.count(
MessageReqService.process_message_rep) > 0
doOrderTimesAfter = ordering_service.spylog.count(
OrderingService._do_order)
# Ensure that the reconnected node has ordered both the missed 3PC-batch and the new 3PC-batch
assert doOrderTimesAfter - doOrderTimesBefore == 2
for node in txnPoolNodeSet:
assert node.domainLedger.size == (init_ledger_size + MISSING_REQS_CNT +
REQS_AFTER_RECONNECT_CNT)
def check_all_ordered():
for node in txnPoolNodeSet:
assert node.domainLedger.size == (init_ledger_size +
MISSING_REQS_CNT +
REQS_AFTER_RECONNECT_CNT)
looper.run(eventually(check_all_ordered, timeout=20))
def test_slow_node_reverts_unordered_state_during_catchup(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Delay COMMITs to a node such that when it needs to catchup, it needs to
revert some unordered state. Also till this time the node should have
receive all COMMITs such that it will apply some of the COMMITs (
for which it has not received txns from catchup).
For this delay COMMITs by long, do catchup for a little older than the
state received in LedgerStatus, once catchup completes, reset delays and
try to process delayed COMMITs, some COMMITs will be rejected but some will
be processed since catchup was done for older ledger.
"""
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 3 * Max3PCBatchSize)
nprs = getNonPrimaryReplicas(txnPoolNodeSet, 0)
slow_node = nprs[-1].node
other_nodes = [n for n in txnPoolNodeSet if n != slow_node]
slow_master_replica = slow_node.master_replica
commit_delay = 150
catchup_rep_delay = 25
# Delay COMMITs to one node
slow_node.nodeIbStasher.delay(cDelay(commit_delay, 0))
# Make the slow node receive txns for a smaller ledger so it still finds
# the need to catchup
delay_batches = 2
make_a_node_catchup_twice(slow_node, other_nodes, DOMAIN_LEDGER_ID,
delay_batches * Max3PCBatchSize)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 6 * Max3PCBatchSize)
ensure_all_nodes_have_same_data(looper, other_nodes)
waitNodeDataInequality(looper, slow_node, *other_nodes)
def is_catchup_needed_count():
return len(
getAllReturnVals(slow_node,
slow_node.is_catchup_needed,
compare_val_to=True))
old_lcu_count = slow_node.spylog.count(slow_node.allLedgersCaughtUp)
old_cn_count = is_catchup_needed_count()
# `slow_node` is slow to receive CatchupRep, so that it
# gets a chance to order COMMITs
slow_node.nodeIbStasher.delay(cr_delay(catchup_rep_delay))
ensure_view_change(looper, txnPoolNodeSet)
# Check last ordered of `other_nodes` is same
for n1, n2 in combinations(other_nodes, 2):
lst_3pc = check_last_ordered_3pc(n1, n2)
def chk1():
# `slow_node` has prepared all 3PC messages which
# `other_nodes` have ordered
assertEquality(slow_master_replica.last_prepared_before_view_change,
lst_3pc)
looper.run(eventually(chk1, retryWait=1))
old_pc_count = slow_master_replica.spylog.count(
slow_master_replica.can_process_since_view_change_in_progress)
# Repair the network so COMMITs are received and processed
slow_node.reset_delays_and_process_delayeds(COMMIT)
def chk2():
# COMMITs are processed for prepared messages
assert slow_master_replica.spylog.count(
slow_master_replica.can_process_since_view_change_in_progress
) > old_pc_count
looper.run(eventually(chk2, retryWait=1, timeout=5))
def chk3():
# Some COMMITs were ordered but stashed and they were processed
rv = getAllReturnVals(slow_node, slow_node.processStashedOrderedReqs)
assert delay_batches in rv
looper.run(eventually(chk3, retryWait=1, timeout=catchup_rep_delay + 5))
def chk4():
# Catchup was done once
assert slow_node.spylog.count(
slow_node.allLedgersCaughtUp) > old_lcu_count
looper.run(
eventually(chk4,
retryWait=1,
timeout=waits.expectedPoolCatchupTime(len(txnPoolNodeSet))))
def chk5():
# Once catchup was done, need of other catchup was not found
assertEquality(is_catchup_needed_count(), old_cn_count)
looper.run(eventually(chk5, retryWait=1, timeout=5))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2 * Max3PCBatchSize)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def test_process_three_phase_msg_and_stashed_for_next_checkpoint(txnPoolNodeSet,
looper,
sdk_pool_handle,
sdk_wallet_client,
chkFreqPatched):
"""
1. Delay checkpoints processing on the slow_node. That is checkpoint on this node
can't be finalized.
2. Order requests for finalize checkpoints.
3. Check that a checkpoint is finalized on all nodes exclude the slow_node.
4. Order a new request.
5. Check that slow_node could not order this request and stashed all 3pc messages.
6. Reset delays.
7. Check that the last request is ordered on the slow_node, checkpoint is finalized
and stashed messages were removed.
"""
for n in txnPoolNodeSet:
for r in n.replicas.values():
r._checkpointer.update_watermark_from_3pc()
slow_node = txnPoolNodeSet[-1]
fast_nodes = txnPoolNodeSet[:-1]
old_stashed = {inst_id: r.stasher.stash_size(STASH_WATERMARKS)
for inst_id, r in slow_node.replicas.items()}
last_ordered = {inst_id: r.last_ordered_3pc
for inst_id, r in slow_node.replicas.items()}
with delay_rules([slow_node.nodeIbStasher, ],
msg_rep_delay(types_to_delay=[PREPREPARE, PREPARE, COMMIT])):
with delay_rules([slow_node.nodeIbStasher, ], chk_delay()):
sdk_send_batches_of_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
num_reqs=1 * CHK_FREQ,
num_batches=CHK_FREQ)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
looper.run(eventually(_check_checkpoint_finalize,
fast_nodes,
CHK_FREQ))
sdk_send_random_and_check(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
stashed_messages = incoming_3pc_msgs_count(len(txnPoolNodeSet))
assert all(r.stasher.stash_size(STASH_WATERMARKS) == old_stashed[inst_id] + stashed_messages
for inst_id, r in slow_node.replicas.items())
_check_batches_ordered(slow_node, last_ordered, CHK_FREQ)
for n in fast_nodes:
_check_batches_ordered(n, last_ordered, CHK_FREQ + 1)
looper.run(eventually(_check_checkpoint_finalize,
[slow_node, ],
CHK_FREQ))
looper.run(eventually(_check_batches_ordered,
slow_node, last_ordered, CHK_FREQ + 1))
assert all(r.stasher.stash_size(STASH_WATERMARKS) == old_stashed[inst_id]
for inst_id, r in slow_node.replicas.items())
def test_replay_new_bouncing(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir,
testNodeClass, tmpdir_factory,
node_config_helper_class, allPluginsPath,
some_txns_done, sdk_pool_handle, sdk_wallet_client):
alpha = txnPoolNodeSet[0]
old_view_no = alpha.viewNo
other_nodes = txnPoolNodeSet[1:]
stopping_at = time.perf_counter()
alpha.stop()
txnPoolNodesLooper.removeProdable(alpha)
txnPoolNodesLooper.run(eventually(checkViewNoForNodes, other_nodes,
old_view_no + 1, retryWait=1, timeout=30))
sdk_send_random_and_check(txnPoolNodesLooper, other_nodes,
sdk_pool_handle,
sdk_wallet_client, 10)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, other_nodes)
for node in other_nodes:
assert alpha.name not in node.nodestack.connecteds
nodeHa, nodeCHa = HA(*alpha.nodestack.ha), HA(*alpha.clientstack.ha)
config_helper = PNodeConfigHelper(alpha.name, tconf, chroot=tdir)
restarted_alpha = testNodeClass(
alpha.name,
config_helper=config_helper,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
txnPoolNodesLooper.add(restarted_alpha)
txnPoolNodeSet[0] = restarted_alpha
restarting_at = time.perf_counter()
print('Stopped for {}'.format(restarting_at - stopping_at))
sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, 10)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet)
for node in txnPoolNodeSet:
node.stop()
txnPoolNodesLooper.removeProdable(node)
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
print('-------------Replaying now---------------------')
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
restarted_alpha, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
# For disconnections
for node in other_nodes:
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
def testManual(do, be, poolNodesStarted, poolTxnStewardData, philCLI,
connectedToTest, nymAddedOut, attrAddedOut, schemaAdded,
issuerKeyAdded, aliceCLI, newKeyringOut, aliceMap, tdir,
syncLinkOutWithEndpoint, jobCertificateClaimMap,
syncedInviteAcceptedOutWithoutClaims, transcriptClaimMap,
reqClaimOut, reqClaimOut1, susanCLI, susanMap):
eventually.slowFactor = 3
# Create steward and add nyms and endpoint atttributes of all agents
_, stewardSeed = poolTxnStewardData
be(philCLI)
do('new keyring Steward',
expect=[
'New keyring Steward created', 'Active keyring set to "Steward"'
])
mapper = {'seed': stewardSeed.decode()}
do('new key with seed {seed}',
expect=['Key created in keyring Steward'],
mapper=mapper)
do('connect test', within=3, expect=connectedToTest)
# Add nym and endpoint for Faber, Acme and Thrift
agentIpAddress = "127.0.0.1"
faberAgentPort = 5555
acmeAgentPort = 6666
thriftAgentPort = 7777
faberHa = "{}:{}".format(agentIpAddress, faberAgentPort)
acmeHa = "{}:{}".format(agentIpAddress, acmeAgentPort)
thriftHa = "{}:{}".format(agentIpAddress, thriftAgentPort)
faberId = 'FuN98eH2eZybECWkofW6A9BKJxxnTatBCopfUiNxo6ZB'
acmeId = '7YD5NKn3P4wVJLesAmA1rr7sLPqW9mR1nhFdKD518k21'
thriftId = '9jegUr9vAMqoqQQUEAiCBYNQDnUbTktQY9nNspxfasZW'
faberPk = '5hmMA64DDQz5NzGJNVtRzNwpkZxktNQds21q3Wxxa62z'
acmePk = 'C5eqjU7NMVMGGfGfx2ubvX5H9X346bQt5qeziVAo3naQ'
thriftPk = 'AGBjYvyM3SFnoiDGAEzkSLHvqyzVkXeMZfKDvdpEsC2x'
for nym, ha, pk in [(faberId, faberHa, faberPk), (acmeId, acmeHa, acmePk),
(thriftId, thriftHa, thriftPk)]:
m = {
'target': nym,
'endpoint': json.dumps({ENDPOINT: {
'ha': ha,
PUBKEY: pk
}})
}
do('send NYM dest={{target}} role={role}'.format(
role=Roles.TRUST_ANCHOR.name),
within=5,
expect=nymAddedOut,
mapper=m)
do('send ATTRIB dest={target} raw={endpoint}',
within=5,
expect=attrAddedOut,
mapper=m)
# Start Faber Agent and Acme Agent
fMap = faberMap(agentIpAddress, faberAgentPort)
aMap = acmeMap(agentIpAddress, acmeAgentPort)
tMap = thriftMap(agentIpAddress, thriftAgentPort)
agentParams = [
(FaberAgent, "Faber College", faberAgentPort, buildFaberWallet),
(AcmeAgent, "Acme Corp", acmeAgentPort, buildAcmeWallet),
(ThriftAgent, "Thrift Bank", thriftAgentPort, buildThriftWallet)
]
for agentCls, agentName, agentPort, buildAgentWalletFunc in \
agentParams:
agentCls.getPassedArgs = lambda _: (agentPort, False)
TestWalletedAgent.createAndRunAgent(agentName, agentCls,
buildAgentWalletFunc(), tdir,
agentPort, philCLI.looper,
TestClient)
for p in philCLI.looper.prodables:
if p.name == 'Faber College':
faberAgent = p
if p.name == 'Acme Corp':
acmeAgent = p
if p.name == 'Thrift Bank':
thriftAgent = p
async def checkTranscriptWritten():
faberId = faberAgent.wallet.defaultId
schemaId = ID(SchemaKey("Transcript", "1.2", faberId))
schema = await faberAgent.issuer.wallet.getSchema(schemaId)
assert schema
assert schema.seqId
issuerKey = faberAgent.issuer.wallet.getPublicKey(schemaId)
assert issuerKey # TODO isinstance(issuerKey, PublicKey)
async def checkJobCertWritten():
acmeId = acmeAgent.wallet.defaultId
schemaId = ID(SchemaKey("Job-Certificate", "0.2", acmeId))
schema = await acmeAgent.issuer.wallet.getSchema(schemaId)
assert schema
assert schema.seqId
issuerKey = await acmeAgent.issuer.wallet.getPublicKey(schemaId)
assert issuerKey
assert issuerKey.seqId
philCLI.looper.run(eventually(checkTranscriptWritten, timeout=10))
philCLI.looper.run(eventually(checkJobCertWritten, timeout=10))
# Defining inner method for closures
def executeGstFlow(name, userCLI, userMap, be, connectedToTest, do, fMap,
aMap, jobCertificateClaimMap, newKeyringOut,
reqClaimOut, reqClaimOut1, syncLinkOutWithEndpoint,
syncedInviteAcceptedOutWithoutClaims, tMap,
transcriptClaimMap):
pass
async def getPublicKey(wallet, schemaId):
return await wallet.getPublicKey(schemaId)
async def getClaim(schemaId):
return await userCLI.agent.prover.wallet.getClaims(schemaId)
# Start User cli
be(userCLI)
setPromptAndKeyring(do, name, newKeyringOut, userMap)
do('connect test', within=3, expect=connectedToTest)
# Accept faber
do('load sample/faber-invitation.sovrin')
syncInvite(be, do, userCLI, syncLinkOutWithEndpoint, fMap)
do('show link faber')
acceptInvitation(be, do, userCLI, fMap,
syncedInviteAcceptedOutWithoutClaims)
# Request claim
do('show claim Transcript')
aliceRequestedTranscriptClaim(
be,
do,
userCLI,
transcriptClaimMap,
reqClaimOut,
None, # Passing None since its not used
None) # Passing None since its not used
faberSchemaId = ID(SchemaKey('Transcript', '1.2', fMap['target']))
faberIssuerKey = userCLI.looper.run(
getPublicKey(faberAgent.issuer.wallet, faberSchemaId))
userFaberIssuerKey = userCLI.looper.run(
getPublicKey(userCLI.agent.prover.wallet, faberSchemaId))
assert faberIssuerKey == userFaberIssuerKey
do('show claim Transcript')
assert userCLI.looper.run(getClaim(faberSchemaId))
# Accept acme
do('load sample/acme-job-application.sovrin')
syncInvite(be, do, userCLI, syncLinkOutWithEndpoint, aMap)
acceptInvitation(be, do, userCLI, aMap,
syncedInviteAcceptedOutWithoutClaims)
# Send claim
do('show claim request Job-Application')
do('set first_name to Alice')
do('set last_name to Garcia')
do('set phone_number to 123-45-6789')
do('show claim request Job-Application')
# Passing some args as None since they are not used in the method
jobApplicationProofSent(be, do, userCLI, aMap, None, None, None)
do('show claim Job-Certificate')
# Request new available claims Job-Certificate
jobCertClaimRequested(be, do, userCLI, None, jobCertificateClaimMap,
reqClaimOut1, None)
acmeSchemaId = ID(SchemaKey('Job-Certificate', '0.2', aMap['target']))
acmeIssuerKey = userCLI.looper.run(
getPublicKey(acmeAgent.issuer.wallet, acmeSchemaId))
userAcmeIssuerKey = userCLI.looper.run(
getPublicKey(userCLI.agent.prover.wallet, acmeSchemaId))
assert acmeIssuerKey == userAcmeIssuerKey
do('show claim Job-Certificate')
assert userCLI.looper.run(getClaim(acmeSchemaId))
# Accept thrift
do('load sample/thrift-loan-application.sovrin')
acceptInvitation(be, do, userCLI, tMap,
syncedInviteAcceptedOutWithoutClaims)
# Send claims
bankBasicClaimSent(be, do, userCLI, tMap, None)
thriftAcmeIssuerKey = userCLI.looper.run(
getPublicKey(thriftAgent.issuer.wallet, acmeSchemaId))
assert acmeIssuerKey == thriftAcmeIssuerKey
passed = False
try:
bankKYCProofSent(be, do, userCLI, tMap, None)
passed = True
except:
thriftFaberIssuerKey = userCLI.looper.run(
getPublicKey(thriftAgent.issuer.wallet, faberSchemaId))
assert faberIssuerKey == thriftFaberIssuerKey
assert passed
executeGstFlow("Alice", aliceCLI, aliceMap, be, connectedToTest, do, fMap,
aMap, jobCertificateClaimMap, newKeyringOut, reqClaimOut,
reqClaimOut1, syncLinkOutWithEndpoint,
syncedInviteAcceptedOutWithoutClaims, tMap,
transcriptClaimMap)
aliceCLI.looper.runFor(3)
executeGstFlow("Susan", susanCLI, susanMap, be, connectedToTest, do, fMap,
aMap, jobCertificateClaimMap, newKeyringOut, reqClaimOut,
reqClaimOut1, syncLinkOutWithEndpoint,
syncedInviteAcceptedOutWithoutClaims, tMap,
transcriptClaimMap)
