def changeNodeKeys(looper, stewardClient, stewardWallet, node, verkey):
nodeNym = hexToFriendly(node.nodestack.verhex)
op = {
TXN_TYPE: NODE,
TARGET_NYM: nodeNym,
VERKEY: verkey,
DATA: {
ALIAS: node.name
}
}
req = stewardWallet.signOp(op)
stewardClient.submitReqs(req)
waitForSufficientRepliesForRequests(looper,
stewardClient,
requests=[req],
fVal=1)
node.nodestack.clearLocalRoleKeep()
node.nodestack.clearRemoteRoleKeeps()
node.nodestack.clearAllDir()
node.clientstack.clearLocalRoleKeep()
node.clientstack.clearRemoteRoleKeeps()
node.clientstack.clearAllDir()
def addNewClient(role, looper, creatorClient: Client, creatorWallet: Wallet,
name: str):
req, wallet = sendAddNewClient(role, name, creatorClient, creatorWallet)
waitForSufficientRepliesForRequests(looper, creatorClient,
requests=[req])
return wallet
def test_client_resends_not_confirmed_request(looper, client1, wallet1,
txnPoolNodeSet):
"""
Check that client resends request to all nodes if it was previously sent
to one node but reply cannot be verified
"""
client = client1
wallet = wallet1
initial_submit_count = client.spylog.count(client.submitReqs)
initial_resent_count = client.spylog.count(client.resendRequests)
def sign_and_send(op):
signed = wallet.signOp(op)
return send_signed_requests(client, [signed])
buy = {'type': 'buy', 'amount': random.randint(10, 100)}
requests = sign_and_send(buy)
waitForSufficientRepliesForRequests(looper, client, requests=requests)
buy = {'type': 'get_buy'}
client._read_only_requests.add('get_buy')
requests = sign_and_send(buy)
waitForSufficientRepliesForRequests(looper, client, requests=requests)
# submitReqs should be called twice: first for but and then got get_buy
assert initial_submit_count + 2 == \
client.spylog.count(client.submitReqs)
assert initial_resent_count + 1 == \
client.spylog.count(client.resendRequests)
def test_dirty_read(looper, nodeSet, client1, wallet1):
"""
Tests the case when read request comes before write request is
not executed on some nodes
"""
slow_nodes = list(nodeSet)[2:4]
for node in slow_nodes:
logger.debug("Making node {} slow".format(node))
make_node_slow(node)
set_request = sendReqsToNodesAndVerifySuffReplies(looper,
wallet1,
client1,
numReqs=1)[0]
received_replies = getRepliesFromClientInbox(inbox=client1.inBox,
reqId=set_request.reqId)
seq_no = received_replies[0]["result"]["seqNo"]
get_request = [wallet1.signOp({"type": GET_TXN, DATA: seq_no})]
send_signed_requests(client1, get_request)
waitForSufficientRepliesForRequests(looper, client1, requests=get_request)
received_replies = getRepliesFromClientInbox(inbox=client1.inBox,
reqId=get_request[0].reqId)
results = [str(reply['result'][DATA]) for reply in received_replies]
assert len(set(results)) == 1
def submit_operation_and_get_replies(looper, wallet, client, operation):
request = wallet.signOp(operation)
wallet.pendRequest(request)
pending = wallet.preparePending()
client.submitReqs(*pending)
waitForSufficientRepliesForRequests(looper, client, requests=pending)
return getRepliesFromClientInbox(client.inBox, request.reqId)
def test_state_proof_returned_for_get_attr(looper,
addedRawAttribute,
attributeName,
attributeData,
trustAnchor,
trustAnchorWallet):
"""
Tests that state proof is returned in the reply for GET_ATTR transactions
"""
client = trustAnchor
get_attr_operation = {
TARGET_NYM: addedRawAttribute.dest,
TXN_TYPE: GET_ATTR,
RAW: attributeName
}
get_attr_request = trustAnchorWallet.signOp(get_attr_operation)
trustAnchorWallet.pendRequest(get_attr_request)
pending = trustAnchorWallet.preparePending()
client.submitReqs(*pending)
waitForSufficientRepliesForRequests(looper, trustAnchor, requests=pending)
replies = getRepliesFromClientInbox(client.inBox, get_attr_request.reqId)
expected_data = attrib_raw_data_serializer.deserialize(attributeData)
for reply in replies:
result = reply['result']
assert DATA in result
data = attrib_raw_data_serializer.deserialize(result[DATA])
assert data == expected_data
assert result[TXN_TIME]
check_valid_proof(reply, client)
def sendMoney(self, to: str, amount: int, nodes, expected: bool = True):
req = self.submit({
TXN_TYPE: CREDIT,
TARGET_NYM: to,
DATA: {
AMOUNT: amount
}
})
if expected:
waitForSufficientRepliesForRequests(self.looper,
self.client,
requests=[req])
else:
timeout = waits.expectedReqNAckQuorumTime()
for node in nodes:
self.looper.run(
eventually(checkReqNack,
self.client,
node,
req.identifier,
req.reqId,
None,
retryWait=1,
timeout=timeout))
return req
def test_msg_max_length_check_node_to_node(looper, txnPoolNodeSet, client1,
wallet1, client1Connected,
clientAndWallet2):
"""
Two clients send 2*N requests each at the same time.
N < MSG_LEN_LIMIT but 2*N > MSG_LEN_LIMIT so the requests pass the max
length check for client-node requests but do not pass the check
for node-node requests.
"""
N = 10
# it is an empirical value for N random requests
# it has to be adjusted if the world changed (see pydoc)
max_len_limit = 3000
patch_msg_len_validators(max_len_limit, txnPoolNodeSet)
client2, wallet2 = clientAndWallet2
reqs1 = sendRandomRequests(wallet1, client1, N)
reqs2 = sendRandomRequests(wallet2, client2, N)
check_reqacks(client1, looper, reqs1, txnPoolNodeSet)
check_reqacks(client2, looper, reqs2, txnPoolNodeSet)
waitForSufficientRepliesForRequests(looper, client1, requests=reqs1)
waitForSufficientRepliesForRequests(looper, client2, requests=reqs2)
def test_proof_in_reply(looper, txnPoolNodeSet,
client1, client1Connected, wallet1):
reqs = sendRandomRequests(wallet1, client1, 1)
waitForSufficientRepliesForRequests(looper, client1, requests=reqs)
req = reqs[0]
result = client1.getReply(req.identifier, req.reqId)[0]
assert result
assert result[TXN_TYPE] == "buy"
assert result[f.IDENTIFIER.nm] == req.identifier
assert result[f.REQ_ID.nm] == req.reqId
assert result[f.SEQ_NO.nm]
assert result[TXN_TIME]
assert STATE_PROOF in result
state_proof = result[STATE_PROOF]
assert ROOT_HASH in state_proof
assert MULTI_SIGNATURE in state_proof
assert PROOF_NODES in state_proof
multi_sig = state_proof[MULTI_SIGNATURE]
assert MULTI_SIGNATURE_SIGNATURE in multi_sig
assert MULTI_SIGNATURE_PARTICIPANTS in multi_sig
assert MULTI_SIGNATURE_VALUE in multi_sig
multi_sig_value = multi_sig[MULTI_SIGNATURE_VALUE]
assert MULTI_SIGNATURE_VALUE_LEDGER_ID in multi_sig_value
assert MULTI_SIGNATURE_VALUE_STATE_ROOT in multi_sig_value
assert MULTI_SIGNATURE_VALUE_TXN_ROOT in multi_sig_value
assert MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT in multi_sig_value
assert MULTI_SIGNATURE_VALUE_TIMESTAMP in multi_sig_value
assert client1.validate_multi_signature(state_proof)
assert client1.validate_proof(result)
def test_state_proof_returned_for_missing_schema(looper, trustAnchor,
trustAnchorWallet):
"""
Tests that state proof is returned in the reply for GET_NYM transactions
"""
client = trustAnchor
dest = trustAnchorWallet.defaultId
schema_name = "test_schema"
schema_version = "1.0"
get_schema_operation = {
TARGET_NYM: dest,
TXN_TYPE: GET_SCHEMA,
DATA: {
NAME: schema_name,
VERSION: schema_version,
}
}
get_schema_request = trustAnchorWallet.signOp(get_schema_operation)
trustAnchorWallet.pendRequest(get_schema_request)
pending = trustAnchorWallet.preparePending()
client.submitReqs(*pending)
waitForSufficientRepliesForRequests(looper, trustAnchor, requests=pending)
replies = getRepliesFromClientInbox(client.inBox, get_schema_request.reqId)
for reply in replies:
result = reply['result']
assert ATTR_NAMES not in result[DATA]
check_valid_proof(reply, client)
def put_load():
port = genHa()[1]
ha = HA('0.0.0.0', port)
name = "hello"
wallet = Wallet(name)
wallet.addIdentifier(signer=DidSigner(
seed=b'000000000000000000000000Steward1'))
client = Client(name, ha=ha)
with Looper(debug=True) as looper:
looper.add(client)
print('Will send {} reqs in all'.format(numReqs))
requests = sendRandomRequests(wallet, client, numReqs)
start = perf_counter()
for i in range(0, numReqs, numReqs // splits):
print('Will wait for {} now'.format(numReqs // splits))
s = perf_counter()
reqs = requests[i:i + numReqs // splits + 1]
waitForSufficientRepliesForRequests(looper,
client,
requests=reqs,
customTimeoutPerReq=100,
override_timeout_limit=True)
print('>>> Got replies for {} requests << in {}'.format(
numReqs // splits,
perf_counter() - s))
end = perf_counter()
print('>>>Total {} in {}<<<'.format(numReqs, end - start))
exit(0)
def test_different_ledger_request_interleave(tconf, looper, txnPoolNodeSet,
client1, wallet1, one_node_added,
client1Connected, tdir,
client_tdir, tdirWithPoolTxns,
steward1, stewardWallet,
allPluginsPath):
"""
Send pool and domain ledger requests such that they interleave, and do
view change in between and verify the pool is functional
"""
new_node = one_node_added
sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 2)
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
# Send domain ledger requests but don't wait for replies
requests = sendRandomRequests(wallet1, client1, 2)
# Add another node by sending pool ledger request
_, _, new_theta = nodeThetaAdded(looper,
txnPoolNodeSet,
tdir,
client_tdir,
tconf,
steward1,
stewardWallet,
allPluginsPath,
name='new_theta')
# Send more domain ledger requests but don't wait for replies
requests.extend(sendRandomRequests(wallet1, client1, 3))
# Do view change without waiting for replies
ensure_view_change(looper, nodes=txnPoolNodeSet)
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1)
# Make sure all requests are completed
waitForSufficientRepliesForRequests(looper, client1, requests=requests)
ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1)
new_steward, new_steward_wallet = addNewSteward(looper, client_tdir,
steward1, stewardWallet,
'another_ste')
# Send another pool ledger request (NODE) but don't wait for completion of
# request
next_node_name = 'next_node'
r = sendAddNewNode(tdir, tconf, next_node_name, new_steward,
new_steward_wallet)
node_req = r[0]
# Send more domain ledger requests but don't wait for replies
requests = [
node_req, *sendRandomRequests(new_steward_wallet, new_steward, 5)
]
# Make sure all requests are completed
waitForSufficientRepliesForRequests(looper, new_steward, requests=requests)
# Make sure pool is functional
ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1)
def addNewNode(looper,
stewardClient,
stewardWallet,
newNodeName,
tdir,
tconf,
allPluginsPath=None,
autoStart=True,
nodeClass=TestNode,
transformOpFunc=None,
do_post_node_creation: Callable = None):
nodeClass = nodeClass or TestNode
req, nodeIp, nodePort, clientIp, clientPort, sigseed \
= sendAddNewNode(tdir, tconf, newNodeName, stewardClient, stewardWallet,
transformOpFunc)
waitForSufficientRepliesForRequests(looper, stewardClient, requests=[req])
return create_and_start_new_node(
looper,
newNodeName,
tdir,
sigseed, (nodeIp, nodePort), (clientIp, clientPort),
tconf,
autoStart,
allPluginsPath,
nodeClass,
do_post_node_creation=do_post_node_creation)
def load():
port = genHa()[1]
ha = HA('0.0.0.0', port)
name = "hello"
wallet = Wallet(name)
wallet.addIdentifier(signer=SimpleSigner(
seed=b'000000000000000000000000Steward1'))
client = Client(name, ha=ha)
with Looper(debug=getConfig().LOOPER_DEBUG) as looper:
looper.add(client)
print('Will send {} reqs in all'.format(numReqs))
requests = sendRandomRequests(wallet, client, numReqs)
start = perf_counter()
for i in range(0, numReqs, numReqs // splits):
print('Will wait for {} now'.format(numReqs // splits))
s = perf_counter()
reqs = requests[i:i + numReqs // splits + 1]
waitForSufficientRepliesForRequests(looper,
client,
requests=reqs,
fVal=2,
customTimeoutPerReq=3)
print('>>> Got replies for {} requests << in {}'.format(
numReqs // splits,
perf_counter() - s))
end = perf_counter()
print('>>>{}<<<'.format(end - start))
exit(0)
def put_load():
port = genHa()[1]
ha = HA('0.0.0.0', port)
name = "hello"
wallet = Wallet(name)
wallet.addIdentifier(
signer=DidSigner(seed=b'000000000000000000000000Steward1'))
client = Client(name, ha=ha)
with Looper(debug=getConfig().LOOPER_DEBUG) as looper:
looper.add(client)
print('Will send {} reqs in all'.format(numReqs))
requests = sendRandomRequests(wallet, client, numReqs)
start = perf_counter()
for i in range(0, numReqs, numReqs // splits):
print('Will wait for {} now'.format(numReqs // splits))
s = perf_counter()
reqs = requests[i:i + numReqs // splits + 1]
waitForSufficientRepliesForRequests(looper, client, requests=reqs,
customTimeoutPerReq=100,
override_timeout_limit=True)
print('>>> Got replies for {} requests << in {}'.
format(numReqs // splits, perf_counter() - s))
end = perf_counter()
print('>>>Total {} in {}<<<'.format(numReqs, end - start))
exit(0)
def testReqExecWhenReturnedByMaster(tdir_for_func, tconf_for_func):
with TestNodeSet(tconf_for_func, count=4, tmpdir=tdir_for_func) as nodeSet:
with Looper(nodeSet) as looper:
client1, wallet1 = setupNodesAndClient(looper,
nodeSet,
tmpdir=tdir_for_func)
req = sendRandomRequest(wallet1, client1)
waitForSufficientRepliesForRequests(looper,
client1,
requests=[req])
async def chk():
for node in nodeSet:
entries = node.spylog.getAll(node.processOrdered.__name__)
for entry in entries:
arg = entry.params['ordered']
result = entry.result
if arg.instId == node.instances.masterId:
assert result
else:
assert result is False
timeout = waits.expectedOrderingTime(
nodeSet.nodes['Alpha'].instances.count)
looper.run(eventually(chk, timeout=timeout))
def requests(looper, wallet1, client1):
requests = []
for i in range(5):
req = sendRandomRequest(wallet1, client1)
waitForSufficientRepliesForRequests(looper, client1, requests=[req])
requests.append(req)
return requests
def addNewNode(looper,
stewardClient,
stewardWallet,
newNodeName,
tdir,
tconf,
allPluginsPath=None,
autoStart=True,
nodeClass=TestNode,
transformOpFunc=None):
nodeClass = nodeClass or TestNode
req, nodeIp, nodePort, clientIp, clientPort, sigseed \
= sendAddNewNode(tdir, tconf, newNodeName, stewardClient, stewardWallet,
transformOpFunc)
waitForSufficientRepliesForRequests(looper, stewardClient, requests=[req])
# initNodeKeysForBothStacks(newNodeName, tdir, sigseed, override=True)
# node = nodeClass(newNodeName, basedirpath=tdir, config=tconf,
# ha=(nodeIp, nodePort), cliha=(clientIp, clientPort),
# pluginPaths=allPluginsPath)
# if autoStart:
# looper.add(node)
# return node
return start_newly_added_node(looper, newNodeName, tdir, sigseed,
(nodeIp, nodePort), (clientIp, clientPort),
tconf, autoStart, allPluginsPath, nodeClass)
def write_wrapped():
req = sendRandomRequest(wallet, client)
waitForSufficientRepliesForRequests(txnPoolNodesLooper,
client,
requests=[req])
txnPoolNodesLooper.runFor(patched_dump_info_period)
return load_info(info_path)
def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, client1,
sent1):
"""
When a request has already been executed the previously executed reply
will be sent again to the client. An acknowledgement will not be sent
for a repeated request.
"""
waitForSufficientRepliesForRequests(looper, client1, requests=[sent1])
originalRequestResponsesLen = nodeCount * 2
duplicateRequestRepliesLen = nodeCount # for a duplicate request we need to
message_parts, err_msg = \
client1.nodestack.prepare_for_sending(sent1, None)
for part in message_parts:
client1.nodestack._enqueueIntoAllRemotes(part, None)
def chk():
assertLength([
response for response in client1.inBox
if (response[0].get(f.RESULT.nm)
and response[0][f.RESULT.nm][f.REQ_ID.nm] == sent1.reqId) or (
response[0].get(OP_FIELD_NAME) == REQACK
and response[0].get(f.REQ_ID.nm) == sent1.reqId)
], originalRequestResponsesLen + duplicateRequestRepliesLen)
responseTimeout = waits.expectedTransactionExecutionTime(nodeCount)
looper.run(eventually(chk, retryWait=1, timeout=responseTimeout))
def setup(tconf, looper, txnPoolNodeSet, client, wallet1):
# Patch the 3phase request sending method to send incorrect digest and
pr, otherR = getPrimaryReplica(txnPoolNodeSet, instId=0), \
getNonPrimaryReplicas(txnPoolNodeSet, instId=0)
reqs = sendRandomRequests(wallet1, client, tconf.Max3PCBatchSize)
waitForSufficientRepliesForRequests(
looper,
client,
requests=reqs,
customTimeoutPerReq=tconf.Max3PCBatchWait)
stateRoot = pr.stateRootHash(DOMAIN_LEDGER_ID, to_str=False)
origMethod = pr.create3PCBatch
malignedOnce = None
def badMethod(self, ledgerId):
nonlocal malignedOnce
pp = origMethod(ledgerId)
if not malignedOnce:
pp = updateNamedTuple(pp, digest=pp.digest + '123')
malignedOnce = True
return pp
pr.create3PCBatch = types.MethodType(badMethod, pr)
sendRandomRequests(wallet1, client, tconf.Max3PCBatchSize)
return pr, otherR, stateRoot
def test_slow_node_has_warn_unordered_log_msg(looper, nodeSet, wallet1,
client1, patch_monitors):
npr = getNonPrimaryReplicas(nodeSet, 0)[0]
slow_node = npr.node
monitor = nodeSet[0].monitor
delay = monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC * \
monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM + 10
delaysCommitProcessing(slow_node, delay=delay)
assert no_any_warn(*nodeSet), \
'all nodes do not have warnings before test'
for i in range(monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM):
req = sendRandomRequest(wallet1, client1)
waitForSufficientRepliesForRequests(looper, client1, requests=[req])
looper.runFor(monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC)
others = [node for node in nodeSet if node.name != slow_node.name]
assert no_any_warn(*others), \
'others do not have warning after test'
assert has_some_warn(slow_node), \
'slow node has the warning'
ordered_requests_keys_len_before = len(monitor.ordered_requests_keys)
# wait at least windows time
looper.runFor(monitor.WARN_NOT_PARTICIPATING_WINDOW_MINS * 60)
req = sendRandomRequest(wallet1, client1)
waitForSufficientRepliesForRequests(looper, client1, requests=[req])
assert no_any_warn(*others), 'others do not have warning'
assert no_last_warn(slow_node), \
'the last call of warn_has_lot_unordered_requests returned False ' \
'so slow node has no the warning for now'
assert len(monitor.ordered_requests_keys) < ordered_requests_keys_len_before, \
"ordered_requests_keys was cleaned up"
def testQueueingReqFromFutureView(delayed_perf_chk, looper, nodeSet, up,
wallet1, client1):
"""
Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT)
that come from a view which is greater than the current view.
- Delay reception and processing of view change messages by a non primary for master instance
=> it starts receiving 3 phase commit messages for next view
"""
lagging_node = get_last_master_non_primary_node(nodeSet)
old_view_no = lagging_node.viewNo
# Delay processing of InstanceChange and ViewChangeDone so node stashes
# 3PC messages
delay_ic = 60
lagging_node.nodeIbStasher.delay(icDelay(delay_ic))
lagging_node.nodeIbStasher.delay(vcd_delay(delay_ic))
logger.debug('{} will delay its view change'.format(lagging_node))
def chk_fut_view(view_no, is_empty):
length = len(lagging_node.msgsForFutureViews.get(view_no, ()))
if is_empty:
assert length == 0
else:
assert length > 0
return length
# No messages queued for future view
chk_fut_view(old_view_no + 1, is_empty=True)
logger.debug(
'{} does not have any messages for future views'.format(lagging_node))
# Every node except Node A should do a view change
ensure_view_change(looper, [n for n in nodeSet if n != lagging_node],
[lagging_node])
# send more requests that will be queued for the lagged node
# sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 3)
reqs = sendRandomRequests(wallet1, client1, 5)
l = looper.run(
eventually(chk_fut_view, old_view_no + 1, False, retryWait=1))
logger.debug('{} has {} messages for future views'.format(lagging_node, l))
waitForSufficientRepliesForRequests(looper, client1, requests=reqs)
# reset delays for the lagging_node node so that it finally makes view
# change
lagging_node.reset_delays_and_process_delayeds()
# Eventually no messages queued for future view
looper.run(
eventually(chk_fut_view,
old_view_no + 1,
True,
retryWait=1,
timeout=delay_ic + 10))
logger.debug(
'{} exhausted pending messages for future views'.format(lagging_node))
send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 2)
def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns,
poolTxnData, poolTxnStewardNames, tconf, shouldBePrimary, tdir):
# prepare new ha for node and client stack
subjectedNode = None
stewardName = None
stewardsSeed = None
for nodeIndex, n in enumerate(txnPoolNodeSet):
if shouldBePrimary == n.has_master_primary:
subjectedNode = n
stewardName = poolTxnStewardNames[nodeIndex]
stewardsSeed = poolTxnData["seeds"][stewardName].encode()
break
nodeStackNewHA, clientStackNewHA = genHa(2)
logger.debug("change HA for node: {} to {}".format(
subjectedNode.name, (nodeStackNewHA, clientStackNewHA)))
nodeSeed = poolTxnData["seeds"][subjectedNode.name].encode()
# change HA
stewardClient, req = changeHA(looper, tconf, subjectedNode.name, nodeSeed,
nodeStackNewHA, stewardName, stewardsSeed,
basedir=tdirWithClientPoolTxns)
waitForSufficientRepliesForRequests(looper, stewardClient,
requests=[req])
# stop node for which HA will be changed
subjectedNode.stop()
looper.removeProdable(subjectedNode)
# start node with new HA
config_helper = PNodeConfigHelper(subjectedNode.name, tconf, chroot=tdir)
restartedNode = TestNode(subjectedNode.name,
config_helper=config_helper,
config=tconf, ha=nodeStackNewHA,
cliha=clientStackNewHA)
looper.add(restartedNode)
txnPoolNodeSet[nodeIndex] = restartedNode
looper.run(checkNodesConnected(txnPoolNodeSet, customTimeout=70))
electionTimeout = waits.expectedPoolElectionTimeout(
nodeCount=len(txnPoolNodeSet),
numOfReelections=3)
ensureElectionsDone(looper,
txnPoolNodeSet,
retryWait=1,
customTimeout=electionTimeout)
# start client and check the node HA
anotherClient, _ = genTestClient(tmpdir=tdirWithClientPoolTxns,
usePoolLedger=True)
looper.add(anotherClient)
looper.run(eventually(anotherClient.ensureConnectedToNodes))
stewardWallet = Wallet(stewardName)
stewardWallet.addIdentifier(signer=DidSigner(seed=stewardsSeed))
sendReqsToNodesAndVerifySuffReplies(
looper, stewardWallet, stewardClient, 8)
def testClientSendingSameRequestAgainBeforeFirstIsProcessed(
looper, nodeSet, up, wallet1, client1):
size = len(client1.inBox)
req = sendRandomRequest(wallet1, client1)
client1.submitReqs(req)
waitForSufficientRepliesForRequests(looper, client1, requests=[req])
# Only REQACK will be sent twice by the node but not REPLY
assert len(client1.inBox) == size + 12
def updateNodeData(looper, stewardClient, stewardWallet, node, node_data):
req = sendUpdateNode(stewardClient, stewardWallet, node, node_data)
waitForSufficientRepliesForRequests(looper, stewardClient, requests=[req])
# TODO: Not needed in ZStack, remove once raet is removed
node.nodestack.clearLocalKeep()
node.nodestack.clearRemoteKeeps()
node.clientstack.clearLocalKeep()
node.clientstack.clearRemoteKeeps()
def getBalance(self) -> int:
req = self.submit({
TXN_TYPE: GET_BAL,
TARGET_NYM: self.wallet.defaultId
})
waitForSufficientRepliesForRequests(self.looper, self.client,
requests=[req])
return self.client.hasConsensus(*req.key)[BALANCE]
def test3PCOverBatchWithThresholdReqs(tconf, looper, txnPoolNodeSet, client,
wallet1):
"""
Check that 3 phase commit happens when threshold number of requests are
received and propagated.
:return:
"""
reqs = sendRandomRequests(wallet1, client, tconf.Max3PCBatchSize)
waitForSufficientRepliesForRequests(looper, client, requests=reqs)
def checkTxns(self):
req = self.submit({
TXN_TYPE: GET_ALL_TXNS,
TARGET_NYM: self.wallet.defaultId
})
waitForSufficientRepliesForRequests(self.looper, self.client,
requests=[req])
return req
def test_request_no_protocol_version(looper, txnPoolNodeSet, client1, wallet1,
request_num):
reqs = random_request_objects(request_num, protocol_version=None)
reqs = sign_request_objects(wallet1, reqs)
for req in reqs:
assert req.protocolVersion is None
send_signed_requests(client1, reqs)
waitForSufficientRepliesForRequests(looper, client1, requests=reqs)
def test3PCOverBatchWithLessThanThresholdReqs(tconf, looper, txnPoolNodeSet,
client, wallet1):
"""
Check that 3 phase commit happens when threshold number of requests are
not received but threshold time has passed
:return:
"""
reqs = sendRandomRequests(wallet1, client, tconf.Max3PCBatchSize - 1)
waitForSufficientRepliesForRequests(looper, client, requests=reqs)
def ensureUpgradeSent(looper, trustee, trusteeWallet, upgradeData):
upgrade, req = sendUpgrade(trustee, trusteeWallet, upgradeData)
waitForSufficientRepliesForRequests(looper, trustee, requests=[req])
def check():
assert trusteeWallet.getPoolUpgrade(upgrade.key).seqNo
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(check, retryWait=1, timeout=timeout))
return upgrade
def ensurePoolConfigSent(looper, trustee, trusteeWallet, sendPoolCfg):
poolCfg, req = sendPoolConfig(trustee, trusteeWallet, sendPoolCfg)
waitForSufficientRepliesForRequests(looper, trustee, requests=[req])
def check():
assert trusteeWallet.getPoolConfig(poolCfg.key).seqNo
timeout = plenumWaits.expectedReqAckQuorumTime()
looper.run(eventually(check, retryWait=1, timeout=timeout))
return poolCfg
def nodeThetaAdded(looper, nodeSet, tdirWithPoolTxns, tconf, steward,
stewardWallet, allPluginsPath, testNodeClass,
testClientClass, tdir):
newStewardName = "testClientSteward" + randomString(3)
newNodeName = "Theta"
newSteward, newStewardWallet = getClientAddedWithRole(nodeSet, tdir,
looper, steward,
stewardWallet,
newStewardName,
role=STEWARD)
sigseed = randomString(32).encode()
nodeSigner = SimpleSigner(seed=sigseed)
(nodeIp, nodePort), (clientIp, clientPort) = genHa(2)
data = {
NODE_IP: nodeIp,
NODE_PORT: nodePort,
CLIENT_IP: clientIp,
CLIENT_PORT: clientPort,
ALIAS: newNodeName,
SERVICES: [VALIDATOR, ]
}
node = Node(nodeSigner.identifier, data, newStewardWallet.defaultId)
newStewardWallet.addNode(node)
reqs = newStewardWallet.preparePending()
req, = newSteward.submitReqs(*reqs)
waitForSufficientRepliesForRequests(looper, newSteward, requests=[req])
def chk():
assert newStewardWallet.getNode(node.id).seqNo is not None
timeout = plenumWaits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
initNodeKeysForBothStacks(newNodeName, tdirWithPoolTxns, sigseed, override=True)
newNode = testNodeClass(newNodeName, basedirpath=tdir, config=tconf,
ha=(nodeIp, nodePort), cliha=(clientIp, clientPort),
pluginPaths=allPluginsPath)
nodeSet.append(newNode)
looper.add(newNode)
looper.run(checkNodesConnected(nodeSet))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward,
*nodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*nodeSet)
return newSteward, newStewardWallet, newNode
def test_successive_batch_do_no_change_state(looper, tdirWithPoolTxns,
tdirWithDomainTxnsUpdated,
tconf, nodeSet,
trustee, trusteeWallet):
"""
Send 2 NYM txns in different batches such that the second batch does not
change state so that state root remains same, but keep the identifier
and reqId different. Make sure the first request is not ordered by the
primary before PRE-PREPARE for the second is sent.
Also check reject and commit
:return:
"""
prim_node = getPrimaryReplica(nodeSet, 0).node
other_nodes = [n for n in nodeSet if n != prim_node]
# Delay only first PRE-PREPARE
pp_seq_no_to_delay = 1
def specific_pre_prepare(wrappedMsg):
nonlocal pp_seq_no_to_delay
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare) and \
msg.instId == 0 and \
msg.ppSeqNo == pp_seq_no_to_delay:
return 5
def delay_commits(wrappedMsg):
msg, sender = wrappedMsg
if isinstance(msg, Commit) and msg.instId == 0:
return 10
def new_identity():
wallet = Wallet(randomString(5))
signer = DidSigner()
new_idr, _ = wallet.addIdentifier(signer=signer)
verkey = wallet.getVerkey(new_idr)
idy = Identity(identifier=new_idr,
verkey=verkey,
role=None)
return idy
def submit_id_req(idy):
nonlocal all_reqs
trusteeWallet.updateTrustAnchoredIdentity(idy)
reqs = trusteeWallet.preparePending()
all_reqs.extend(reqs)
trustee.submitReqs(*reqs)
def check_verkey(i, vk):
for node in nodeSet:
data = node.reqHandler.idrCache.getNym(i, isCommitted=True)
assert data[VERKEY] == vk
def check_uncommitted(count):
for node in nodeSet:
assert len(node.reqHandler.idrCache.unCommitted) == count
for node in other_nodes:
node.nodeIbStasher.delay(specific_pre_prepare)
idy = new_identity()
new_idr = idy.identifier
verkey = idy.verkey
all_reqs = []
# Setting the same verkey twice but in different batches with different
# request ids
for _ in range(3):
submit_id_req(idy)
looper.runFor(.2)
waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs,
add_delay_to_timeout=5)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_idr, verkey)
pp_seq_no_to_delay = 4
for node in other_nodes:
node.nodeIbStasher.delay(specific_pre_prepare)
# Setting the verkey to `x`, then `y` and then back to `x` but in different
# batches with different with different request ids. The idea is to change
# state root to `t` then `t'` and then back to `t` and observe that no
# errors are encountered
idy = new_identity()
new_idr = idy.identifier
verkey = idy.verkey
submit_id_req(idy)
looper.runFor(.2)
new_verkey = SimpleSigner().verkey
idy.verkey = new_verkey
submit_id_req(idy)
looper.runFor(.2)
idy.verkey = verkey
submit_id_req(idy)
looper.runFor(.2)
waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs,
add_delay_to_timeout=5)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_idr, verkey)
# Dleay COMMITs so that IdrCache can be checked for correct
# number of entries
uncommitteds = {}
methods = {}
for node in nodeSet:
node.nodeIbStasher.delay(delay_commits)
cache = node.reqHandler.idrCache
uncommitteds[cache._name] = []
cre = cache.currentBatchCreated
com = cache.onBatchCommitted
methods[cache._name] = (cre, com)
# Patch methods to record and check roots after commit
def patched_cre(self, stateRoot):
uncommitteds[self._name].append(stateRoot)
return methods[self._name][0](stateRoot)
def patched_com(self, stateRoot):
assert uncommitteds[self._name][0] == stateRoot
rv = methods[self._name][1](stateRoot)
uncommitteds[self._name] = uncommitteds[self._name][1:]
return rv
cache.currentBatchCreated = types.MethodType(patched_cre, cache)
cache.onBatchCommitted = types.MethodType(patched_com, cache)
# Set verkey of multiple identities
more = 5
keys = {}
for _ in range(more):
idy = new_identity()
keys[idy.identifier] = idy.verkey
submit_id_req(idy)
looper.runFor(.01)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, more, retryWait=1))
waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs,
add_delay_to_timeout=10)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
# The verkeys are correct
for i, v in keys.items():
check_verkey(i, v)
keys = {}
for _ in range(3):
idy = new_identity()
keys[idy.identifier] = idy.verkey
submit_id_req(idy)
looper.runFor(.01)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, 3, retryWait=1))
# Check batch reject
for node in nodeSet:
cache = node.reqHandler.idrCache
initial = cache.unCommitted
cache.batchRejected()
# After reject, last entry is removed
assert cache.unCommitted == initial[:-1]
root = cache.unCommitted[0][0]
cache.onBatchCommitted(root)
# Calling commit with same root results in Assertion error
with pytest.raises(AssertionError):
cache.onBatchCommitted(root)
def test_successive_batch_do_no_change_state(looper,
tdirWithDomainTxnsUpdated,
tdirWithClientPoolTxns,
tconf, nodeSet, trustee,
trusteeWallet, monkeypatch):
"""
Send 2 NYM txns in different batches such that the second batch does not
change state so that state root remains same, but keep the identifier
and reqId different. Make sure the first request is not ordered by the
primary before PRE-PREPARE for the second is sent.
Also check reject and commit
:return:
"""
all_reqs = []
# Delay only first PRE-PREPARE
pp_seq_no_to_delay = 1
delay_pp_duration = 5
delay_cm_duration = 10
def delay_commits(wrappedMsg):
msg, sender = wrappedMsg
if isinstance(msg, Commit) and msg.instId == 0:
return delay_cm_duration
def new_identity():
wallet = Wallet(randomString(5))
signer = DidSigner()
new_idr, _ = wallet.addIdentifier(signer=signer)
verkey = wallet.getVerkey(new_idr)
idy = Identity(identifier=new_idr,
verkey=verkey,
role=None)
return idy, wallet
def submit_id_req(idy, wallet=None, client=None):
nonlocal all_reqs
wallet = wallet if wallet is not None else trusteeWallet
client = client if client is not None else trustee
wallet.updateTrustAnchoredIdentity(idy)
reqs = wallet.preparePending()
all_reqs.extend(reqs)
client.submitReqs(*reqs)
return reqs
def submit_id_req_and_wait(idy, wallet=None, client=None):
reqs = submit_id_req(idy, wallet=wallet, client=client)
looper.runFor(.2)
return reqs
def check_verkey(i, vk):
for node in nodeSet:
data = node.idrCache.getNym(i, isCommitted=True)
assert data[VERKEY] == vk
def check_uncommitted(count):
for node in nodeSet:
assert len(node.idrCache.un_committed) == count
for node in nodeSet:
for rpl in node.replicas:
monkeypatch.setattr(rpl, '_request_missing_three_phase_messages',
lambda *x, **y: None)
idy, new_wallet = new_identity()
new_idr = idy.identifier
verkey = idy.verkey
submit_id_req(idy)
waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs[-1:],
add_delay_to_timeout=delay_cm_duration)
for node in nodeSet:
node.nodeIbStasher.delay(delay_commits)
new_client, _ = genTestClient(nodeSet, tmpdir=tdirWithClientPoolTxns,
usePoolLedger=True)
looper.add(new_client)
looper.run(new_client.ensureConnectedToNodes(count=len(nodeSet)))
new_client.registerObserver(new_wallet.handleIncomingReply, name='temp')
idy.seqNo = None
# Setting the same verkey thrice but in different batches with different
# request ids
for _ in range(3):
req, = submit_id_req_and_wait(idy, wallet=new_wallet, client=new_client)
logger.debug('{} sent request {} to change verkey'.
format(new_client, req))
waitForSufficientRepliesForRequests(looper, new_client,
requests=all_reqs[-3:],
add_delay_to_timeout=delay_cm_duration)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_idr, verkey)
new_client.deregisterObserver(name='temp')
# Setting the verkey to `x`, then `y` and then back to `x` but in different
# batches with different request ids. The idea is to change
# state root to `t` then `t'` and then back to `t` and observe that no
# errors are encountered
idy, new_wallet = new_identity()
submit_id_req(idy)
waitForSufficientRepliesForRequests(looper, trustee, requests=all_reqs[-1:],
add_delay_to_timeout=delay_cm_duration)
new_client.registerObserver(new_wallet.handleIncomingReply)
idy.seqNo = None
x_signer = SimpleSigner(identifier=idy.identifier)
idy.verkey = x_signer.verkey
req, = submit_id_req_and_wait(idy, wallet=new_wallet, client=new_client)
new_wallet.updateSigner(idy.identifier, x_signer)
logger.debug('{} sent request {} to change verkey'.
format(new_client, req))
y_signer = SimpleSigner(identifier=idy.identifier)
idy.verkey = y_signer.verkey
req, = submit_id_req_and_wait(idy, wallet=new_wallet, client=new_client)
new_wallet.updateSigner(idy.identifier, y_signer)
logger.debug('{} sent request {} to change verkey'.
format(new_client, req))
idy.verkey = x_signer.verkey
req, = submit_id_req_and_wait(idy, wallet=new_wallet, client=new_client)
new_wallet.updateSigner(idy.identifier, x_signer)
logger.debug('{} sent request {} to change verkey'.
format(new_client, req))
waitForSufficientRepliesForRequests(looper, new_client,
requests=all_reqs[-3:],
add_delay_to_timeout=delay_cm_duration)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_idr, verkey)
monkeypatch.undo()
# Delay COMMITs so that IdrCache can be checked for correct
# number of entries
uncommitteds = {}
methods = {}
for node in nodeSet:
cache = node.idrCache
uncommitteds[cache._name] = []
cre = cache.currentBatchCreated
com = cache.onBatchCommitted
methods[cache._name] = (cre, com)
# Patch methods to record and check roots after commit
def patched_cre(self, stateRoot):
uncommitteds[self._name].append(stateRoot)
return methods[self._name][0](stateRoot)
def patched_com(self, stateRoot):
assert uncommitteds[self._name][0] == stateRoot
rv = methods[self._name][1](stateRoot)
uncommitteds[self._name] = uncommitteds[self._name][1:]
return rv
cache.currentBatchCreated = types.MethodType(patched_cre, cache)
cache.onBatchCommitted = types.MethodType(patched_com, cache)
# Set verkey of multiple identities
more = 5
keys = {}
for _ in range(more):
idy, _ = new_identity()
keys[idy.identifier] = idy.verkey
submit_id_req(idy)
looper.runFor(.01)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, more, retryWait=1))
waitForSufficientRepliesForRequests(looper, trustee,
requests=all_reqs[-more:],
add_delay_to_timeout=delay_cm_duration)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
# The verkeys are correct
for i, v in keys.items():
check_verkey(i, v)
waitNodeDataEquality(looper, nodeSet[0], *nodeSet[1:])
keys = {}
for _ in range(3):
idy, _ = new_identity()
keys[idy.identifier] = idy.verkey
submit_id_req(idy)
looper.runFor(.01)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, 3, retryWait=1))
# Check batch reject
for node in nodeSet:
cache = node.idrCache
initial = cache.un_committed
cache.batchRejected()
# After reject, last entry is removed
assert cache.un_committed == initial[:-1]
root = cache.un_committed[0][0]
cache.onBatchCommitted(root)
# Calling commit with same root results in Assertion error
with pytest.raises(AssertionError):
cache.onBatchCommitted(root)
