def testRequestDynamicValidation(tconf, looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
"""
Check that for requests which fail dynamic (state based) validation,
REJECT is sent to the client
:return:
"""
origMethods = []
names = {node.name: 0 for node in txnPoolNodeSet}
def rejectingMethod(self, req):
names[self.name] += 1
# Raise rejection for last request of batch
if tconf.Max3PCBatchSize - names[self.name] == 0:
raise UnauthorizedClientRequest(req.identifier,
req.reqId,
'Simulated rejection')
for node in txnPoolNodeSet:
origMethods.append(node.doDynamicValidation)
node.doDynamicValidation = types.MethodType(rejectingMethod, node)
reqs = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client,
tconf.Max3PCBatchSize)
sdk_get_and_check_replies(looper, reqs[:-1])
with pytest.raises(RequestRejectedException) as e:
sdk_get_and_check_replies(looper, reqs[-1:])
assert 'Simulated rejection' in e._excinfo[1].args[0]
for i, node in enumerate(txnPoolNodeSet):
node.doDynamicValidation = origMethods[i]
def test_plugin_client_req_fields(txn_pool_node_set_post_creation, looper,
sdk_wallet_steward, sdk_pool_handle):
"""
Test that plugin's addition of request fields and their validation is
successful
"""
op = {
TXN_TYPE: GET_BAL,
DATA: {'id': '123'}
}
# Valid field value results in successful processing
req_obj = sdk_gen_request(op, identifier=sdk_wallet_steward[1],
fix_length_dummy=randomString(dummy_field_length))
req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet_steward,
req_obj)
sdk_get_reply(looper, req)
# Invalid field value results in proper failure
_, did = sdk_wallet_steward
req = sdk_gen_request(op, identifier=did, fix_length_dummy=randomString(dummy_field_length + 1))
reqs = sdk_sign_request_objects(looper, sdk_wallet_steward, [req])
reqs = sdk_send_signed_requests(sdk_pool_handle, reqs)
with pytest.raises(RequestNackedException) as e:
sdk_get_and_check_replies(looper, reqs)
assert 'should have length' in e._excinfo[1].args[0]
def test_hook_pre_send_reply(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
def hook_add_field(*args, **kwargs):
kwargs['committed_txns'][0][foo] = foo
register_hook(txnPoolNodeSet, NodeHooks.PRE_SEND_REPLY, hook_add_field)
# Reply on request
signed_reqs = sdk_signed_random_requests(looper, sdk_wallet_client, 1)
reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs)
reply = sdk_get_and_check_replies(looper, reqs)[0]
assert foo in reply[1][f.RESULT.nm]
assert reply[1][f.RESULT.nm][foo] == foo
# Reply on repeated request
reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs)
reply = sdk_get_and_check_replies(looper, reqs)[0]
assert foo in reply[1][f.RESULT.nm]
assert reply[1][f.RESULT.nm][foo] == foo
# Reply on get_txn request
_, did = sdk_wallet_client
request = sdk_build_get_txn_request(
looper, did, reply[1][f.RESULT.nm][TXN_METADATA][f.SEQ_NO.nm])
request_couple = sdk_sign_and_send_prepared_request(looper,
sdk_wallet_client,
sdk_pool_handle,
request)
reply = sdk_get_and_check_replies(looper, [request_couple])[0]
assert foo in reply[1][f.RESULT.nm]['data']
assert reply[1][f.RESULT.nm]['data'][foo] == foo
def sdk_ensure_pool_config_sent(looper, sdk_pool_handle, sdk_wallet_trustee, pool_config_data):
_, did = sdk_wallet_trustee
pool_cfg = WPoolConfig(trustee=did, **pool_config_data)
req = pool_cfg.ledgerRequest()
req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet_trustee, req)
sdk_get_and_check_replies(looper, [req])
return pool_cfg
def testStewardCannotAddNodeWithOutFullFieldsSet(looper, tdir, tconf,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward):
"""
The case:
Steward accidentally sends the NODE txn without full fields set.
The expected result:
Steward gets NAck response from the pool.
"""
new_node_name = "Epsilon"
new_steward_wallet_handle = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_steward,
alias='New steward' + randomString(
3),
role=STEWARD_STRING)
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \
prepare_new_node_data(tconf, tdir, new_node_name)
_, steward_did = new_steward_wallet_handle
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name=new_node_name,
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
key_proof=key_proof))
# case from the ticket
request_json = json.loads(node_request)
request_json['operation'][DATA][NODE_PORT + ' '] = \
request_json['operation'][DATA][NODE_PORT]
del request_json['operation'][DATA][NODE_PORT]
node_request1 = json.dumps(request_json)
request_couple = sdk_sign_and_send_prepared_request(looper,
new_steward_wallet_handle,
sdk_pool_handle,
node_request1)
with pytest.raises(RequestNackedException) as e:
sdk_get_and_check_replies(looper, [request_couple])
assert 'missed fields - node_port' in e._excinfo[1].args[0]
for fn in (NODE_IP, CLIENT_IP, NODE_PORT, CLIENT_PORT):
request_json = json.loads(node_request)
del request_json['operation'][DATA][fn]
node_request2 = json.dumps(request_json)
request_couple = sdk_sign_and_send_prepared_request(looper,
new_steward_wallet_handle,
sdk_pool_handle,
node_request2)
# wait NAcks with exact message. it does not works for just 'is missed'
# because the 'is missed' will check only first few cases
with pytest.raises(RequestNackedException) as e:
sdk_get_and_check_replies(looper, [request_couple])
assert 'missed fields' in e._excinfo[1].args[0]
def testLoggingTxnStateForInvalidRequest(
looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, logsearch):
logsPropagate, _ = logsearch(files=['propagator.py'], funcs=['propagate'],
msgs=['propagating.*request.*from client'])
logsReject, _ = logsearch(files=['replica.py'], funcs=['consume_req_queue_for_pre_prepare'],
msgs=['encountered exception.*while processing.*will reject'])
seed = randomString(32)
wh, _ = sdk_wallet_client
nym_request, _ = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_client, seed,
"name", STEWARD_STRING))
request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client,
sdk_pool_handle, nym_request)
with pytest.raises(RequestRejectedException) as e:
sdk_get_and_check_replies(looper, [request_couple])
assert 'Only Steward is allowed to do these transactions' in e._excinfo[1].args[0]
request = json.loads(nym_request)
req_id = str(request[f.REQ_ID.nm])
digest = get_key_from_req(request)
assert any(digest in record.getMessage() for record in logsPropagate)
assert any(req_id in record.getMessage() for record in logsReject)
def test_get_txn_response_as_expected(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_steward):
seed = randomString(32)
wh, _ = sdk_wallet_steward
# filling nym request and getting steward did
# if role == None, we are adding client
nym_request, new_did = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_steward, seed,
None, None))
# sending request using 'sdk_' functions
request_couple = sdk_sign_and_send_prepared_request(
looper, sdk_wallet_steward,
sdk_pool_handle, nym_request)
result1 = sdk_get_and_check_replies(looper,
[request_couple])[0][1]['result']
seqNo = get_seq_no(result1)
_, steward_did = sdk_wallet_steward
request = sdk_build_get_txn_request(looper, steward_did, seqNo)
request_couple = \
sdk_sign_and_send_prepared_request(looper,
sdk_wallet_steward,
sdk_pool_handle,
request)
result2 = sdk_get_and_check_replies(looper,
[request_couple])[0][1]['result']
assert result1 == result2['data']
def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper,
sdk_wallet_steward, sdk_pool_handle):
"""
Check plugin dynamic validation fails and passes
"""
op = {
TXN_TYPE: AUCTION_END,
DATA: {'id': 'abcdef'}
}
reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ])
reqs = sdk_send_signed_requests(sdk_pool_handle, reqs)
with pytest.raises(CommonSdkIOException) as exc_info:
sdk_get_and_check_replies(looper, reqs)
exc_info.match('Got an error with code 113')
op = {
TXN_TYPE: AUCTION_START,
DATA: {'id': 'xyz'}
}
successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle)
op = {
TXN_TYPE: AUCTION_END,
DATA: {'id': 'xyz'}
}
successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle)
def testOrderingCase2(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Scenario -> A client sends requests, some nodes delay COMMITs to few
specific nodes such some nodes achieve commit quorum later for those
requests compared to other nodes. But all nodes `ORDER` request in the same
order of ppSeqNos
https://www.pivotaltracker.com/n/projects/1889887/stories/133655009
"""
pr, replicas = getPrimaryReplica(txnPoolNodeSet, instId=0), \
getNonPrimaryReplicas(txnPoolNodeSet, instId=0)
assert len(replicas) == 6
rep0 = pr
rep1 = replicas[0]
rep2 = replicas[1]
rep3 = replicas[2]
rep4 = replicas[3]
rep5 = replicas[4]
rep6 = replicas[5]
node0 = rep0.node
node1 = rep1.node
node2 = rep2.node
node3 = rep3.node
node4 = rep4.node
node5 = rep5.node
node6 = rep6.node
ppSeqsToDelay = 5
commitDelay = 3 # delay each COMMIT by this number of seconds
delayedPpSeqNos = set()
requestCount = 10
def specificCommits(wrappedMsg):
nonlocal node3, node4, node5
msg, sender = wrappedMsg
if isinstance(msg, PrePrepare):
if len(delayedPpSeqNos) < ppSeqsToDelay:
delayedPpSeqNos.add(msg.ppSeqNo)
logger.debug('ppSeqNo {} be delayed'.format(msg.ppSeqNo))
if isinstance(msg, Commit) and msg.instId == 0 and \
sender in (n.name for n in (node3, node4, node5)) and \
msg.ppSeqNo in delayedPpSeqNos:
return commitDelay
for node in (node1, node2):
logger.debug('{} would be delaying commits'.format(node))
node.nodeIbStasher.delay(specificCommits)
sdk_reqs = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, requestCount)
timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet))
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=timeout)
sdk_get_and_check_replies(looper, sdk_reqs)
def set_verkey(looper, sdk_pool_handle, sdk_wallet_sender, dest, verkey):
wh, _ = sdk_wallet_sender
nym_request, new_did = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_sender, None,
None, TRUST_ANCHOR_STRING, dest, verkey, False if verkey else True))
request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_sender,
sdk_pool_handle, nym_request)
sdk_get_and_check_replies(looper, [request_couple])
return wh, new_did
def sdk_add_attribute_and_check(looper, sdk_pool_handle, sdk_wallet_handle, attrib, dest=None):
_, s_did = sdk_wallet_handle
t_did = dest or s_did
attrib_req = looper.loop.run_until_complete(
build_attrib_request(s_did, t_did, None, attrib, None))
request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_handle,
sdk_pool_handle, attrib_req)
sdk_get_and_check_replies(looper, [request_couple])
return request_couple
def test_slow_nodes_catchup_before_selecting_primary_in_new_view(
tconf,
looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
one_node_added):
"""
Delay 3PC messages to one node and view change messages to some others
(including primary) so the node that does not receive enough 3PC messages is
behind but learns of the view change quickly and starts catchup.
Other nodes learn of the view change late and thus keep on processing
requests
"""
new_node = one_node_added
nprs = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)]
primary_node = getPrimaryReplica(txnPoolNodeSet, 0).node
slow_node = nprs[-1]
# nodes_slow_to_inst_chg = [primary_node] + nprs[:2]
nodes_slow_to_inst_chg = [n for n in txnPoolNodeSet if n != slow_node]
delay_3pc = 100
delay_ic = 5
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2 * Max3PCBatchSize)
delay_3pc_messages([slow_node], 0, delay_3pc)
for n in nodes_slow_to_inst_chg:
n.nodeIbStasher.delay(icDelay(delay_ic))
def start_count(): return sum([1 for e in slow_node.ledgerManager.spylog.getAll(
slow_node.ledgerManager.startCatchUpProcess.__name__)
if e.params['ledgerId'] == DOMAIN_LEDGER_ID])
s = start_count()
requests = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 10 * Max3PCBatchSize)
ensure_view_change(looper, nodes=txnPoolNodeSet,
exclude_from_check=nodes_slow_to_inst_chg)
sdk_get_and_check_replies(looper, requests)
waitNodeDataEquality(looper, slow_node, *txnPoolNodeSet[:-1])
e = start_count()
assert e - s >= 2
looper.run(eventually(checkViewNoForNodes, slow_node.viewNo))
checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 2 * Max3PCBatchSize)
waitNodeDataEquality(looper, new_node, *nodes_slow_to_inst_chg)
def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, sent1):
"""
When a request has already been executed the previously executed reply
will be sent again to the client. An acknowledgement will not be sent
for a repeated request.
"""
sdk_get_and_check_replies(looper, sent1)
req = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client)
sdk_get_and_check_replies(looper, [req])
def sdk_node_theta_added(looper,
txnPoolNodeSet,
tdir,
tconf,
sdk_pool_handle,
sdk_wallet_trustee,
allPluginsPath,
node_config_helper_class,
testNodeClass,
name=None):
new_steward_name = "testClientSteward" + randomString(3)
new_node_name = name or "Theta"
new_steward_wallet = sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_trustee,
alias=new_steward_name,
role=STEWARD_STRING)
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort = \
prepare_new_node_data(tconf, tdir, new_node_name,
configClass=node_config_helper_class)
# filling node request
_, steward_did = new_steward_wallet
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name=new_node_name,
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
services=[VALIDATOR]))
# sending request using 'sdk_' functions
request_couple = sdk_sign_and_send_prepared_request(looper, new_steward_wallet,
sdk_pool_handle, node_request)
# waitng for replies
sdk_get_and_check_replies(looper, [request_couple])
new_node = create_and_start_new_node(looper, new_node_name, tdir, sigseed,
(nodeIp, nodePort), (clientIp, clientPort),
tconf, True, allPluginsPath,
testNodeClass,
configClass=node_config_helper_class)
txnPoolNodeSet.append(new_node)
looper.run(checkNodesConnected(txnPoolNodeSet))
sdk_pool_refresh(looper, sdk_pool_handle)
return new_steward_wallet, new_node
def read_wrapped(txn_type):
op = {
TXN_TYPE: txn_type,
f.LEDGER_ID.nm: DOMAIN_LEDGER_ID,
DATA: 1
}
req = Request(identifier=did,
operation=op, reqId=getTimeBasedId(),
protocolVersion=CURRENT_PROTOCOL_VERSION)
sdk_get_and_check_replies(looper, [sdk_sign_and_submit_req_obj(
looper, sdk_pool_handle, sdk_wallet_client, req)])
return node._info_tool.info
def test_request_with_correct_version(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
request_num):
_, did = sdk_wallet_client
reqs_obj = sdk_random_request_objects(request_num, identifier=did,
protocol_version=CURRENT_PROTOCOL_VERSION)
for req_obj in reqs_obj:
assert req_obj.protocolVersion == CURRENT_PROTOCOL_VERSION
signed_reqs = sdk_sign_request_objects(looper, sdk_wallet_client, reqs_obj)
reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs)
sdk_get_and_check_replies(looper, reqs)
def test_fail_validator_info_command(
sdk_pool_handle, sdk_wallet_client, looper):
op = {
TXN_TYPE: VALIDATOR_INFO
}
req_obj = sdk_gen_request(op, identifier=sdk_wallet_client[1])
req = sdk_sign_and_submit_req_obj(looper,
sdk_pool_handle,
sdk_wallet_client,
req_obj)
with pytest.raises(RequestRejectedException) as excinfo:
sdk_get_and_check_replies(looper, [req], 100)
assert excinfo.match("None role cannot do action with type = " +
VALIDATOR_INFO)
def test_fail_pool_restart_with_steward_role(
sdk_pool_handle, sdk_wallet_steward, looper):
op = {
TXN_TYPE: POOL_RESTART,
ACTION: START,
}
req_obj = sdk_gen_request(op, identifier=sdk_wallet_steward[1])
req = sdk_sign_and_submit_req_obj(looper,
sdk_pool_handle,
sdk_wallet_steward,
req_obj)
with pytest.raises(RequestRejectedException) as excinfo:
sdk_get_and_check_replies(looper, [req], 100)
assert excinfo.match("STEWARD cannot do action with type = " +
POOL_RESTART)
def test_fail_pool_restart_with_invalid_datetime(
sdk_pool_handle, sdk_wallet_steward, looper):
invalid_datetime = "12.05.2018 4/40"
op = {
TXN_TYPE: POOL_RESTART,
ACTION: START,
DATETIME: invalid_datetime
}
req_obj = sdk_gen_request(op, identifier=sdk_wallet_steward[1])
req = sdk_sign_and_submit_req_obj(looper,
sdk_pool_handle,
sdk_wallet_steward,
req_obj)
with pytest.raises(RequestNackedException) as excinfo:
sdk_get_and_check_replies(looper, [req], 100)
assert excinfo.match("datetime " + invalid_datetime + " is not valid")
def test_dirty_read(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client):
"""
Tests the case when read request comes before write request is
not executed on some nodes
"""
slow_nodes = list(txnPoolNodeSet)[2:4]
for node in slow_nodes:
logger.debug("Making node {} slow".format(node))
make_node_slow(node)
received_replies = sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client,
1)
result = received_replies[0][1]["result"]
seq_no = get_seq_no(result)
_, did = sdk_wallet_client
req = sdk_build_get_txn_request(looper, did, seq_no)
request = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client,
sdk_pool_handle, req)
received_replies = sdk_get_and_check_replies(looper, [request])
results = [str(get_payload_data(reply['result'][DATA])) for _, reply in received_replies]
assert len(set(results)) == 1
def sdk_send_update_node(looper, sdk_submitter_wallet,
sdk_pool_handle,
destination, alias,
node_ip, node_port,
client_ip, client_port,
services=[VALIDATOR],
bls_key=None,
key_proof=None):
_, submitter_did = sdk_submitter_wallet
# filling node request
node_request = looper.loop.run_until_complete(
prepare_node_request(submitter_did,
new_node_name=alias,
clientIp=client_ip,
clientPort=client_port,
nodeIp=node_ip,
nodePort=node_port,
bls_key=bls_key,
destination=destination,
services=services,
key_proof=key_proof))
# sending request using 'sdk_' functions
request_couple = sdk_sign_and_send_prepared_request(looper, sdk_submitter_wallet,
sdk_pool_handle, node_request)
# waitng for replies
reply = sdk_get_and_check_replies(looper, [request_couple])[0][1]
sdk_pool_refresh(looper, sdk_pool_handle)
return reply
def test_get_txn_for_invalid_seq_no(looper, txnPoolNodeSet,
sdk_wallet_steward,
sdk_pool_handle):
_, steward_did = sdk_wallet_steward
# setting incorrect data
request = sdk_build_get_txn_request(looper, steward_did,
INVALID_SEQ_NO)
request_couple = \
sdk_sign_and_send_prepared_request(looper,
sdk_wallet_steward,
sdk_pool_handle,
request)
with pytest.raises(RequestNackedException) as e:
sdk_get_and_check_replies(looper, [request_couple])
assert 'cannot be smaller' in e._excinfo[1].args[0]
def test_get_txn_for_invalid_ledger_id(looper, txnPoolNodeSet,
sdk_wallet_steward,
sdk_pool_handle):
_, steward_did = sdk_wallet_steward
request = sdk_build_get_txn_request(looper, steward_did, 1)
# setting incorrect Ledger_ID
request_json = json.loads(request)
request_json['operation']['ledgerId'] = INVALID_LEDGER_ID
request = json.dumps(request_json)
request_couple = \
sdk_sign_and_send_prepared_request(looper,
sdk_wallet_steward,
sdk_pool_handle,
request)
with pytest.raises(RequestNackedException) as e:
sdk_get_and_check_replies(looper, [request_couple])
assert 'expected one of' in e._excinfo[1].args[0]
def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet,
alias=None, role=None, seed=None,
dest=None, verkey=None,skipverkey=False):
seed = seed or randomString(32)
alias = alias or randomString(5)
wh, _ = creators_wallet
# filling nym request and getting steward did
# if role == None, we are adding client
nym_request, new_did = looper.loop.run_until_complete(
prepare_nym_request(creators_wallet, seed,
alias, role, dest, verkey, skipverkey))
# sending request using 'sdk_' functions
request_couple = sdk_sign_and_send_prepared_request(looper, creators_wallet,
sdk_pool_handle, nym_request)
# waitng for replies
sdk_get_and_check_replies(looper, [request_couple])
return wh, new_did
def sdk_add_new_node(looper,
sdk_pool_handle,
steward_wallet_handle,
new_node_name,
tdir, tconf,
allPluginsPath=None, autoStart=True, nodeClass=TestNode,
do_post_node_creation: Callable = None,
services=[VALIDATOR]):
nodeClass = nodeClass or TestNode
sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \
prepare_new_node_data(tconf, tdir, new_node_name)
# filling node request
_, steward_did = steward_wallet_handle
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name=new_node_name,
clientIp=clientIp,
clientPort=clientPort,
nodeIp=nodeIp,
nodePort=nodePort,
bls_key=bls_key,
sigseed=sigseed,
services=services,
key_proof=key_proof))
# sending request using 'sdk_' functions
request_couple = sdk_sign_and_send_prepared_request(looper, steward_wallet_handle,
sdk_pool_handle, node_request)
# waitng for replies
sdk_get_and_check_replies(looper, [request_couple])
return create_and_start_new_node(looper, new_node_name, tdir, sigseed,
(nodeIp, nodePort), (clientIp, clientPort),
tconf, autoStart, allPluginsPath,
nodeClass,
do_post_node_creation=do_post_node_creation,
configClass=PNodeConfigHelper)
def replied1(looper, txnPoolNodeSet, sdk_wallet_client,
committed1, faultyNodes, sent1):
numOfNodes = len(txnPoolNodeSet)
numOfInstances = getNoInstances(numOfNodes)
quorum = numOfInstances * (numOfNodes - faultyNodes)
_, did = sdk_wallet_client
def checkOrderedCount():
resp = [requestReturnedToNode(node,
committed1.digest,
instId)
for node in txnPoolNodeSet for instId in range(numOfInstances)]
assert resp.count(True) >= quorum
orderingTimeout = waits.expectedOrderingTime(numOfInstances)
looper.run(eventually(checkOrderedCount,
retryWait=1,
timeout=orderingTimeout))
sdk_get_and_check_replies(looper, sent1)
return committed1
def test_apply_stashed_partially_ordered(looper,
txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
test_node = getNonPrimaryReplicas(txnPoolNodeSet)[0].node
test_stasher = test_node.nodeIbStasher
ledger_size = max(node.domainLedger.size for node in txnPoolNodeSet)
def check_pool_ordered_some_requests():
assert max(node.domainLedger.size for node in txnPoolNodeSet) > ledger_size
def check_test_node_has_stashed_ordered_requests():
assert len(test_node.stashedOrderedReqs) > 0
# Delay COMMITs so requests are not ordered on test node
with delay_rules(test_stasher, cDelay()):
reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, TOTAL_REQUESTS)
looper.run(eventually(check_pool_ordered_some_requests))
# Get some of txns that need to be ordered
ledger_info = test_node.ledgerManager.getLedgerInfoByType(DOMAIN_LEDGER_ID)
txns = ledger_info.ledger.uncommittedTxns
txns = txns[:len(txns) // 2]
assert len(txns) > 1
# Emulate incomplete catchup simultaneous with generation of ORDERED message
test_node.mode = Mode.syncing
test_node.master_replica.revert_unordered_batches()
looper.run(eventually(check_test_node_has_stashed_ordered_requests))
for txn in txns:
ledger_info.ledger.add(txn)
ledger_info.postTxnAddedToLedgerClbk(DOMAIN_LEDGER_ID, txn)
test_node.mode = Mode.participating
test_node.processStashedOrderedReqs()
ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
sdk_get_and_check_replies(looper, reqs)
def sdk_change_node_keys(looper, node, sdk_wallet_steward, sdk_pool_handle,
verkey):
_, steward_did = sdk_wallet_steward
node_dest = hexToFriendly(node.nodestack.verhex)
node_request = looper.loop.run_until_complete(
prepare_node_request(steward_did,
new_node_name=node.name,
destination=node_dest))
request_json = json.loads(node_request)
request_json['operation'][VERKEY] = verkey
node_request1 = json.dumps(request_json)
request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward,
sdk_pool_handle, node_request1)
sdk_get_and_check_replies(looper, [request_couple])
node.nodestack.clearLocalRoleKeep()
node.nodestack.clearRemoteRoleKeeps()
node.nodestack.clearAllDir()
node.clientstack.clearLocalRoleKeep()
node.clientstack.clearRemoteRoleKeeps()
node.clientstack.clearAllDir()
def testDoNotBlacklistClient(looper, txnPoolNodeSet,
sdk_wallet_client, sdk_pool_handle,
poolTxnClientNames):
"""
Client should be not be blacklisted by node on sending an unsigned request
"""
client_name = poolTxnClientNames[0]
_, did = sdk_wallet_client
# No node should blacklist the client
req_obj = sdk_random_request_objects(1, identifier=did,
protocol_version=CURRENT_PROTOCOL_VERSION)[0]
reqs = sdk_send_signed_requests(sdk_pool_handle, [json.dumps(req_obj.as_dict)])
with pytest.raises(RequestNackedException) as e:
sdk_get_and_check_replies(looper, reqs)
assert 'MissingSignature' in e._excinfo[1].args[0]
def chk():
for node in txnPoolNodeSet:
assert not node.isClientBlacklisted(client_name)
timeout = waits.expectedClientToPoolConnectionTimeout(len(txnPoolNodeSet))
looper.run(eventually(chk, retryWait=1, timeout=timeout))
def test_get_txn_for_non_existing_seq_no(looper, txnPoolNodeSet,
sdk_wallet_steward,
sdk_pool_handle):
_, steward_did = sdk_wallet_steward
# setting incorrect data
def generate_non_existing_seq_no():
return randint(500, 1000)
request = sdk_build_get_txn_request(looper, steward_did,
generate_non_existing_seq_no())
request_couple = \
sdk_sign_and_send_prepared_request(looper,
sdk_wallet_steward,
sdk_pool_handle,
request)
reply = sdk_get_and_check_replies(looper, [request_couple])[0][1]
assert reply['result'][DATA] is None
def test_sdk_steward_send_many(looper, sdk_pool_handle, sdk_wallet_steward):
resp_task = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_steward, 30)
repl = sdk_get_and_check_replies(looper, resp_task)
for _, resp in repl:
assert resp['result']
def test_successive_batch_do_no_change_state(looper, tconf, nodeSet,
sdk_pool_handle,
sdk_wallet_trustee, monkeypatch):
"""
Send 2 NYM txns in different batches such that the second batch does not
change state so that state root remains same, but keep the identifier
and reqId different. Make sure the first request is not ordered by the
primary before PRE-PREPARE for the second is sent.
Also check reject and commit
:return:
"""
# Disable view change during this test
for n in nodeSet:
n.nodeIbStasher.delay(icDelay())
# Delay only first PRE-PREPARE
delay_cm_duration = 10
def delay_commits(wrappedMsg):
msg, sender = wrappedMsg
if isinstance(msg, Commit) and msg.instId == 0:
return delay_cm_duration
def check_verkey(i, vk):
for node in nodeSet:
data = node.idrCache.getNym(i, isCommitted=True)
assert data[VERKEY] == vk
def check_uncommitted(count):
for node in nodeSet:
assert len(node.idrCache.un_committed) == count
for node in nodeSet:
for rpl in node.replicas:
monkeypatch.setattr(rpl, '_request_missing_three_phase_messages',
lambda *x, **y: None)
wh, did = sdk_wallet_trustee
seed = randomString(32)
(new_did, verkey) = looper.loop.run_until_complete(
create_and_store_my_did(wh, json.dumps({'seed': seed})))
sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_trustee,
dest=new_did,
verkey=verkey)
for node in nodeSet:
node.nodeIbStasher.delay(delay_commits)
# Setting the same verkey thrice but in different batches with different
# request ids
for _ in range(3):
verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_did,
new_did)
logger.debug('{} rotates his key to {}'.format(new_did, verkey))
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_did, verkey)
# Setting the verkey to `x`, then `y` and then back to `x` but in different
# batches with different request ids. The idea is to change
# state root to `t` then `t'` and then back to `t` and observe that no
# errors are encountered
seed = randomString(32)
(new_client_did, verkey) = looper.loop.run_until_complete(
create_and_store_my_did(wh, json.dumps({'seed': seed})))
sdk_add_new_nym(looper,
sdk_pool_handle,
sdk_wallet_trustee,
dest=new_client_did,
verkey=verkey)
x_seed = randomString(32)
verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did,
new_client_did)
logger.debug('{} rotates his key to {}'.format(new_client_did, verkey))
y_seed = randomString(32)
sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did,
new_client_did)
logger.debug('{} rotates his key to {}'.format(new_client_did, verkey))
verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did,
new_client_did)
logger.debug('{} rotates his key to {}'.format(new_client_did, verkey))
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
check_verkey(new_client_did, verkey)
monkeypatch.undo()
# Delay COMMITs so that IdrCache can be checked for correct
# number of entries
uncommitteds = {}
methods = {}
for node in nodeSet:
cache = node.idrCache # type: IdrCache
uncommitteds[cache._name] = []
# Since the post batch creation handler is registered (added to a list),
# find it and patch it
dh = node.get_req_handler(DOMAIN_LEDGER_ID)
for i, handler in enumerate(dh.post_batch_creation_handlers):
# Find the cache's post create handler, not hardcoding names of
# class or functions as they can change with refactoring.
if handler.__func__.__qualname__ == '{}.{}'.format(
cache.__class__.__name__,
cache.currentBatchCreated.__name__):
cre = dh.post_batch_creation_handlers[i]
break
com = cache.onBatchCommitted
methods[cache._name] = (cre, com)
# Patch methods to record and check roots after commit
def patched_cre(self, stateRoot):
uncommitteds[self._name].append(stateRoot)
return methods[self._name][0](stateRoot)
def patched_com(self, stateRoot):
assert uncommitteds[self._name][0] == stateRoot
rv = methods[self._name][1](stateRoot)
uncommitteds[self._name] = uncommitteds[self._name][1:]
return rv
dh.post_batch_creation_handlers[i] = types.MethodType(
patched_cre, cache)
cache.onBatchCommitted = types.MethodType(patched_com, cache)
# Set verkey of multiple identities
more = 5
keys = {}
reqs = []
for _ in range(more):
seed = randomString(32)
nym_request, new_did = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_trustee, seed, None, None))
keys[new_did] = json.loads(nym_request)['operation']['verkey']
reqs.append(
sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee,
sdk_pool_handle, nym_request))
looper.runFor(tconf.Max3PCBatchWait + 0.1)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, more, retryWait=1))
sdk_get_and_check_replies(looper, reqs)
# Number of uncommitted entries is 0
looper.run(eventually(check_uncommitted, 0))
# The verkeys are correct
for i, v in keys.items():
check_verkey(i, v)
waitNodeDataEquality(looper, nodeSet[0], *nodeSet[1:])
for _ in range(3):
seed = randomString(32)
nym_request, new_did = looper.loop.run_until_complete(
prepare_nym_request(sdk_wallet_trustee, seed, None, None))
reqs.append(
sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee,
sdk_pool_handle, nym_request))
looper.runFor(tconf.Max3PCBatchWait + 0.1)
# Correct number of uncommitted entries
looper.run(eventually(check_uncommitted, 3, retryWait=1))
# Check batch reject
for node in nodeSet:
cache = node.idrCache
initial = cache.un_committed
cache.batchRejected()
# After reject, last entry is removed
assert cache.un_committed == initial[:-1]
root = cache.un_committed[0][0]
cache.onBatchCommitted(root)
# Calling commit with same root results in Assertion error
with pytest.raises(AssertionError):
cache.onBatchCommitted(root)
def test_checkpoints_removed_in_view_change(chkFreqPatched,
txnPoolNodeSet,
looper,
sdk_pool_handle,
sdk_wallet_client):
'''
Check that checkpoint finalize in view change before catchup doesn't clean
necessary data from requests and 3pc queues.
'''
slow_nodes = txnPoolNodeSet[:3]
fast_nodes = txnPoolNodeSet[3:]
# delay checkpoints processing for slow_nodes
delay_msg(slow_nodes, chk_delay)
# send txns for finalizing current checkpoint
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, CHK_FREQ)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
# delay commits processing for slow_nodes
delay_msg(slow_nodes, cDelay)
requests = sdk_send_random_requests(looper, sdk_pool_handle,
sdk_wallet_client, 1)
# check that slow nodes have prepared certificate with new txn
looper.run(eventually(last_prepared_certificate,
slow_nodes,
(0, CHK_FREQ + 1)))
# check that fast_nodes ordered new txn
looper.run(eventually(last_ordered_check,
fast_nodes,
(0, CHK_FREQ + 1)))
# check that fast_nodes finalized first checkpoint and slow_nodes are not
looper.run(eventually(check_checkpoint_finalize,
fast_nodes,
1, CHK_FREQ))
for n in slow_nodes:
assert not n.master_replica._checkpointer._checkpoint_state[(1, CHK_FREQ)].isStable
# View change start emulation for change viewNo and fix last prepare
# certificate, because if we start a real view change then checkpoints will
# clean and the first checkpoint would not be need in finalizing.
for node in txnPoolNodeSet:
node.viewNo += 1
node.master_replica.on_view_change_start()
# reset delay for checkpoints
reset_delay(slow_nodes, CHECKPOINT)
# reset view change emulation and start real view change for finish it in
# a normal mode with catchup
for node in txnPoolNodeSet:
node.viewNo -= 1
ensure_view_change(looper, txnPoolNodeSet)
for n in slow_nodes:
assert not n.master_replica._checkpointer._checkpoint_state[(1, CHK_FREQ)].isStable
# Check ordering the last txn before catchup. Check client reply is enough
# because slow_nodes contains 3 nodes and without their replies sdk method
# for get reply will not successfully finish.
reset_delay(slow_nodes, COMMIT)
sdk_get_and_check_replies(looper, requests)
looper.run(eventually(last_ordered_check,
txnPoolNodeSet,
(0, CHK_FREQ + 1)))
# check view change finish and checkpoints were cleaned
ensureElectionsDone(looper, txnPoolNodeSet)
for n in slow_nodes:
assert (1, CHK_FREQ) not in n.master_replica._checkpointer._checkpoint_state
# check that all nodes have same data after new txns ordering
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, CHK_FREQ)
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def sdk_get_frozen_ledgers(looper, sdk_pool_handle, sdk_wallet):
req = build_get_frozen_ledgers_request(sdk_wallet[1])
rep = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet, req)
return sdk_get_and_check_replies(looper, [rep])[0]
def setup_aml(looper, txnPoolNodeSet, taa_aml_request_module, sdk_pool_handle,
sdk_wallet_trustee):
req = sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee,
sdk_pool_handle,
taa_aml_request_module)
sdk_get_and_check_replies(looper, [req])
def test_primary_send_incorrect_pp(looper, txnPoolNodeSet, tconf,
allPluginsPath, sdk_pool_handle,
sdk_wallet_steward, monkeypatch):
"""
Test steps:
Delay message requests with PrePrepares on `slow_node`
Patch sending for PrePrepare on the `malicious_primary` to send an invalid PrePrepare to slow_node
Order a new request
Start a view change
Make sure it's finished on all nodes
Make sure that the lagging node has same data with other nodes
"""
start_view_no = txnPoolNodeSet[0].viewNo
slow_node = txnPoolNodeSet[-1]
malicious_primary = txnPoolNodeSet[0]
other_nodes = [
n for n in txnPoolNodeSet if n not in [slow_node, malicious_primary]
]
timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet))
ensure_all_nodes_have_same_data(looper,
txnPoolNodeSet,
custom_timeout=timeout)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
old_sender = malicious_primary.master_replica._ordering_service._send
def patched_sender(msg, dst=None, stat=None):
if isinstance(msg, PrePrepare) and msg:
old_sender(msg, [n.name for n in other_nodes], stat)
pp_dict = msg._asdict()
pp_dict["ppTime"] += 1
pp = PrePrepare(**pp_dict)
old_sender(pp, [slow_node.name], stat)
monkeypatch.undo()
monkeypatch.setattr(malicious_primary.master_replica._ordering_service,
'_send', patched_sender)
monkeypatch.setattr(slow_node.master_replica._ordering_service,
'_validate_applied_pre_prepare', lambda a, b, c: None)
with delay_rules(slow_node.nodeIbStasher,
msg_rep_delay(types_to_delay=[PREPREPARE])):
preprepare_process_num = slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_preprepare)
resp_task = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_steward)
def chk():
assert preprepare_process_num + 1 == slow_node.master_replica._ordering_service.spylog.count(
OrderingService.process_preprepare)
looper.run(eventually(chk))
_, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0]
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_steward, 1)
for n in txnPoolNodeSet:
n.view_changer.on_master_degradation()
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
waitForViewChange(looper,
txnPoolNodeSet,
expectedViewNo=start_view_no + 1)
ensureElectionsDone(looper=looper,
nodes=txnPoolNodeSet,
instances_list=[0, 1])
ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward,
sdk_pool_handle)
def test_chain_set_fees_and_xfer_batch_size_2(looper, helpers,
nodeSetWithIntegratedTokenPlugin,
sdk_pool_handle,
sdk_wallet_trustee, mint_tokens,
addresses, poolConfigWTFF):
"""
Set FEES for XFER for 2
Send any transaction to config ledger.
Send XFER with fees 2 from A to B
Set FEES for XFER for 3
Send any transaction to config ledger.
Send XFER with fees 3 from A to B
Check that first XFER is not written and second XFER is.
"""
A, B = addresses
current_amount = get_amount_from_token_txn(mint_tokens)
seq_no = get_seq_no(mint_tokens)
transfer_summ = 20
# Set fees and some config txn
helpers.node.set_fees_directly({XFER_PUBLIC_FEES_ALIAS: 42})
fees_xfer_2 = {XFER_PUBLIC_FEES_ALIAS: 2}
fees_2_rsp = helpers.general.set_fees_without_waiting(fees_xfer_2)
sdk_pool_config_sent(looper, sdk_pool_handle, sdk_wallet_trustee,
poolConfigWTFF)
sdk_get_and_check_replies(looper, fees_2_rsp)
# XFER with fees 2 from A to B
_, _, a_b_transfer_2 = send_and_check_transfer(helpers, [A, B],
fees_xfer_2,
looper,
current_amount,
seq_no,
transfer_summ=transfer_summ,
check_reply=False)
# Set fees for XFER to 3
fees_xfer_3 = {XFER_PUBLIC_FEES_ALIAS: 3}
fees_3_rsp = helpers.general.set_fees_without_waiting(fees_xfer_3)
sdk_pool_config_sent(looper, sdk_pool_handle, sdk_wallet_trustee,
poolConfigWTFF)
sdk_get_and_check_replies(looper, fees_3_rsp)
# Send XFER with fees from A to B
a_amount, seq_no, a_b_transfer_3 = send_and_check_transfer(
helpers, [A, B],
fees_xfer_3,
looper,
current_amount,
seq_no,
transfer_summ=transfer_summ,
check_reply=False)
for n in nodeSetWithIntegratedTokenPlugin:
fee_rq = n.read_manager.request_handlers[
FeesTransactions.GET_FEES.value]
assert fee_rq
assert fee_rq.get_fees(is_committed=True,
with_proof=False) == fees_xfer_3
with pytest.raises(RequestRejectedException):
sdk_get_and_check_replies(looper, a_b_transfer_2)
sdk_get_and_check_replies(looper, a_b_transfer_3)
a_get = helpers.general.do_get_utxo(A)
assert a_get[OUTPUTS][1][AMOUNT] == a_amount
assert a_get[OUTPUTS][1][SEQNO] == seq_no
b_get = helpers.general.do_get_utxo(B)
assert b_get[OUTPUTS][0][AMOUNT] == transfer_summ
assert b_get[OUTPUTS][0][SEQNO] == seq_no
ensure_all_nodes_have_same_data(looper, nodeSetWithIntegratedTokenPlugin)
def sdk_send_freeze_ledgers(looper, sdk_pool_handle, sdk_wallets, ledgers_ids: [int]):
req = build_freeze_ledgers_request(sdk_wallets[0][1], ledgers_ids)
signed_reqs = sdk_multi_sign_request_objects(looper, sdk_wallets, [req])
reps = sdk_send_signed_requests(sdk_pool_handle, signed_reqs)
return sdk_get_and_check_replies(looper, reps)[0]
def test_sdk_new_steward_send(looper, sdk_pool_handle, sdk_wallet_new_steward):
resp_task = sdk_send_random_request(looper, sdk_pool_handle,
sdk_wallet_new_steward)
_, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0]
assert j_resp['result']
def prepare(self):
self.default_auth_rule = self.get_default_auth_rule()
self.changed_auth_rule = self.get_changed_auth_rule()
req = self.taa_aml_request()
rep = sdk_sign_and_send_prepared_request(self.looper, self.trustee_wallet, self.sdk_pool_handle, req)
sdk_get_and_check_replies(self.looper, [rep])
def sdk_send_txn_author_agreement(looper, sdk_pool_handle, sdk_wallet,
text: str, version: str):
req = looper.loop.run_until_complete(
build_txn_author_agreement_request(sdk_wallet[1], text, version))
rep = sdk_sign_and_submit_req(sdk_pool_handle, sdk_wallet, req)
return sdk_get_and_check_replies(looper, [rep])[0]
def successful_op(looper, op, sdk_wallet, sdk_pool_handle):
req_obj = sdk_gen_request(op, identifier=sdk_wallet[1])
req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle,
sdk_wallet, req_obj)
return sdk_get_and_check_replies(looper, [req])
def sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee,
upgrade_data):
req = sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_trustee,
upgrade_data)
sdk_get_and_check_replies(looper, [req])
