def testRequestDynamicValidation(tconf, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Check that for requests which fail dynamic (state based) validation, REJECT is sent to the client :return: """ origMethods = [] names = {node.name: 0 for node in txnPoolNodeSet} def rejectingMethod(self, req): names[self.name] += 1 # Raise rejection for last request of batch if tconf.Max3PCBatchSize - names[self.name] == 0: raise UnauthorizedClientRequest(req.identifier, req.reqId, 'Simulated rejection') for node in txnPoolNodeSet: origMethods.append(node.doDynamicValidation) node.doDynamicValidation = types.MethodType(rejectingMethod, node) reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, tconf.Max3PCBatchSize) sdk_get_and_check_replies(looper, reqs[:-1]) with pytest.raises(RequestRejectedException) as e: sdk_get_and_check_replies(looper, reqs[-1:]) assert 'Simulated rejection' in e._excinfo[1].args[0] for i, node in enumerate(txnPoolNodeSet): node.doDynamicValidation = origMethods[i]
def test_plugin_client_req_fields(txn_pool_node_set_post_creation, looper, sdk_wallet_steward, sdk_pool_handle): """ Test that plugin's addition of request fields and their validation is successful """ op = { TXN_TYPE: GET_BAL, DATA: {'id': '123'} } # Valid field value results in successful processing req_obj = sdk_gen_request(op, identifier=sdk_wallet_steward[1], fix_length_dummy=randomString(dummy_field_length)) req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet_steward, req_obj) sdk_get_reply(looper, req) # Invalid field value results in proper failure _, did = sdk_wallet_steward req = sdk_gen_request(op, identifier=did, fix_length_dummy=randomString(dummy_field_length + 1)) reqs = sdk_sign_request_objects(looper, sdk_wallet_steward, [req]) reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, reqs) assert 'should have length' in e._excinfo[1].args[0]
def test_hook_pre_send_reply(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): def hook_add_field(*args, **kwargs): kwargs['committed_txns'][0][foo] = foo register_hook(txnPoolNodeSet, NodeHooks.PRE_SEND_REPLY, hook_add_field) # Reply on request signed_reqs = sdk_signed_random_requests(looper, sdk_wallet_client, 1) reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs) reply = sdk_get_and_check_replies(looper, reqs)[0] assert foo in reply[1][f.RESULT.nm] assert reply[1][f.RESULT.nm][foo] == foo # Reply on repeated request reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs) reply = sdk_get_and_check_replies(looper, reqs)[0] assert foo in reply[1][f.RESULT.nm] assert reply[1][f.RESULT.nm][foo] == foo # Reply on get_txn request _, did = sdk_wallet_client request = sdk_build_get_txn_request( looper, did, reply[1][f.RESULT.nm][TXN_METADATA][f.SEQ_NO.nm]) request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client, sdk_pool_handle, request) reply = sdk_get_and_check_replies(looper, [request_couple])[0] assert foo in reply[1][f.RESULT.nm]['data'] assert reply[1][f.RESULT.nm]['data'][foo] == foo
def sdk_ensure_pool_config_sent(looper, sdk_pool_handle, sdk_wallet_trustee, pool_config_data): _, did = sdk_wallet_trustee pool_cfg = WPoolConfig(trustee=did, **pool_config_data) req = pool_cfg.ledgerRequest() req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet_trustee, req) sdk_get_and_check_replies(looper, [req]) return pool_cfg
def testStewardCannotAddNodeWithOutFullFieldsSet(looper, tdir, tconf, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward): """ The case: Steward accidentally sends the NODE txn without full fields set. The expected result: Steward gets NAck response from the pool. """ new_node_name = "Epsilon" new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, alias='New steward' + randomString( 3), role=STEWARD_STRING) sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \ prepare_new_node_data(tconf, tdir, new_node_name) _, steward_did = new_steward_wallet_handle node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name=new_node_name, clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, key_proof=key_proof)) # case from the ticket request_json = json.loads(node_request) request_json['operation'][DATA][NODE_PORT + ' '] = \ request_json['operation'][DATA][NODE_PORT] del request_json['operation'][DATA][NODE_PORT] node_request1 = json.dumps(request_json) request_couple = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle, sdk_pool_handle, node_request1) with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, [request_couple]) assert 'missed fields - node_port' in e._excinfo[1].args[0] for fn in (NODE_IP, CLIENT_IP, NODE_PORT, CLIENT_PORT): request_json = json.loads(node_request) del request_json['operation'][DATA][fn] node_request2 = json.dumps(request_json) request_couple = sdk_sign_and_send_prepared_request(looper, new_steward_wallet_handle, sdk_pool_handle, node_request2) # wait NAcks with exact message. it does not works for just 'is missed' # because the 'is missed' will check only first few cases with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, [request_couple]) assert 'missed fields' in e._excinfo[1].args[0]
def testLoggingTxnStateForInvalidRequest( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, logsearch): logsPropagate, _ = logsearch(files=['propagator.py'], funcs=['propagate'], msgs=['propagating.*request.*from client']) logsReject, _ = logsearch(files=['replica.py'], funcs=['consume_req_queue_for_pre_prepare'], msgs=['encountered exception.*while processing.*will reject']) seed = randomString(32) wh, _ = sdk_wallet_client nym_request, _ = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_client, seed, "name", STEWARD_STRING)) request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client, sdk_pool_handle, nym_request) with pytest.raises(RequestRejectedException) as e: sdk_get_and_check_replies(looper, [request_couple]) assert 'Only Steward is allowed to do these transactions' in e._excinfo[1].args[0] request = json.loads(nym_request) req_id = str(request[f.REQ_ID.nm]) digest = get_key_from_req(request) assert any(digest in record.getMessage() for record in logsPropagate) assert any(req_id in record.getMessage() for record in logsReject)
def test_get_txn_response_as_expected(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward): seed = randomString(32) wh, _ = sdk_wallet_steward # filling nym request and getting steward did # if role == None, we are adding client nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_steward, seed, None, None)) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request( looper, sdk_wallet_steward, sdk_pool_handle, nym_request) result1 = sdk_get_and_check_replies(looper, [request_couple])[0][1]['result'] seqNo = get_seq_no(result1) _, steward_did = sdk_wallet_steward request = sdk_build_get_txn_request(looper, steward_did, seqNo) request_couple = \ sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward, sdk_pool_handle, request) result2 = sdk_get_and_check_replies(looper, [request_couple])[0][1]['result'] assert result1 == result2['data']
def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, sdk_wallet_steward, sdk_pool_handle): """ Check plugin dynamic validation fails and passes """ op = { TXN_TYPE: AUCTION_END, DATA: {'id': 'abcdef'} } reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) with pytest.raises(CommonSdkIOException) as exc_info: sdk_get_and_check_replies(looper, reqs) exc_info.match('Got an error with code 113') op = { TXN_TYPE: AUCTION_START, DATA: {'id': 'xyz'} } successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) op = { TXN_TYPE: AUCTION_END, DATA: {'id': 'xyz'} } successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle)
def testOrderingCase2(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Scenario -> A client sends requests, some nodes delay COMMITs to few specific nodes such some nodes achieve commit quorum later for those requests compared to other nodes. But all nodes `ORDER` request in the same order of ppSeqNos https://www.pivotaltracker.com/n/projects/1889887/stories/133655009 """ pr, replicas = getPrimaryReplica(txnPoolNodeSet, instId=0), \ getNonPrimaryReplicas(txnPoolNodeSet, instId=0) assert len(replicas) == 6 rep0 = pr rep1 = replicas[0] rep2 = replicas[1] rep3 = replicas[2] rep4 = replicas[3] rep5 = replicas[4] rep6 = replicas[5] node0 = rep0.node node1 = rep1.node node2 = rep2.node node3 = rep3.node node4 = rep4.node node5 = rep5.node node6 = rep6.node ppSeqsToDelay = 5 commitDelay = 3 # delay each COMMIT by this number of seconds delayedPpSeqNos = set() requestCount = 10 def specificCommits(wrappedMsg): nonlocal node3, node4, node5 msg, sender = wrappedMsg if isinstance(msg, PrePrepare): if len(delayedPpSeqNos) < ppSeqsToDelay: delayedPpSeqNos.add(msg.ppSeqNo) logger.debug('ppSeqNo {} be delayed'.format(msg.ppSeqNo)) if isinstance(msg, Commit) and msg.instId == 0 and \ sender in (n.name for n in (node3, node4, node5)) and \ msg.ppSeqNo in delayedPpSeqNos: return commitDelay for node in (node1, node2): logger.debug('{} would be delaying commits'.format(node)) node.nodeIbStasher.delay(specificCommits) sdk_reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, requestCount) timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=timeout) sdk_get_and_check_replies(looper, sdk_reqs)
def set_verkey(looper, sdk_pool_handle, sdk_wallet_sender, dest, verkey): wh, _ = sdk_wallet_sender nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_sender, None, None, TRUST_ANCHOR_STRING, dest, verkey, False if verkey else True)) request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_sender, sdk_pool_handle, nym_request) sdk_get_and_check_replies(looper, [request_couple]) return wh, new_did
def sdk_add_attribute_and_check(looper, sdk_pool_handle, sdk_wallet_handle, attrib, dest=None): _, s_did = sdk_wallet_handle t_did = dest or s_did attrib_req = looper.loop.run_until_complete( build_attrib_request(s_did, t_did, None, attrib, None)) request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_handle, sdk_pool_handle, attrib_req) sdk_get_and_check_replies(looper, [request_couple]) return request_couple
def test_slow_nodes_catchup_before_selecting_primary_in_new_view( tconf, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, one_node_added): """ Delay 3PC messages to one node and view change messages to some others (including primary) so the node that does not receive enough 3PC messages is behind but learns of the view change quickly and starts catchup. Other nodes learn of the view change late and thus keep on processing requests """ new_node = one_node_added nprs = [r.node for r in getNonPrimaryReplicas(txnPoolNodeSet, 0)] primary_node = getPrimaryReplica(txnPoolNodeSet, 0).node slow_node = nprs[-1] # nodes_slow_to_inst_chg = [primary_node] + nprs[:2] nodes_slow_to_inst_chg = [n for n in txnPoolNodeSet if n != slow_node] delay_3pc = 100 delay_ic = 5 sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2 * Max3PCBatchSize) delay_3pc_messages([slow_node], 0, delay_3pc) for n in nodes_slow_to_inst_chg: n.nodeIbStasher.delay(icDelay(delay_ic)) def start_count(): return sum([1 for e in slow_node.ledgerManager.spylog.getAll( slow_node.ledgerManager.startCatchUpProcess.__name__) if e.params['ledgerId'] == DOMAIN_LEDGER_ID]) s = start_count() requests = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 10 * Max3PCBatchSize) ensure_view_change(looper, nodes=txnPoolNodeSet, exclude_from_check=nodes_slow_to_inst_chg) sdk_get_and_check_replies(looper, requests) waitNodeDataEquality(looper, slow_node, *txnPoolNodeSet[:-1]) e = start_count() assert e - s >= 2 looper.run(eventually(checkViewNoForNodes, slow_node.viewNo)) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2 * Max3PCBatchSize) waitNodeDataEquality(looper, new_node, *nodes_slow_to_inst_chg)
def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, sent1): """ When a request has already been executed the previously executed reply will be sent again to the client. An acknowledgement will not be sent for a repeated request. """ sdk_get_and_check_replies(looper, sent1) req = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) sdk_get_and_check_replies(looper, [req])
def sdk_node_theta_added(looper, txnPoolNodeSet, tdir, tconf, sdk_pool_handle, sdk_wallet_trustee, allPluginsPath, node_config_helper_class, testNodeClass, name=None): new_steward_name = "testClientSteward" + randomString(3) new_node_name = name or "Theta" new_steward_wallet = sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, alias=new_steward_name, role=STEWARD_STRING) sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort = \ prepare_new_node_data(tconf, tdir, new_node_name, configClass=node_config_helper_class) # filling node request _, steward_did = new_steward_wallet node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name=new_node_name, clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, services=[VALIDATOR])) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request(looper, new_steward_wallet, sdk_pool_handle, node_request) # waitng for replies sdk_get_and_check_replies(looper, [request_couple]) new_node = create_and_start_new_node(looper, new_node_name, tdir, sigseed, (nodeIp, nodePort), (clientIp, clientPort), tconf, True, allPluginsPath, testNodeClass, configClass=node_config_helper_class) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) return new_steward_wallet, new_node
def read_wrapped(txn_type): op = { TXN_TYPE: txn_type, f.LEDGER_ID.nm: DOMAIN_LEDGER_ID, DATA: 1 } req = Request(identifier=did, operation=op, reqId=getTimeBasedId(), protocolVersion=CURRENT_PROTOCOL_VERSION) sdk_get_and_check_replies(looper, [sdk_sign_and_submit_req_obj( looper, sdk_pool_handle, sdk_wallet_client, req)]) return node._info_tool.info
def test_request_with_correct_version(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, request_num): _, did = sdk_wallet_client reqs_obj = sdk_random_request_objects(request_num, identifier=did, protocol_version=CURRENT_PROTOCOL_VERSION) for req_obj in reqs_obj: assert req_obj.protocolVersion == CURRENT_PROTOCOL_VERSION signed_reqs = sdk_sign_request_objects(looper, sdk_wallet_client, reqs_obj) reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs) sdk_get_and_check_replies(looper, reqs)
def test_fail_validator_info_command( sdk_pool_handle, sdk_wallet_client, looper): op = { TXN_TYPE: VALIDATOR_INFO } req_obj = sdk_gen_request(op, identifier=sdk_wallet_client[1]) req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet_client, req_obj) with pytest.raises(RequestRejectedException) as excinfo: sdk_get_and_check_replies(looper, [req], 100) assert excinfo.match("None role cannot do action with type = " + VALIDATOR_INFO)
def test_fail_pool_restart_with_steward_role( sdk_pool_handle, sdk_wallet_steward, looper): op = { TXN_TYPE: POOL_RESTART, ACTION: START, } req_obj = sdk_gen_request(op, identifier=sdk_wallet_steward[1]) req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet_steward, req_obj) with pytest.raises(RequestRejectedException) as excinfo: sdk_get_and_check_replies(looper, [req], 100) assert excinfo.match("STEWARD cannot do action with type = " + POOL_RESTART)
def test_fail_pool_restart_with_invalid_datetime( sdk_pool_handle, sdk_wallet_steward, looper): invalid_datetime = "12.05.2018 4/40" op = { TXN_TYPE: POOL_RESTART, ACTION: START, DATETIME: invalid_datetime } req_obj = sdk_gen_request(op, identifier=sdk_wallet_steward[1]) req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet_steward, req_obj) with pytest.raises(RequestNackedException) as excinfo: sdk_get_and_check_replies(looper, [req], 100) assert excinfo.match("datetime " + invalid_datetime + " is not valid")
def test_dirty_read(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Tests the case when read request comes before write request is not executed on some nodes """ slow_nodes = list(txnPoolNodeSet)[2:4] for node in slow_nodes: logger.debug("Making node {} slow".format(node)) make_node_slow(node) received_replies = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) result = received_replies[0][1]["result"] seq_no = get_seq_no(result) _, did = sdk_wallet_client req = sdk_build_get_txn_request(looper, did, seq_no) request = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client, sdk_pool_handle, req) received_replies = sdk_get_and_check_replies(looper, [request]) results = [str(get_payload_data(reply['result'][DATA])) for _, reply in received_replies] assert len(set(results)) == 1
def sdk_send_update_node(looper, sdk_submitter_wallet, sdk_pool_handle, destination, alias, node_ip, node_port, client_ip, client_port, services=[VALIDATOR], bls_key=None, key_proof=None): _, submitter_did = sdk_submitter_wallet # filling node request node_request = looper.loop.run_until_complete( prepare_node_request(submitter_did, new_node_name=alias, clientIp=client_ip, clientPort=client_port, nodeIp=node_ip, nodePort=node_port, bls_key=bls_key, destination=destination, services=services, key_proof=key_proof)) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request(looper, sdk_submitter_wallet, sdk_pool_handle, node_request) # waitng for replies reply = sdk_get_and_check_replies(looper, [request_couple])[0][1] sdk_pool_refresh(looper, sdk_pool_handle) return reply
def test_get_txn_for_invalid_seq_no(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle): _, steward_did = sdk_wallet_steward # setting incorrect data request = sdk_build_get_txn_request(looper, steward_did, INVALID_SEQ_NO) request_couple = \ sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward, sdk_pool_handle, request) with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, [request_couple]) assert 'cannot be smaller' in e._excinfo[1].args[0]
def test_get_txn_for_invalid_ledger_id(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle): _, steward_did = sdk_wallet_steward request = sdk_build_get_txn_request(looper, steward_did, 1) # setting incorrect Ledger_ID request_json = json.loads(request) request_json['operation']['ledgerId'] = INVALID_LEDGER_ID request = json.dumps(request_json) request_couple = \ sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward, sdk_pool_handle, request) with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, [request_couple]) assert 'expected one of' in e._excinfo[1].args[0]
def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet, alias=None, role=None, seed=None, dest=None, verkey=None,skipverkey=False): seed = seed or randomString(32) alias = alias or randomString(5) wh, _ = creators_wallet # filling nym request and getting steward did # if role == None, we are adding client nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(creators_wallet, seed, alias, role, dest, verkey, skipverkey)) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request(looper, creators_wallet, sdk_pool_handle, nym_request) # waitng for replies sdk_get_and_check_replies(looper, [request_couple]) return wh, new_did
def sdk_add_new_node(looper, sdk_pool_handle, steward_wallet_handle, new_node_name, tdir, tconf, allPluginsPath=None, autoStart=True, nodeClass=TestNode, do_post_node_creation: Callable = None, services=[VALIDATOR]): nodeClass = nodeClass or TestNode sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort, key_proof = \ prepare_new_node_data(tconf, tdir, new_node_name) # filling node request _, steward_did = steward_wallet_handle node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name=new_node_name, clientIp=clientIp, clientPort=clientPort, nodeIp=nodeIp, nodePort=nodePort, bls_key=bls_key, sigseed=sigseed, services=services, key_proof=key_proof)) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request(looper, steward_wallet_handle, sdk_pool_handle, node_request) # waitng for replies sdk_get_and_check_replies(looper, [request_couple]) return create_and_start_new_node(looper, new_node_name, tdir, sigseed, (nodeIp, nodePort), (clientIp, clientPort), tconf, autoStart, allPluginsPath, nodeClass, do_post_node_creation=do_post_node_creation, configClass=PNodeConfigHelper)
def replied1(looper, txnPoolNodeSet, sdk_wallet_client, committed1, faultyNodes, sent1): numOfNodes = len(txnPoolNodeSet) numOfInstances = getNoInstances(numOfNodes) quorum = numOfInstances * (numOfNodes - faultyNodes) _, did = sdk_wallet_client def checkOrderedCount(): resp = [requestReturnedToNode(node, committed1.digest, instId) for node in txnPoolNodeSet for instId in range(numOfInstances)] assert resp.count(True) >= quorum orderingTimeout = waits.expectedOrderingTime(numOfInstances) looper.run(eventually(checkOrderedCount, retryWait=1, timeout=orderingTimeout)) sdk_get_and_check_replies(looper, sent1) return committed1
def test_apply_stashed_partially_ordered(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): test_node = getNonPrimaryReplicas(txnPoolNodeSet)[0].node test_stasher = test_node.nodeIbStasher ledger_size = max(node.domainLedger.size for node in txnPoolNodeSet) def check_pool_ordered_some_requests(): assert max(node.domainLedger.size for node in txnPoolNodeSet) > ledger_size def check_test_node_has_stashed_ordered_requests(): assert len(test_node.stashedOrderedReqs) > 0 # Delay COMMITs so requests are not ordered on test node with delay_rules(test_stasher, cDelay()): reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, TOTAL_REQUESTS) looper.run(eventually(check_pool_ordered_some_requests)) # Get some of txns that need to be ordered ledger_info = test_node.ledgerManager.getLedgerInfoByType(DOMAIN_LEDGER_ID) txns = ledger_info.ledger.uncommittedTxns txns = txns[:len(txns) // 2] assert len(txns) > 1 # Emulate incomplete catchup simultaneous with generation of ORDERED message test_node.mode = Mode.syncing test_node.master_replica.revert_unordered_batches() looper.run(eventually(check_test_node_has_stashed_ordered_requests)) for txn in txns: ledger_info.ledger.add(txn) ledger_info.postTxnAddedToLedgerClbk(DOMAIN_LEDGER_ID, txn) test_node.mode = Mode.participating test_node.processStashedOrderedReqs() ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) sdk_get_and_check_replies(looper, reqs)
def sdk_change_node_keys(looper, node, sdk_wallet_steward, sdk_pool_handle, verkey): _, steward_did = sdk_wallet_steward node_dest = hexToFriendly(node.nodestack.verhex) node_request = looper.loop.run_until_complete( prepare_node_request(steward_did, new_node_name=node.name, destination=node_dest)) request_json = json.loads(node_request) request_json['operation'][VERKEY] = verkey node_request1 = json.dumps(request_json) request_couple = sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward, sdk_pool_handle, node_request1) sdk_get_and_check_replies(looper, [request_couple]) node.nodestack.clearLocalRoleKeep() node.nodestack.clearRemoteRoleKeeps() node.nodestack.clearAllDir() node.clientstack.clearLocalRoleKeep() node.clientstack.clearRemoteRoleKeeps() node.clientstack.clearAllDir()
def testDoNotBlacklistClient(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, poolTxnClientNames): """ Client should be not be blacklisted by node on sending an unsigned request """ client_name = poolTxnClientNames[0] _, did = sdk_wallet_client # No node should blacklist the client req_obj = sdk_random_request_objects(1, identifier=did, protocol_version=CURRENT_PROTOCOL_VERSION)[0] reqs = sdk_send_signed_requests(sdk_pool_handle, [json.dumps(req_obj.as_dict)]) with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, reqs) assert 'MissingSignature' in e._excinfo[1].args[0] def chk(): for node in txnPoolNodeSet: assert not node.isClientBlacklisted(client_name) timeout = waits.expectedClientToPoolConnectionTimeout(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout))
def test_get_txn_for_non_existing_seq_no(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle): _, steward_did = sdk_wallet_steward # setting incorrect data def generate_non_existing_seq_no(): return randint(500, 1000) request = sdk_build_get_txn_request(looper, steward_did, generate_non_existing_seq_no()) request_couple = \ sdk_sign_and_send_prepared_request(looper, sdk_wallet_steward, sdk_pool_handle, request) reply = sdk_get_and_check_replies(looper, [request_couple])[0][1] assert reply['result'][DATA] is None
def test_sdk_steward_send_many(looper, sdk_pool_handle, sdk_wallet_steward): resp_task = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_steward, 30) repl = sdk_get_and_check_replies(looper, resp_task) for _, resp in repl: assert resp['result']
def test_successive_batch_do_no_change_state(looper, tconf, nodeSet, sdk_pool_handle, sdk_wallet_trustee, monkeypatch): """ Send 2 NYM txns in different batches such that the second batch does not change state so that state root remains same, but keep the identifier and reqId different. Make sure the first request is not ordered by the primary before PRE-PREPARE for the second is sent. Also check reject and commit :return: """ # Disable view change during this test for n in nodeSet: n.nodeIbStasher.delay(icDelay()) # Delay only first PRE-PREPARE delay_cm_duration = 10 def delay_commits(wrappedMsg): msg, sender = wrappedMsg if isinstance(msg, Commit) and msg.instId == 0: return delay_cm_duration def check_verkey(i, vk): for node in nodeSet: data = node.idrCache.getNym(i, isCommitted=True) assert data[VERKEY] == vk def check_uncommitted(count): for node in nodeSet: assert len(node.idrCache.un_committed) == count for node in nodeSet: for rpl in node.replicas: monkeypatch.setattr(rpl, '_request_missing_three_phase_messages', lambda *x, **y: None) wh, did = sdk_wallet_trustee seed = randomString(32) (new_did, verkey) = looper.loop.run_until_complete( create_and_store_my_did(wh, json.dumps({'seed': seed}))) sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, dest=new_did, verkey=verkey) for node in nodeSet: node.nodeIbStasher.delay(delay_commits) # Setting the same verkey thrice but in different batches with different # request ids for _ in range(3): verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_did, new_did) logger.debug('{} rotates his key to {}'.format(new_did, verkey)) # Number of uncommitted entries is 0 looper.run(eventually(check_uncommitted, 0)) check_verkey(new_did, verkey) # Setting the verkey to `x`, then `y` and then back to `x` but in different # batches with different request ids. The idea is to change # state root to `t` then `t'` and then back to `t` and observe that no # errors are encountered seed = randomString(32) (new_client_did, verkey) = looper.loop.run_until_complete( create_and_store_my_did(wh, json.dumps({'seed': seed}))) sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_trustee, dest=new_client_did, verkey=verkey) x_seed = randomString(32) verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did, new_client_did) logger.debug('{} rotates his key to {}'.format(new_client_did, verkey)) y_seed = randomString(32) sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did, new_client_did) logger.debug('{} rotates his key to {}'.format(new_client_did, verkey)) verkey = sdk_rotate_verkey(looper, sdk_pool_handle, wh, new_client_did, new_client_did) logger.debug('{} rotates his key to {}'.format(new_client_did, verkey)) # Number of uncommitted entries is 0 looper.run(eventually(check_uncommitted, 0)) check_verkey(new_client_did, verkey) monkeypatch.undo() # Delay COMMITs so that IdrCache can be checked for correct # number of entries uncommitteds = {} methods = {} for node in nodeSet: cache = node.idrCache # type: IdrCache uncommitteds[cache._name] = [] # Since the post batch creation handler is registered (added to a list), # find it and patch it dh = node.get_req_handler(DOMAIN_LEDGER_ID) for i, handler in enumerate(dh.post_batch_creation_handlers): # Find the cache's post create handler, not hardcoding names of # class or functions as they can change with refactoring. if handler.__func__.__qualname__ == '{}.{}'.format( cache.__class__.__name__, cache.currentBatchCreated.__name__): cre = dh.post_batch_creation_handlers[i] break com = cache.onBatchCommitted methods[cache._name] = (cre, com) # Patch methods to record and check roots after commit def patched_cre(self, stateRoot): uncommitteds[self._name].append(stateRoot) return methods[self._name][0](stateRoot) def patched_com(self, stateRoot): assert uncommitteds[self._name][0] == stateRoot rv = methods[self._name][1](stateRoot) uncommitteds[self._name] = uncommitteds[self._name][1:] return rv dh.post_batch_creation_handlers[i] = types.MethodType( patched_cre, cache) cache.onBatchCommitted = types.MethodType(patched_com, cache) # Set verkey of multiple identities more = 5 keys = {} reqs = [] for _ in range(more): seed = randomString(32) nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_trustee, seed, None, None)) keys[new_did] = json.loads(nym_request)['operation']['verkey'] reqs.append( sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee, sdk_pool_handle, nym_request)) looper.runFor(tconf.Max3PCBatchWait + 0.1) # Correct number of uncommitted entries looper.run(eventually(check_uncommitted, more, retryWait=1)) sdk_get_and_check_replies(looper, reqs) # Number of uncommitted entries is 0 looper.run(eventually(check_uncommitted, 0)) # The verkeys are correct for i, v in keys.items(): check_verkey(i, v) waitNodeDataEquality(looper, nodeSet[0], *nodeSet[1:]) for _ in range(3): seed = randomString(32) nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(sdk_wallet_trustee, seed, None, None)) reqs.append( sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee, sdk_pool_handle, nym_request)) looper.runFor(tconf.Max3PCBatchWait + 0.1) # Correct number of uncommitted entries looper.run(eventually(check_uncommitted, 3, retryWait=1)) # Check batch reject for node in nodeSet: cache = node.idrCache initial = cache.un_committed cache.batchRejected() # After reject, last entry is removed assert cache.un_committed == initial[:-1] root = cache.un_committed[0][0] cache.onBatchCommitted(root) # Calling commit with same root results in Assertion error with pytest.raises(AssertionError): cache.onBatchCommitted(root)
def test_checkpoints_removed_in_view_change(chkFreqPatched, txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): ''' Check that checkpoint finalize in view change before catchup doesn't clean necessary data from requests and 3pc queues. ''' slow_nodes = txnPoolNodeSet[:3] fast_nodes = txnPoolNodeSet[3:] # delay checkpoints processing for slow_nodes delay_msg(slow_nodes, chk_delay) # send txns for finalizing current checkpoint sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, CHK_FREQ) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # delay commits processing for slow_nodes delay_msg(slow_nodes, cDelay) requests = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 1) # check that slow nodes have prepared certificate with new txn looper.run(eventually(last_prepared_certificate, slow_nodes, (0, CHK_FREQ + 1))) # check that fast_nodes ordered new txn looper.run(eventually(last_ordered_check, fast_nodes, (0, CHK_FREQ + 1))) # check that fast_nodes finalized first checkpoint and slow_nodes are not looper.run(eventually(check_checkpoint_finalize, fast_nodes, 1, CHK_FREQ)) for n in slow_nodes: assert not n.master_replica._checkpointer._checkpoint_state[(1, CHK_FREQ)].isStable # View change start emulation for change viewNo and fix last prepare # certificate, because if we start a real view change then checkpoints will # clean and the first checkpoint would not be need in finalizing. for node in txnPoolNodeSet: node.viewNo += 1 node.master_replica.on_view_change_start() # reset delay for checkpoints reset_delay(slow_nodes, CHECKPOINT) # reset view change emulation and start real view change for finish it in # a normal mode with catchup for node in txnPoolNodeSet: node.viewNo -= 1 ensure_view_change(looper, txnPoolNodeSet) for n in slow_nodes: assert not n.master_replica._checkpointer._checkpoint_state[(1, CHK_FREQ)].isStable # Check ordering the last txn before catchup. Check client reply is enough # because slow_nodes contains 3 nodes and without their replies sdk method # for get reply will not successfully finish. reset_delay(slow_nodes, COMMIT) sdk_get_and_check_replies(looper, requests) looper.run(eventually(last_ordered_check, txnPoolNodeSet, (0, CHK_FREQ + 1))) # check view change finish and checkpoints were cleaned ensureElectionsDone(looper, txnPoolNodeSet) for n in slow_nodes: assert (1, CHK_FREQ) not in n.master_replica._checkpointer._checkpoint_state # check that all nodes have same data after new txns ordering sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, CHK_FREQ) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)
def sdk_get_frozen_ledgers(looper, sdk_pool_handle, sdk_wallet): req = build_get_frozen_ledgers_request(sdk_wallet[1]) rep = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet, req) return sdk_get_and_check_replies(looper, [rep])[0]
def setup_aml(looper, txnPoolNodeSet, taa_aml_request_module, sdk_pool_handle, sdk_wallet_trustee): req = sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee, sdk_pool_handle, taa_aml_request_module) sdk_get_and_check_replies(looper, [req])
def test_primary_send_incorrect_pp(looper, txnPoolNodeSet, tconf, allPluginsPath, sdk_pool_handle, sdk_wallet_steward, monkeypatch): """ Test steps: Delay message requests with PrePrepares on `slow_node` Patch sending for PrePrepare on the `malicious_primary` to send an invalid PrePrepare to slow_node Order a new request Start a view change Make sure it's finished on all nodes Make sure that the lagging node has same data with other nodes """ start_view_no = txnPoolNodeSet[0].viewNo slow_node = txnPoolNodeSet[-1] malicious_primary = txnPoolNodeSet[0] other_nodes = [ n for n in txnPoolNodeSet if n not in [slow_node, malicious_primary] ] timeout = waits.expectedPoolCatchupTime(nodeCount=len(txnPoolNodeSet)) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet, custom_timeout=timeout) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1) old_sender = malicious_primary.master_replica._ordering_service._send def patched_sender(msg, dst=None, stat=None): if isinstance(msg, PrePrepare) and msg: old_sender(msg, [n.name for n in other_nodes], stat) pp_dict = msg._asdict() pp_dict["ppTime"] += 1 pp = PrePrepare(**pp_dict) old_sender(pp, [slow_node.name], stat) monkeypatch.undo() monkeypatch.setattr(malicious_primary.master_replica._ordering_service, '_send', patched_sender) monkeypatch.setattr(slow_node.master_replica._ordering_service, '_validate_applied_pre_prepare', lambda a, b, c: None) with delay_rules(slow_node.nodeIbStasher, msg_rep_delay(types_to_delay=[PREPREPARE])): preprepare_process_num = slow_node.master_replica._ordering_service.spylog.count( OrderingService.process_preprepare) resp_task = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_steward) def chk(): assert preprepare_process_num + 1 == slow_node.master_replica._ordering_service.spylog.count( OrderingService.process_preprepare) looper.run(eventually(chk)) _, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0] sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 1) for n in txnPoolNodeSet: n.view_changer.on_master_degradation() ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=start_view_no + 1) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, instances_list=[0, 1]) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle)
def test_chain_set_fees_and_xfer_batch_size_2(looper, helpers, nodeSetWithIntegratedTokenPlugin, sdk_pool_handle, sdk_wallet_trustee, mint_tokens, addresses, poolConfigWTFF): """ Set FEES for XFER for 2 Send any transaction to config ledger. Send XFER with fees 2 from A to B Set FEES for XFER for 3 Send any transaction to config ledger. Send XFER with fees 3 from A to B Check that first XFER is not written and second XFER is. """ A, B = addresses current_amount = get_amount_from_token_txn(mint_tokens) seq_no = get_seq_no(mint_tokens) transfer_summ = 20 # Set fees and some config txn helpers.node.set_fees_directly({XFER_PUBLIC_FEES_ALIAS: 42}) fees_xfer_2 = {XFER_PUBLIC_FEES_ALIAS: 2} fees_2_rsp = helpers.general.set_fees_without_waiting(fees_xfer_2) sdk_pool_config_sent(looper, sdk_pool_handle, sdk_wallet_trustee, poolConfigWTFF) sdk_get_and_check_replies(looper, fees_2_rsp) # XFER with fees 2 from A to B _, _, a_b_transfer_2 = send_and_check_transfer(helpers, [A, B], fees_xfer_2, looper, current_amount, seq_no, transfer_summ=transfer_summ, check_reply=False) # Set fees for XFER to 3 fees_xfer_3 = {XFER_PUBLIC_FEES_ALIAS: 3} fees_3_rsp = helpers.general.set_fees_without_waiting(fees_xfer_3) sdk_pool_config_sent(looper, sdk_pool_handle, sdk_wallet_trustee, poolConfigWTFF) sdk_get_and_check_replies(looper, fees_3_rsp) # Send XFER with fees from A to B a_amount, seq_no, a_b_transfer_3 = send_and_check_transfer( helpers, [A, B], fees_xfer_3, looper, current_amount, seq_no, transfer_summ=transfer_summ, check_reply=False) for n in nodeSetWithIntegratedTokenPlugin: fee_rq = n.read_manager.request_handlers[ FeesTransactions.GET_FEES.value] assert fee_rq assert fee_rq.get_fees(is_committed=True, with_proof=False) == fees_xfer_3 with pytest.raises(RequestRejectedException): sdk_get_and_check_replies(looper, a_b_transfer_2) sdk_get_and_check_replies(looper, a_b_transfer_3) a_get = helpers.general.do_get_utxo(A) assert a_get[OUTPUTS][1][AMOUNT] == a_amount assert a_get[OUTPUTS][1][SEQNO] == seq_no b_get = helpers.general.do_get_utxo(B) assert b_get[OUTPUTS][0][AMOUNT] == transfer_summ assert b_get[OUTPUTS][0][SEQNO] == seq_no ensure_all_nodes_have_same_data(looper, nodeSetWithIntegratedTokenPlugin)
def sdk_send_freeze_ledgers(looper, sdk_pool_handle, sdk_wallets, ledgers_ids: [int]): req = build_freeze_ledgers_request(sdk_wallets[0][1], ledgers_ids) signed_reqs = sdk_multi_sign_request_objects(looper, sdk_wallets, [req]) reps = sdk_send_signed_requests(sdk_pool_handle, signed_reqs) return sdk_get_and_check_replies(looper, reps)[0]
def test_sdk_new_steward_send(looper, sdk_pool_handle, sdk_wallet_new_steward): resp_task = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_new_steward) _, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0] assert j_resp['result']
def prepare(self): self.default_auth_rule = self.get_default_auth_rule() self.changed_auth_rule = self.get_changed_auth_rule() req = self.taa_aml_request() rep = sdk_sign_and_send_prepared_request(self.looper, self.trustee_wallet, self.sdk_pool_handle, req) sdk_get_and_check_replies(self.looper, [rep])
def sdk_send_txn_author_agreement(looper, sdk_pool_handle, sdk_wallet, text: str, version: str): req = looper.loop.run_until_complete( build_txn_author_agreement_request(sdk_wallet[1], text, version)) rep = sdk_sign_and_submit_req(sdk_pool_handle, sdk_wallet, req) return sdk_get_and_check_replies(looper, [rep])[0]
def successful_op(looper, op, sdk_wallet, sdk_pool_handle): req_obj = sdk_gen_request(op, identifier=sdk_wallet[1]) req = sdk_sign_and_submit_req_obj(looper, sdk_pool_handle, sdk_wallet, req_obj) return sdk_get_and_check_replies(looper, [req])
def sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgrade_data): req = sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_trustee, upgrade_data) sdk_get_and_check_replies(looper, [req])