def test_removed_replica_restored_on_view_change( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf, tdir, allPluginsPath, chkFreqPatched, view_change): """ 1. Remove replica on some node which is not master primary 2. Reconnect the node which was master primary so far 3. Check that nodes and replicas correctly added """ ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) node = get_last_master_non_primary_node(txnPoolNodeSet) start_replicas_count = node.replicas.num_replicas instance_id = start_replicas_count - 1 node.replicas.remove_replica(instance_id) check_replica_removed(node, start_replicas_count, instance_id) # trigger view change on all nodes master_primary = get_master_primary_node(txnPoolNodeSet) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, master_primary) txnPoolNodeSet.remove(master_primary) looper.removeProdable(master_primary) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) restarted_node = start_stopped_node(master_primary, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet.append(restarted_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1, customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) assert start_replicas_count == node.replicas.num_replicas
def test_removed_replica_restored_on_view_change( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, tconf, tdir, allPluginsPath, chkFreqPatched, view_change): """ 1. Remove replica on some node which is not master primary 2. Reconnect the node which was master primary so far 3. Check that nodes and replicas correctly added """ ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) node = get_last_master_non_primary_node(txnPoolNodeSet) start_replicas_count = node.replicas.num_replicas instance_id = start_replicas_count - 1 node.replicas.remove_replica(instance_id) check_replica_removed(node, start_replicas_count, instance_id) # trigger view change on all nodes master_primary = get_master_primary_node(txnPoolNodeSet) disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, master_primary) txnPoolNodeSet.remove(master_primary) looper.removeProdable(master_primary) looper.runFor(tconf.ToleratePrimaryDisconnection + 2) restarted_node = start_stopped_node(master_primary, looper, tconf, tdir, allPluginsPath) txnPoolNodeSet.append(restarted_node) looper.run(checkNodesConnected(txnPoolNodeSet)) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=1, customTimeout=2 * tconf.NEW_VIEW_TIMEOUT) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) assert start_replicas_count == node.replicas.num_replicas ensure_all_nodes_have_same_data(looper, txnPoolNodeSet)
def testQueueingReqFromFutureView(delayed_perf_chk, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT) that come from a view which is greater than the current view. - Delay reception and processing of view change messages by a non primary for master instance => it starts receiving 3 phase commit messages for next view """ lagging_node = get_last_master_non_primary_node(txnPoolNodeSet) old_view_no = lagging_node.viewNo # Delay processing of InstanceChange and ViewChangeDone so node stashes # 3PC messages delay_ic = 60 lagging_node.nodeIbStasher.delay(icDelay(delay_ic)) lagging_node.nodeIbStasher.delay(vcd_delay(delay_ic)) logger.debug('{} will delay its view change'.format(lagging_node)) def chk_fut_view(view_no, is_empty): length = len(lagging_node.msgsForFutureViews.get(view_no, ())) if is_empty: assert length == 0 else: assert length > 0 return length # No messages queued for future view chk_fut_view(old_view_no + 1, is_empty=True) logger.debug( '{} does not have any messages for future views'.format(lagging_node)) # Every node except Node A should do a view change ensure_view_change(looper, [n for n in txnPoolNodeSet if n != lagging_node], [lagging_node]) # send more requests that will be queued for the lagged node reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 5) l = looper.run( eventually(chk_fut_view, old_view_no + 1, False, retryWait=1)) logger.debug('{} has {} messages for future views'.format(lagging_node, l)) sdk_get_replies(looper, reqs) # reset delays for the lagging_node node so that it finally makes view # change lagging_node.reset_delays_and_process_delayeds() # Eventually no messages queued for future view looper.run( eventually(chk_fut_view, old_view_no + 1, True, retryWait=1, timeout=delay_ic + 10)) logger.debug( '{} exhausted pending messages for future views'.format(lagging_node)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2)
def slow_nodes(node_set): return [ get_first_master_non_primary_node(node_set), get_last_master_non_primary_node(node_set) ]
def slow_nodes(node_set): return [get_first_master_non_primary_node(node_set), get_last_master_non_primary_node(node_set)]