def __init__(self, tmpdir, config=None):
self.basedirpath = tmpdir
self.name = 'Node1'
self.f = 1
self.replicas = dict()
self.requests = []
self.rank = None
self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4']
self.nodeReg = {
name: HA("127.0.0.1", 0) for name in self.allNodeNames
}
self.totalNodes = len(self.allNodeNames)
self.mode = Mode.starting
self.config = config or getConfigOnce()
self.replicas = {
0: Replica(node=self, instId=0, isMaster=True, config=self.config),
1: Replica(node=self, instId=1, isMaster=False, config=self.config),
2: Replica(node=self, instId=2, isMaster=False, config=self.config),
}
self._found = False
self.ledgerManager = LedgerManager(self, ownedByNode=True)
ledger0 = FakeLedger(0, 10)
ledger1 = FakeLedger(1, 5)
self.ledgerManager.addLedger(0, ledger0)
self.ledgerManager.addLedger(1, ledger1)
self.quorums = Quorums(self.totalNodes)
self.view_changer = ViewChanger(self)
self.elector = PrimarySelector(self)
self.metrics = NullMetricsCollector()
def test_replay_on_new_node(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir,
testNodeClass, tmpdir_factory,
node_config_helper_class, allPluginsPath,
sdk_new_node_caught_up): # noqa: F811
new_node = sdk_new_node_caught_up
for node in txnPoolNodeSet:
txnPoolNodesLooper.removeProdable(node)
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
for node in txnPoolNodeSet:
node.stop()
print('-------------Replaying now---------------------')
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
new_node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
def test_view_change_after_some_txns(txnPoolNodesLooper, txnPoolNodeSet,
some_txns_done, testNodeClass, viewNo, # noqa
sdk_pool_handle, sdk_wallet_client,
node_config_helper_class, tconf, tdir,
allPluginsPath, tmpdir_factory):
"""
Check that view change is done after processing some of txns
"""
ensure_view_change(txnPoolNodesLooper, txnPoolNodeSet)
ensureElectionsDone(looper=txnPoolNodesLooper, nodes=txnPoolNodeSet)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, nodes=txnPoolNodeSet)
sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 10)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet)
for node in txnPoolNodeSet:
txnPoolNodesLooper.removeProdable(node)
node.stop()
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
print('-------------Replaying now---------------------')
for node in txnPoolNodeSet:
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
def test_replay_recorded_msgs(txnPoolNodesLooper,
txnPoolNodeSet, some_txns_done, testNodeClass,
node_config_helper_class, tconf, tdir,
allPluginsPath, tmpdir_factory):
# Run a pool of nodes with each having a recorder.
# After some txns, record state, replay each node's recorder on a
# clean node and check that state matches the initial state
ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet)
for node in txnPoolNodeSet:
txnPoolNodesLooper.removeProdable(node)
for node in txnPoolNodeSet:
node.stop()
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
print('-------------Replaying now---------------------')
for node in txnPoolNodeSet:
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
def setup_plugins():
# TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field
global PLUGIN_LEDGER_IDS
global PLUGIN_CLIENT_REQUEST_FIELDS
config = getConfigOnce()
plugin_root = config.PLUGIN_ROOT
try:
plugin_root = importlib.import_module(plugin_root)
except ImportError:
raise ImportError('Incorrect plugin root {}. No such package found'.
format(plugin_root))
sys.path.insert(0, plugin_root.__path__[0])
enabled_plugins = config.ENABLED_PLUGINS
installed_packages = {p.project_name: p for p in pip.get_installed_distributions()}
for plugin_name in enabled_plugins:
plugin = find_and_load_plugin(plugin_name, plugin_root, installed_packages)
plugin_globals = plugin.__dict__
if 'LEDGER_IDS' in plugin_globals:
PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS'])
if 'CLIENT_REQUEST_FIELDS' in plugin_globals:
PLUGIN_CLIENT_REQUEST_FIELDS.update(plugin_globals['CLIENT_REQUEST_FIELDS'])
# Reloading message types since some some schemas would have been changed
import plenum.common.messages.fields
import plenum.common.messages.node_messages
import plenum.common.messages.node_message_factory
importlib.reload(plenum.common.messages.node_messages)
importlib.reload(plenum.common.messages.node_message_factory)
def setup_plugins():
# TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field
global PLUGIN_LEDGER_IDS
global PLUGIN_CLIENT_REQUEST_FIELDS
config = getConfigOnce()
plugin_root = config.PLUGIN_ROOT
try:
plugin_root = importlib.import_module(plugin_root)
except ImportError:
raise ImportError(
'Incorrect plugin root {}. No such package found'.format(
plugin_root))
enabled_plugins = config.ENABLED_PLUGINS
for plugin_name in enabled_plugins:
plugin_path = os.path.join(plugin_root.__path__[0], plugin_name,
'__init__.py')
spec = spec_from_file_location('__init__.py', plugin_path)
init = module_from_spec(spec)
spec.loader.exec_module(init)
plugin_globals = init.__dict__
if 'LEDGER_IDS' in plugin_globals:
PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS'])
if 'CLIENT_REQUEST_FIELDS' in plugin_globals:
PLUGIN_CLIENT_REQUEST_FIELDS.update(
plugin_globals['CLIENT_REQUEST_FIELDS'])
# Reloading message types since some some schemas would have been changed
import plenum.common.messages.node_messages
import plenum.common.messages.node_message_factory
importlib.reload(plenum.common.messages.node_messages)
importlib.reload(plenum.common.messages.node_message_factory)
def test_replay_recorded_msgs(txnPoolNodesLooper, txnPoolNodeSet,
some_txns_done, testNodeClass,
node_config_helper_class, tconf, tdir,
allPluginsPath, tmpdir_factory):
# Run a pool of nodes with each having a recorder.
# After some txns, record state, replay each node's recorder on a
# clean node and check that state matches the initial state
ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet)
for node in txnPoolNodeSet:
txnPoolNodesLooper.removeProdable(node)
for node in txnPoolNodeSet:
node.stop()
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
print('-------------Replaying now---------------------')
for node in txnPoolNodeSet:
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
def setup_plugins():
import os # noqa
import pip # noqa
import importlib # noqa
from importlib.util import module_from_spec, spec_from_file_location # noqa: E402
import plenum # noqa: E402
import plenum.server.plugin # noqa: E402
from plenum.common.config_util import getConfigOnce # noqa: E402
def find_and_load_plugin(plugin_name, plugin_root, installed_packages):
if plugin_name in installed_packages:
# TODO: Need a test for installed packages
plugin_name = plugin_name.replace('-', '_')
plugin = importlib.import_module(plugin_name)
else:
plugin_path = os.path.join(plugin_root.__path__[0], plugin_name,
'__init__.py')
spec = spec_from_file_location('__init__.py', plugin_path)
plugin = module_from_spec(spec)
spec.loader.exec_module(plugin)
return plugin
# TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field
global PLUGIN_LEDGER_IDS
global PLUGIN_CLIENT_REQUEST_FIELDS
config = getConfigOnce()
plugin_root = config.PLUGIN_ROOT
try:
plugin_root = importlib.import_module(plugin_root)
except ImportError:
raise ImportError(
'Incorrect plugin root {}. No such package found'.format(
plugin_root))
sys.path.insert(0, plugin_root.__path__[0])
enabled_plugins = config.ENABLED_PLUGINS
installed_packages = {
p.project_name: p
for p in pip.get_installed_distributions()
}
for plugin_name in enabled_plugins:
plugin = find_and_load_plugin(plugin_name, plugin_root,
installed_packages)
plugin_globals = plugin.__dict__
if 'LEDGER_IDS' in plugin_globals:
PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS'])
if 'CLIENT_REQUEST_FIELDS' in plugin_globals:
PLUGIN_CLIENT_REQUEST_FIELDS.update(
plugin_globals['CLIENT_REQUEST_FIELDS'])
# Reloading message types since some some schemas would have been changed
import plenum.common.messages.fields
import plenum.common.messages.node_messages
import plenum.common.messages.node_message_factory
importlib.reload(plenum.common.messages.node_messages)
importlib.reload(plenum.common.messages.node_message_factory)
def __init__(self, tmpdir, config=None):
node_names = ['Node1', 'Node2', 'Node3', 'Node4']
self.basedirpath = tmpdir
self.name = node_names[0]
self.viewNo = 0
self.db_manager = DatabaseManager()
self.timer = QueueTimer()
self.f = 1
self.replicas = dict()
self.requests = Requests()
self.rank = None
self.allNodeNames = node_names
self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames}
self.nodeIds = []
self.totalNodes = len(self.allNodeNames)
self.poolManager = FakeSomething(
node_names_ordered_by_rank=lambda: node_names)
self.mode = Mode.starting
self.monitor = FakeSomething(isMasterDegraded=lambda: False)
self.config = config or getConfigOnce()
self.nodeStatusDB = None
self.quorums = Quorums(self.totalNodes)
self.nodestack = FakeSomething(connecteds=set(self.allNodeNames))
self.write_manager = FakeSomething(
node_reg_handler=NodeRegHandler(self.db_manager))
self.primaries_selector = RoundRobinConstantNodesPrimariesSelector(
node_names)
self.replicas = {
0: Replica(node=self, instId=0, isMaster=True, config=self.config),
1: Replica(node=self, instId=1, isMaster=False,
config=self.config),
2: Replica(node=self, instId=2, isMaster=False, config=self.config)
}
self.requiredNumberOfInstances = 2
self._found = False
self.ledgerManager = LedgerManager(self)
ledger0 = FakeLedger(0, 10)
ledger1 = FakeLedger(1, 5)
self.ledgerManager.addLedger(0, ledger0)
self.ledgerManager.addLedger(1, ledger1)
self.quorums = Quorums(self.totalNodes)
self.metrics = NullMetricsCollector()
# For catchup testing
self.view_change_in_progress = False
self.ledgerManager.last_caught_up_3PC = (0, 0)
self.master_last_ordered_3PC = (0, 0)
self.seqNoDB = {}
# callbacks
self.onBatchCreated = lambda self, *args, **kwargs: True
def setup_plugins():
global PLUGIN_LEDGER_IDS
config = getConfigOnce()
ENABLED_PLUGINS = config.ENABLED_PLUGINS
for plugin_name in ENABLED_PLUGINS:
plugin_path = os.path.join(plenum.server.plugin.__path__[0],
plugin_name, '__init__.py')
spec = spec_from_file_location('__init__.py', plugin_path)
init = module_from_spec(spec)
spec.loader.exec_module(init)
plugin_globals = init.__dict__
if 'LEDGER_IDS' in plugin_globals:
PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS'])
if 'CLIENT_REQUEST_FIELDS' in plugin_globals:
PLUGIN_CLIENT_REQUEST_FIELDS.update(
plugin_globals['CLIENT_REQUEST_FIELDS'])
def __init__(self, tmpdir, config=None):
self.basedirpath = tmpdir
self.name = 'Node1'
self.internal_bus = InternalBus()
self.db_manager = DatabaseManager()
self.timer = QueueTimer()
self.f = 1
self.replicas = dict()
self.requests = Requests()
self.rank = None
self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4']
self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames}
self.nodeIds = []
self.totalNodes = len(self.allNodeNames)
self.mode = Mode.starting
self.config = config or getConfigOnce()
self.nodeStatusDB = None
self.replicas = {
0: Replica(node=self, instId=0, isMaster=True, config=self.config),
1: Replica(node=self, instId=1, isMaster=False,
config=self.config),
2: Replica(node=self, instId=2, isMaster=False,
config=self.config),
}
self._found = False
self.ledgerManager = LedgerManager(self)
ledger0 = FakeLedger(0, 10)
ledger1 = FakeLedger(1, 5)
self.ledgerManager.addLedger(0, ledger0)
self.ledgerManager.addLedger(1, ledger1)
self.quorums = Quorums(self.totalNodes)
self.view_changer = create_view_changer(self)
self.elector = PrimarySelector(self)
self.metrics = NullMetricsCollector()
# For catchup testing
self.catchup_rounds_without_txns = 0
self.view_change_in_progress = False
self.ledgerManager.last_caught_up_3PC = (0, 0)
self.master_last_ordered_3PC = (0, 0)
self.seqNoDB = {}
# callbacks
self.onBatchCreated = lambda self, *args, **kwargs: True
def test_replay_on_new_node(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir,
testNodeClass, tmpdir_factory,
node_config_helper_class, allPluginsPath,
sdk_new_node_caught_up): # noqa: F811
new_node = sdk_new_node_caught_up
for node in txnPoolNodeSet:
txnPoolNodesLooper.removeProdable(node)
node.stop()
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
print('-------------Replaying now---------------------')
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
new_node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
def setup_plugins():
# TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field
global PLUGIN_LEDGER_IDS
global PLUGIN_CLIENT_REQUEST_FIELDS
config = getConfigOnce()
plugin_root = config.PLUGIN_ROOT
try:
plugin_root = importlib.import_module(plugin_root)
except ImportError:
raise ImportError(
'Incorrect plugin root {}. No such package found'.format(
plugin_root))
sys.path.insert(0, plugin_root.__path__[0])
enabled_plugins = config.ENABLED_PLUGINS
installed_packages = {
p.project_name: p
for p in pip.get_installed_distributions()
}
for plugin_name in enabled_plugins:
plugin = find_and_load_plugin(plugin_name, plugin_root,
installed_packages)
plugin_globals = plugin.__dict__
if 'LEDGER_IDS' in plugin_globals:
PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS'])
if 'CLIENT_REQUEST_FIELDS' in plugin_globals:
PLUGIN_CLIENT_REQUEST_FIELDS.update(
plugin_globals['CLIENT_REQUEST_FIELDS'])
# Reloading message types since some some schemas would have been changed
import plenum.common.messages.fields
import plenum.common.messages.node_messages
import plenum.common.messages.node_message_factory
importlib.reload(plenum.common.messages.node_messages)
importlib.reload(plenum.common.messages.node_message_factory)
def __init__(self, tmpdir, config=None):
self.basedirpath = tmpdir
self.name = 'Node1'
self.f = 1
self.replicas = dict()
self.requests = Requests()
self.rank = None
self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4']
self.nodeReg = {
name: HA("127.0.0.1", 0) for name in self.allNodeNames
}
self.totalNodes = len(self.allNodeNames)
self.mode = Mode.starting
self.config = config or getConfigOnce()
self.replicas = {
0: Replica(node=self, instId=0, isMaster=True, config=self.config),
1: Replica(node=self, instId=1, isMaster=False, config=self.config),
2: Replica(node=self, instId=2, isMaster=False, config=self.config),
}
self._found = False
self.ledgerManager = LedgerManager(self, ownedByNode=True)
ledger0 = FakeLedger(0, 10)
ledger1 = FakeLedger(1, 5)
self.ledgerManager.addLedger(0, ledger0)
self.ledgerManager.addLedger(1, ledger1)
self.quorums = Quorums(self.totalNodes)
self.view_changer = ViewChanger(self)
self.elector = PrimarySelector(self)
self.metrics = NullMetricsCollector()
# For catchup testing
self.catchup_rounds_without_txns = 0
self.view_change_in_progress = False
self.ledgerManager.last_caught_up_3PC = (0, 0)
self.master_last_ordered_3PC = (0, 0)
self.seqNoDB = {}
def test_replay_new_bouncing(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir,
testNodeClass, tmpdir_factory,
node_config_helper_class, allPluginsPath,
some_txns_done, sdk_pool_handle,
sdk_wallet_client):
alpha = txnPoolNodeSet[0]
old_view_no = alpha.viewNo
other_nodes = txnPoolNodeSet[1:]
stopping_at = time.perf_counter()
alpha.stop()
txnPoolNodesLooper.removeProdable(alpha)
txnPoolNodesLooper.run(
eventually(checkViewNoForNodes,
other_nodes,
old_view_no + 1,
retryWait=1,
timeout=30))
sdk_send_random_and_check(txnPoolNodesLooper, other_nodes, sdk_pool_handle,
sdk_wallet_client, 10)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, other_nodes)
for node in other_nodes:
assert alpha.name not in node.nodestack.connecteds
nodeHa, nodeCHa = HA(*alpha.nodestack.ha), HA(*alpha.clientstack.ha)
config_helper = PNodeConfigHelper(alpha.name, tconf, chroot=tdir)
restarted_alpha = testNodeClass(alpha.name,
config_helper=config_helper,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
txnPoolNodesLooper.add(restarted_alpha)
txnPoolNodeSet[0] = restarted_alpha
restarting_at = time.perf_counter()
print('Stopped for {}'.format(restarting_at - stopping_at))
sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 10)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet)
for node in txnPoolNodeSet:
node.stop()
txnPoolNodesLooper.removeProdable(node)
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
print('-------------Replaying now---------------------')
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
restarted_alpha, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
# For disconnections
for node in other_nodes:
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
def test_replay_new_bouncing(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir,
testNodeClass, tmpdir_factory,
node_config_helper_class, allPluginsPath,
some_txns_done, sdk_pool_handle, sdk_wallet_client):
alpha = txnPoolNodeSet[0]
old_view_no = alpha.viewNo
other_nodes = txnPoolNodeSet[1:]
stopping_at = time.perf_counter()
alpha.stop()
txnPoolNodesLooper.removeProdable(alpha)
txnPoolNodesLooper.run(eventually(checkViewNoForNodes, other_nodes,
old_view_no + 1, retryWait=1, timeout=30))
sdk_send_random_and_check(txnPoolNodesLooper, other_nodes,
sdk_pool_handle,
sdk_wallet_client, 10)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, other_nodes)
for node in other_nodes:
assert alpha.name not in node.nodestack.connecteds
nodeHa, nodeCHa = HA(*alpha.nodestack.ha), HA(*alpha.clientstack.ha)
config_helper = PNodeConfigHelper(alpha.name, tconf, chroot=tdir)
restarted_alpha = testNodeClass(
alpha.name,
config_helper=config_helper,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
txnPoolNodesLooper.add(restarted_alpha)
txnPoolNodeSet[0] = restarted_alpha
restarting_at = time.perf_counter()
print('Stopped for {}'.format(restarting_at - stopping_at))
sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client, 10)
ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet)
for node in txnPoolNodeSet:
node.stop()
txnPoolNodesLooper.removeProdable(node)
config = getConfigOnce()
reload_modules_for_replay(tconf)
replayable_node_class, basedirpath = get_replayable_node_class(
tmpdir_factory, tdir, testNodeClass, config)
print('-------------Replaying now---------------------')
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
restarted_alpha, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
# For disconnections
for node in other_nodes:
create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet,
node, replayable_node_class,
node_config_helper_class, tconf,
basedirpath, allPluginsPath)
