def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = dict() self.requests = [] self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector()
def test_replay_on_new_node(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir, testNodeClass, tmpdir_factory, node_config_helper_class, allPluginsPath, sdk_new_node_caught_up): # noqa: F811 new_node = sdk_new_node_caught_up for node in txnPoolNodeSet: txnPoolNodesLooper.removeProdable(node) config = getConfigOnce() reload_modules_for_replay(tconf) replayable_node_class, basedirpath = get_replayable_node_class( tmpdir_factory, tdir, testNodeClass, config) for node in txnPoolNodeSet: node.stop() print('-------------Replaying now---------------------') create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, new_node, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath)
def test_view_change_after_some_txns(txnPoolNodesLooper, txnPoolNodeSet, some_txns_done, testNodeClass, viewNo, # noqa sdk_pool_handle, sdk_wallet_client, node_config_helper_class, tconf, tdir, allPluginsPath, tmpdir_factory): """ Check that view change is done after processing some of txns """ ensure_view_change(txnPoolNodesLooper, txnPoolNodeSet) ensureElectionsDone(looper=txnPoolNodesLooper, nodes=txnPoolNodeSet) ensure_all_nodes_have_same_data(txnPoolNodesLooper, nodes=txnPoolNodeSet) sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 10) ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet) for node in txnPoolNodeSet: txnPoolNodesLooper.removeProdable(node) node.stop() config = getConfigOnce() reload_modules_for_replay(tconf) replayable_node_class, basedirpath = get_replayable_node_class( tmpdir_factory, tdir, testNodeClass, config) print('-------------Replaying now---------------------') for node in txnPoolNodeSet: create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, node, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath)
def test_replay_recorded_msgs(txnPoolNodesLooper, txnPoolNodeSet, some_txns_done, testNodeClass, node_config_helper_class, tconf, tdir, allPluginsPath, tmpdir_factory): # Run a pool of nodes with each having a recorder. # After some txns, record state, replay each node's recorder on a # clean node and check that state matches the initial state ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet) for node in txnPoolNodeSet: txnPoolNodesLooper.removeProdable(node) for node in txnPoolNodeSet: node.stop() config = getConfigOnce() reload_modules_for_replay(tconf) replayable_node_class, basedirpath = get_replayable_node_class( tmpdir_factory, tdir, testNodeClass, config) print('-------------Replaying now---------------------') for node in txnPoolNodeSet: create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, node, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath)
def setup_plugins(): # TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field global PLUGIN_LEDGER_IDS global PLUGIN_CLIENT_REQUEST_FIELDS config = getConfigOnce() plugin_root = config.PLUGIN_ROOT try: plugin_root = importlib.import_module(plugin_root) except ImportError: raise ImportError('Incorrect plugin root {}. No such package found'. format(plugin_root)) sys.path.insert(0, plugin_root.__path__[0]) enabled_plugins = config.ENABLED_PLUGINS installed_packages = {p.project_name: p for p in pip.get_installed_distributions()} for plugin_name in enabled_plugins: plugin = find_and_load_plugin(plugin_name, plugin_root, installed_packages) plugin_globals = plugin.__dict__ if 'LEDGER_IDS' in plugin_globals: PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS']) if 'CLIENT_REQUEST_FIELDS' in plugin_globals: PLUGIN_CLIENT_REQUEST_FIELDS.update(plugin_globals['CLIENT_REQUEST_FIELDS']) # Reloading message types since some some schemas would have been changed import plenum.common.messages.fields import plenum.common.messages.node_messages import plenum.common.messages.node_message_factory importlib.reload(plenum.common.messages.node_messages) importlib.reload(plenum.common.messages.node_message_factory)
def setup_plugins(): # TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field global PLUGIN_LEDGER_IDS global PLUGIN_CLIENT_REQUEST_FIELDS config = getConfigOnce() plugin_root = config.PLUGIN_ROOT try: plugin_root = importlib.import_module(plugin_root) except ImportError: raise ImportError( 'Incorrect plugin root {}. No such package found'.format( plugin_root)) enabled_plugins = config.ENABLED_PLUGINS for plugin_name in enabled_plugins: plugin_path = os.path.join(plugin_root.__path__[0], plugin_name, '__init__.py') spec = spec_from_file_location('__init__.py', plugin_path) init = module_from_spec(spec) spec.loader.exec_module(init) plugin_globals = init.__dict__ if 'LEDGER_IDS' in plugin_globals: PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS']) if 'CLIENT_REQUEST_FIELDS' in plugin_globals: PLUGIN_CLIENT_REQUEST_FIELDS.update( plugin_globals['CLIENT_REQUEST_FIELDS']) # Reloading message types since some some schemas would have been changed import plenum.common.messages.node_messages import plenum.common.messages.node_message_factory importlib.reload(plenum.common.messages.node_messages) importlib.reload(plenum.common.messages.node_message_factory)
def setup_plugins(): import os # noqa import pip # noqa import importlib # noqa from importlib.util import module_from_spec, spec_from_file_location # noqa: E402 import plenum # noqa: E402 import plenum.server.plugin # noqa: E402 from plenum.common.config_util import getConfigOnce # noqa: E402 def find_and_load_plugin(plugin_name, plugin_root, installed_packages): if plugin_name in installed_packages: # TODO: Need a test for installed packages plugin_name = plugin_name.replace('-', '_') plugin = importlib.import_module(plugin_name) else: plugin_path = os.path.join(plugin_root.__path__[0], plugin_name, '__init__.py') spec = spec_from_file_location('__init__.py', plugin_path) plugin = module_from_spec(spec) spec.loader.exec_module(plugin) return plugin # TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field global PLUGIN_LEDGER_IDS global PLUGIN_CLIENT_REQUEST_FIELDS config = getConfigOnce() plugin_root = config.PLUGIN_ROOT try: plugin_root = importlib.import_module(plugin_root) except ImportError: raise ImportError( 'Incorrect plugin root {}. No such package found'.format( plugin_root)) sys.path.insert(0, plugin_root.__path__[0]) enabled_plugins = config.ENABLED_PLUGINS installed_packages = { p.project_name: p for p in pip.get_installed_distributions() } for plugin_name in enabled_plugins: plugin = find_and_load_plugin(plugin_name, plugin_root, installed_packages) plugin_globals = plugin.__dict__ if 'LEDGER_IDS' in plugin_globals: PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS']) if 'CLIENT_REQUEST_FIELDS' in plugin_globals: PLUGIN_CLIENT_REQUEST_FIELDS.update( plugin_globals['CLIENT_REQUEST_FIELDS']) # Reloading message types since some some schemas would have been changed import plenum.common.messages.fields import plenum.common.messages.node_messages import plenum.common.messages.node_message_factory importlib.reload(plenum.common.messages.node_messages) importlib.reload(plenum.common.messages.node_message_factory)
def __init__(self, tmpdir, config=None): node_names = ['Node1', 'Node2', 'Node3', 'Node4'] self.basedirpath = tmpdir self.name = node_names[0] self.viewNo = 0 self.db_manager = DatabaseManager() self.timer = QueueTimer() self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = node_names self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.nodeIds = [] self.totalNodes = len(self.allNodeNames) self.poolManager = FakeSomething( node_names_ordered_by_rank=lambda: node_names) self.mode = Mode.starting self.monitor = FakeSomething(isMasterDegraded=lambda: False) self.config = config or getConfigOnce() self.nodeStatusDB = None self.quorums = Quorums(self.totalNodes) self.nodestack = FakeSomething(connecteds=set(self.allNodeNames)) self.write_manager = FakeSomething( node_reg_handler=NodeRegHandler(self.db_manager)) self.primaries_selector = RoundRobinConstantNodesPrimariesSelector( node_names) self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config) } self.requiredNumberOfInstances = 2 self._found = False self.ledgerManager = LedgerManager(self) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.metrics = NullMetricsCollector() # For catchup testing self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} # callbacks self.onBatchCreated = lambda self, *args, **kwargs: True
def setup_plugins(): global PLUGIN_LEDGER_IDS config = getConfigOnce() ENABLED_PLUGINS = config.ENABLED_PLUGINS for plugin_name in ENABLED_PLUGINS: plugin_path = os.path.join(plenum.server.plugin.__path__[0], plugin_name, '__init__.py') spec = spec_from_file_location('__init__.py', plugin_path) init = module_from_spec(spec) spec.loader.exec_module(init) plugin_globals = init.__dict__ if 'LEDGER_IDS' in plugin_globals: PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS']) if 'CLIENT_REQUEST_FIELDS' in plugin_globals: PLUGIN_CLIENT_REQUEST_FIELDS.update( plugin_globals['CLIENT_REQUEST_FIELDS'])
def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.internal_bus = InternalBus() self.db_manager = DatabaseManager() self.timer = QueueTimer() self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = {name: HA("127.0.0.1", 0) for name in self.allNodeNames} self.nodeIds = [] self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.nodeStatusDB = None self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = create_view_changer(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {} # callbacks self.onBatchCreated = lambda self, *args, **kwargs: True
def test_replay_on_new_node(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir, testNodeClass, tmpdir_factory, node_config_helper_class, allPluginsPath, sdk_new_node_caught_up): # noqa: F811 new_node = sdk_new_node_caught_up for node in txnPoolNodeSet: txnPoolNodesLooper.removeProdable(node) node.stop() config = getConfigOnce() reload_modules_for_replay(tconf) replayable_node_class, basedirpath = get_replayable_node_class( tmpdir_factory, tdir, testNodeClass, config) print('-------------Replaying now---------------------') create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, new_node, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath)
def setup_plugins(): # TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field global PLUGIN_LEDGER_IDS global PLUGIN_CLIENT_REQUEST_FIELDS config = getConfigOnce() plugin_root = config.PLUGIN_ROOT try: plugin_root = importlib.import_module(plugin_root) except ImportError: raise ImportError( 'Incorrect plugin root {}. No such package found'.format( plugin_root)) sys.path.insert(0, plugin_root.__path__[0]) enabled_plugins = config.ENABLED_PLUGINS installed_packages = { p.project_name: p for p in pip.get_installed_distributions() } for plugin_name in enabled_plugins: plugin = find_and_load_plugin(plugin_name, plugin_root, installed_packages) plugin_globals = plugin.__dict__ if 'LEDGER_IDS' in plugin_globals: PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS']) if 'CLIENT_REQUEST_FIELDS' in plugin_globals: PLUGIN_CLIENT_REQUEST_FIELDS.update( plugin_globals['CLIENT_REQUEST_FIELDS']) # Reloading message types since some some schemas would have been changed import plenum.common.messages.fields import plenum.common.messages.node_messages import plenum.common.messages.node_message_factory importlib.reload(plenum.common.messages.node_messages) importlib.reload(plenum.common.messages.node_message_factory)
def __init__(self, tmpdir, config=None): self.basedirpath = tmpdir self.name = 'Node1' self.f = 1 self.replicas = dict() self.requests = Requests() self.rank = None self.allNodeNames = [self.name, 'Node2', 'Node3', 'Node4'] self.nodeReg = { name: HA("127.0.0.1", 0) for name in self.allNodeNames } self.totalNodes = len(self.allNodeNames) self.mode = Mode.starting self.config = config or getConfigOnce() self.replicas = { 0: Replica(node=self, instId=0, isMaster=True, config=self.config), 1: Replica(node=self, instId=1, isMaster=False, config=self.config), 2: Replica(node=self, instId=2, isMaster=False, config=self.config), } self._found = False self.ledgerManager = LedgerManager(self, ownedByNode=True) ledger0 = FakeLedger(0, 10) ledger1 = FakeLedger(1, 5) self.ledgerManager.addLedger(0, ledger0) self.ledgerManager.addLedger(1, ledger1) self.quorums = Quorums(self.totalNodes) self.view_changer = ViewChanger(self) self.elector = PrimarySelector(self) self.metrics = NullMetricsCollector() # For catchup testing self.catchup_rounds_without_txns = 0 self.view_change_in_progress = False self.ledgerManager.last_caught_up_3PC = (0, 0) self.master_last_ordered_3PC = (0, 0) self.seqNoDB = {}
def test_replay_new_bouncing(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir, testNodeClass, tmpdir_factory, node_config_helper_class, allPluginsPath, some_txns_done, sdk_pool_handle, sdk_wallet_client): alpha = txnPoolNodeSet[0] old_view_no = alpha.viewNo other_nodes = txnPoolNodeSet[1:] stopping_at = time.perf_counter() alpha.stop() txnPoolNodesLooper.removeProdable(alpha) txnPoolNodesLooper.run( eventually(checkViewNoForNodes, other_nodes, old_view_no + 1, retryWait=1, timeout=30)) sdk_send_random_and_check(txnPoolNodesLooper, other_nodes, sdk_pool_handle, sdk_wallet_client, 10) ensure_all_nodes_have_same_data(txnPoolNodesLooper, other_nodes) for node in other_nodes: assert alpha.name not in node.nodestack.connecteds nodeHa, nodeCHa = HA(*alpha.nodestack.ha), HA(*alpha.clientstack.ha) config_helper = PNodeConfigHelper(alpha.name, tconf, chroot=tdir) restarted_alpha = testNodeClass(alpha.name, config_helper=config_helper, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) txnPoolNodesLooper.add(restarted_alpha) txnPoolNodeSet[0] = restarted_alpha restarting_at = time.perf_counter() print('Stopped for {}'.format(restarting_at - stopping_at)) sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 10) ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet) for node in txnPoolNodeSet: node.stop() txnPoolNodesLooper.removeProdable(node) config = getConfigOnce() reload_modules_for_replay(tconf) replayable_node_class, basedirpath = get_replayable_node_class( tmpdir_factory, tdir, testNodeClass, config) print('-------------Replaying now---------------------') create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, restarted_alpha, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath) # For disconnections for node in other_nodes: create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, node, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath)
def test_replay_new_bouncing(txnPoolNodesLooper, txnPoolNodeSet, tconf, tdir, testNodeClass, tmpdir_factory, node_config_helper_class, allPluginsPath, some_txns_done, sdk_pool_handle, sdk_wallet_client): alpha = txnPoolNodeSet[0] old_view_no = alpha.viewNo other_nodes = txnPoolNodeSet[1:] stopping_at = time.perf_counter() alpha.stop() txnPoolNodesLooper.removeProdable(alpha) txnPoolNodesLooper.run(eventually(checkViewNoForNodes, other_nodes, old_view_no + 1, retryWait=1, timeout=30)) sdk_send_random_and_check(txnPoolNodesLooper, other_nodes, sdk_pool_handle, sdk_wallet_client, 10) ensure_all_nodes_have_same_data(txnPoolNodesLooper, other_nodes) for node in other_nodes: assert alpha.name not in node.nodestack.connecteds nodeHa, nodeCHa = HA(*alpha.nodestack.ha), HA(*alpha.clientstack.ha) config_helper = PNodeConfigHelper(alpha.name, tconf, chroot=tdir) restarted_alpha = testNodeClass( alpha.name, config_helper=config_helper, config=tconf, ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath) txnPoolNodesLooper.add(restarted_alpha) txnPoolNodeSet[0] = restarted_alpha restarting_at = time.perf_counter() print('Stopped for {}'.format(restarting_at - stopping_at)) sdk_send_random_and_check(txnPoolNodesLooper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 10) ensure_all_nodes_have_same_data(txnPoolNodesLooper, txnPoolNodeSet) for node in txnPoolNodeSet: node.stop() txnPoolNodesLooper.removeProdable(node) config = getConfigOnce() reload_modules_for_replay(tconf) replayable_node_class, basedirpath = get_replayable_node_class( tmpdir_factory, tdir, testNodeClass, config) print('-------------Replaying now---------------------') create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, restarted_alpha, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath) # For disconnections for node in other_nodes: create_replayable_node_and_check(txnPoolNodesLooper, txnPoolNodeSet, node, replayable_node_class, node_config_helper_class, tconf, basedirpath, allPluginsPath)