def formatPartitionTable(self, row_list): nm = NodeManager() nm.update(dummy_app, 1, self.neoctl.getNodeList(node_type=NodeTypes.STORAGE)) pt = object.__new__(PartitionTable) pt._load(None, None, row_list, nm.getByUUID) pt.addNodeList(nm.getByStateList(NodeStates.RUNNING)) return '\n'.join(line[4:] for line in pt._format())
def setUp(self): super(MasterHandlerTests, self).setUp() self.db = Mock() self.app = Mock({'getDB': self.db, 'txn_contexts': ()}) self.app.nm = NodeManager() self.app.dispatcher = Mock() self._next_port = 3000
def __init__(self, app, name, master_addresses): self.app = weakref.proxy(app) self.name = name self.nm = NodeManager() for master_address in master_addresses: self.nm.createMaster(address=master_address)
class BackupApplication(object): pt = None server = None # like in BaseApplication uuid = None def __init__(self, app, name, master_addresses): self.app = weakref.proxy(app) self.name = name self.nm = NodeManager() for master_address in master_addresses: self.nm.createMaster(address=master_address) em = property(lambda self: self.app.em) ssl = property(lambda self: self.app.ssl) def close(self): self.nm.close() del self.__dict__ def setUUID(self, uuid): if self.uuid != uuid: self.uuid = uuid logging.info('Upstream Node ID: %s', uuid_str(uuid)) def log(self): self.nm.log() if self.pt is not None: self.pt.log() def provideService(self): logging.info('provide backup') poll = self.em.poll app = self.app pt = app.pt while True: app.changeClusterState(ClusterStates.STARTING_BACKUP) bootstrap = BootstrapManager(self, NodeTypes.CLIENT, backup=app.name) # {offset -> node} self.primary_partition_dict = {} # [[tid]] self.tid_list = tuple([] for _ in xrange(pt.getPartitions())) try: while True: for node in pt.getNodeSet(readable=True): if not app.isStorageReady(node.getUUID()): break else: break poll(1) node, conn = bootstrap.getPrimaryConnection() try: app.changeClusterState(ClusterStates.BACKINGUP) del bootstrap, node self.ignore_invalidations = True conn.setHandler(BackupHandler(self)) conn.ask(Packets.AskLastTransaction()) # debug variable to log how big 'tid_list' can be. self.debug_tid_count = 0 while True: poll(1) except PrimaryFailure, msg: logging.error('upstream master is down: %s', msg) finally: app.backup_tid = pt.getBackupTid() try: conn.close() except PrimaryFailure: pass try: del self.pt except AttributeError: pass for node in app.nm.getClientList(True): node.getConnection().close() except StateChangedException, e: if e.args[0] != ClusterStates.STOPPING_BACKUP: raise app.changeClusterState(*e.args) tid = app.backup_tid # Wait for non-primary partitions to catch up, # so that all UP_TO_DATE cells are really UP_TO_DATE. # XXX: Another possibility could be to outdate such cells, and # they would be quickly updated at the beginning of the # RUNNING phase. This may simplify code. # Any unfinished replication from upstream will be truncated. while pt.getBackupTid(min) < tid: poll(1) last_tid = app.getLastTransaction() handler = EventHandler(app) if tid < last_tid: assert tid != ZERO_TID logging.warning("Truncating at %s (last_tid was %s)", dump(app.backup_tid), dump(last_tid)) else: # We will do a dummy truncation, just to leave backup mode, # so it's fine to start automatically if there's any # missing storage. # XXX: Consider using another method to leave backup mode, # at least when there's nothing to truncate. Because # in case of StoppedOperation during VERIFYING state, # this flag will be wrongly set to False. app._startup_allowed = True # If any error happened before reaching this line, we'd go back # to backup mode, which is the right mode to recover. del app.backup_tid # Now back to RECOVERY... return tid finally:
def setUp(self): NeoUnitTestBase.setUp(self) self.manager = NodeManager()
class NodeManagerTests(NeoUnitTestBase): def setUp(self): NeoUnitTestBase.setUp(self) self.manager = NodeManager() def _addStorage(self): self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID()) def _addMaster(self): self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID()) def _addClient(self): self.client = ClientNode(self.manager, None, self.getClientUUID()) def _addAdmin(self): self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID()) def checkNodes(self, node_list): manager = self.manager self.assertEqual(sorted(manager.getList()), sorted(node_list)) def checkMasters(self, master_list): manager = self.manager self.assertEqual(manager.getMasterList(), master_list) def checkStorages(self, storage_list): manager = self.manager self.assertEqual(manager.getStorageList(), storage_list) def checkClients(self, client_list): manager = self.manager self.assertEqual(manager.getClientList(), client_list) def checkByServer(self, node): node_found = self.manager.getByAddress(node.getAddress()) self.assertEqual(node_found, node) def checkByUUID(self, node): node_found = self.manager.getByUUID(node.getUUID()) self.assertEqual(node_found, node) def checkIdentified(self, node_list, pool_set=None): identified_node_list = self.manager.getIdentifiedList(pool_set) self.assertEqual(set(identified_node_list), set(node_list)) def testInit(self): """ Check the manager is empty when started """ manager = self.manager self.checkNodes([]) self.checkMasters([]) self.checkStorages([]) self.checkClients([]) address = ('127.0.0.1', 10000) self.assertEqual(manager.getByAddress(address), None) self.assertEqual(manager.getByAddress(None), None) uuid = self.getNewUUID(None) self.assertEqual(manager.getByUUID(uuid), None) self.assertEqual(manager.getByUUID(None), None) def testAdd(self): """ Check if new nodes are registered in the manager """ manager = self.manager self.checkNodes([]) # storage self._addStorage() self.checkNodes([self.storage]) self.checkStorages([self.storage]) self.checkMasters([]) self.checkClients([]) self.checkByServer(self.storage) self.checkByUUID(self.storage) # master self._addMaster() self.checkNodes([self.storage, self.master]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([]) self.checkByServer(self.master) self.checkByUUID(self.master) # client self._addClient() self.checkNodes([self.storage, self.master, self.client]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) # client node has no address self.assertEqual(manager.getByAddress(self.client.getAddress()), None) self.checkByUUID(self.client) # admin self._addAdmin() self.checkNodes([self.storage, self.master, self.client, self.admin]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) self.checkByServer(self.admin) self.checkByUUID(self.admin) def testUpdate(self): """ Check manager content update """ # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() self.checkNodes([self.master, self.storage, self.client, self.admin]) self.checkMasters([self.master]) self.checkStorages([self.storage]) self.checkClients([self.client]) # build changes old_address = self.master.getAddress() new_address = ('127.0.0.1', 2001) old_uuid = self.storage.getUUID() new_uuid = self.getStorageUUID() node_list = ( (NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN), (NodeTypes.MASTER, new_address, self.master.getUUID(), NodeStates.RUNNING), (NodeTypes.STORAGE, self.storage.getAddress(), new_uuid, NodeStates.RUNNING), (NodeTypes.ADMIN, self.admin.getAddress(), self.admin.getUUID(), NodeStates.UNKNOWN), ) # update manager content manager.update(node_list) # - the client gets down self.checkClients([]) # - master change it's address self.checkMasters([self.master]) self.assertEqual(manager.getByAddress(old_address), None) self.master.setAddress(new_address) self.checkByServer(self.master) # - storage change it's UUID storage_list = manager.getStorageList() self.assertTrue(len(storage_list), 1) new_storage = storage_list[0] self.assertNotEqual(new_storage.getUUID(), old_uuid) self.assertEqual(new_storage.getState(), NodeStates.RUNNING) # admin is still here but in UNKNOWN state self.checkNodes([self.master, self.admin, new_storage]) self.assertEqual(self.admin.getState(), NodeStates.UNKNOWN) def testIdentified(self): # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() # switch node to connected self.checkIdentified([]) self.master.setConnection(Mock()) self.checkIdentified([self.master]) self.storage.setConnection(Mock()) self.checkIdentified([self.master, self.storage]) self.client.setConnection(Mock()) self.checkIdentified([self.master, self.storage, self.client]) self.admin.setConnection(Mock()) self.checkIdentified([self.master, self.storage, self.client, self.admin]) # check the pool_set attribute self.checkIdentified([self.master], pool_set=[self.master.getUUID()]) self.checkIdentified([self.storage], pool_set=[self.storage.getUUID()]) self.checkIdentified([self.client], pool_set=[self.client.getUUID()]) self.checkIdentified([self.admin], pool_set=[self.admin.getUUID()]) self.checkIdentified([self.master, self.storage], pool_set=[ self.master.getUUID(), self.storage.getUUID()])
class NodeManagerTests(NeoUnitTestBase): def setUp(self): NeoUnitTestBase.setUp(self) self.manager = NodeManager() def _addStorage(self): self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID()) def _addMaster(self): self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID()) def _addClient(self): self.client = ClientNode(self.manager, None, self.getClientUUID()) def _addAdmin(self): self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID()) def checkNodes(self, node_list): manager = self.manager self.assertEqual(sorted(manager.getList()), sorted(node_list)) def checkMasters(self, master_list): manager = self.manager self.assertEqual(manager.getMasterList(), master_list) def checkStorages(self, storage_list): manager = self.manager self.assertEqual(manager.getStorageList(), storage_list) def checkClients(self, client_list): manager = self.manager self.assertEqual(manager.getClientList(), client_list) def checkByServer(self, node): node_found = self.manager.getByAddress(node.getAddress()) self.assertEqual(node_found, node) def checkByUUID(self, node): node_found = self.manager.getByUUID(node.getUUID()) self.assertEqual(node_found, node) def checkIdentified(self, node_list, pool_set=None): identified_node_list = self.manager.getIdentifiedList(pool_set) self.assertEqual(set(identified_node_list), set(node_list)) def testInit(self): """ Check the manager is empty when started """ manager = self.manager self.checkNodes([]) self.checkMasters([]) self.checkStorages([]) self.checkClients([]) address = ('127.0.0.1', 10000) self.assertEqual(manager.getByAddress(address), None) self.assertEqual(manager.getByAddress(None), None) uuid = self.getNewUUID(None) self.assertEqual(manager.getByUUID(uuid), None) self.assertEqual(manager.getByUUID(None), None) def testAdd(self): """ Check if new nodes are registered in the manager """ manager = self.manager self.checkNodes([]) # storage self._addStorage() self.checkNodes([self.storage]) self.checkStorages([self.storage]) self.checkMasters([]) self.checkClients([]) self.checkByServer(self.storage) self.checkByUUID(self.storage) # master self._addMaster() self.checkNodes([self.storage, self.master]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([]) self.checkByServer(self.master) self.checkByUUID(self.master) # client self._addClient() self.checkNodes([self.storage, self.master, self.client]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) # client node has no address self.assertEqual(manager.getByAddress(self.client.getAddress()), None) self.checkByUUID(self.client) # admin self._addAdmin() self.checkNodes([self.storage, self.master, self.client, self.admin]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) self.checkByServer(self.admin) self.checkByUUID(self.admin) def testUpdate(self): """ Check manager content update """ # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() self.checkNodes([self.master, self.storage, self.client, self.admin]) self.checkMasters([self.master]) self.checkStorages([self.storage]) self.checkClients([self.client]) # build changes old_address = self.master.getAddress() new_address = ('127.0.0.1', 2001) old_uuid = self.storage.getUUID() new_uuid = self.getStorageUUID() node_list = ( (NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN), (NodeTypes.MASTER, new_address, self.master.getUUID(), NodeStates.RUNNING), (NodeTypes.STORAGE, self.storage.getAddress(), new_uuid, NodeStates.RUNNING), (NodeTypes.ADMIN, self.admin.getAddress(), self.admin.getUUID(), NodeStates.UNKNOWN), ) # update manager content manager.update(node_list) # - the client gets down self.checkClients([]) # - master change it's address self.checkMasters([self.master]) self.assertEqual(manager.getByAddress(old_address), None) self.master.setAddress(new_address) self.checkByServer(self.master) # - storage change it's UUID storage_list = manager.getStorageList() self.assertTrue(len(storage_list), 1) new_storage = storage_list[0] self.assertNotEqual(new_storage.getUUID(), old_uuid) self.assertEqual(new_storage.getState(), NodeStates.RUNNING) # admin is still here but in UNKNOWN state self.checkNodes([self.master, self.admin, new_storage]) self.assertEqual(self.admin.getState(), NodeStates.UNKNOWN) def testIdentified(self): # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() # switch node to connected self.checkIdentified([]) self.master.setConnection(Mock()) self.checkIdentified([self.master]) self.storage.setConnection(Mock()) self.checkIdentified([self.master, self.storage]) self.client.setConnection(Mock()) self.checkIdentified([self.master, self.storage, self.client]) self.admin.setConnection(Mock()) self.checkIdentified( [self.master, self.storage, self.client, self.admin]) # check the pool_set attribute self.checkIdentified([self.master], pool_set=[self.master.getUUID()]) self.checkIdentified([self.storage], pool_set=[self.storage.getUUID()]) self.checkIdentified([self.client], pool_set=[self.client.getUUID()]) self.checkIdentified([self.admin], pool_set=[self.admin.getUUID()]) self.checkIdentified( [self.master, self.storage], pool_set=[self.master.getUUID(), self.storage.getUUID()])
class BackupApplication(object): pt = None def __init__(self, app, name, master_addresses): self.app = weakref.proxy(app) self.name = name self.nm = NodeManager() for master_address in master_addresses: self.nm.createMaster(address=master_address) em = property(lambda self: self.app.em) ssl = property(lambda self: self.app.ssl) def close(self): self.nm.close() del self.__dict__ def log(self): self.nm.log() if self.pt is not None: self.pt.log() def provideService(self): logging.info('provide backup') poll = self.em.poll app = self.app pt = app.pt while True: app.changeClusterState(ClusterStates.STARTING_BACKUP) bootstrap = BootstrapManager(self, self.name, NodeTypes.CLIENT) # {offset -> node} self.primary_partition_dict = {} # [[tid]] self.tid_list = tuple([] for _ in xrange(pt.getPartitions())) try: while True: for node in pt.getNodeSet(readable=True): if not app.isStorageReady(node.getUUID()): break else: break poll(1) node, conn, uuid, num_partitions, num_replicas = \ bootstrap.getPrimaryConnection() try: app.changeClusterState(ClusterStates.BACKINGUP) del bootstrap, node if num_partitions != pt.getPartitions(): raise RuntimeError("inconsistent number of partitions") self.pt = PartitionTable(num_partitions, num_replicas) conn.setHandler(BackupHandler(self)) conn.ask(Packets.AskNodeInformation()) conn.ask(Packets.AskPartitionTable()) conn.ask(Packets.AskLastTransaction()) # debug variable to log how big 'tid_list' can be. self.debug_tid_count = 0 while True: poll(1) except PrimaryFailure, msg: logging.error('upstream master is down: %s', msg) finally: app.backup_tid = pt.getBackupTid() try: conn.close() except PrimaryFailure: pass try: del self.pt except AttributeError: pass except StateChangedException, e: if e.args[0] != ClusterStates.STOPPING_BACKUP: raise app.changeClusterState(*e.args) tid = app.backup_tid # Wait for non-primary partitions to catch up, # so that all UP_TO_DATE cells are really UP_TO_DATE. # XXX: Another possibility could be to outdate such cells, and # they would be quickly updated at the beginning of the # RUNNING phase. This may simplify code. # Any unfinished replication from upstream will be truncated. while pt.getBackupTid(min) < tid: poll(1) last_tid = app.getLastTransaction() handler = EventHandler(app) if tid < last_tid: assert tid != ZERO_TID logging.warning("Truncating at %s (last_tid was %s)", dump(app.backup_tid), dump(last_tid)) else: # We will do a dummy truncation, just to leave backup mode, # so it's fine to start automatically if there's any # missing storage. # XXX: Consider using another method to leave backup mode, # at least when there's nothing to truncate. Because # in case of StoppedOperation during VERIFYING state, # this flag will be wrongly set to False. app._startup_allowed = True # If any error happened before reaching this line, we'd go back # to backup mode, which is the right mode to recover. del app.backup_tid # Now back to RECOVERY... return tid finally: