def test_07_getNodeSet(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) # add two kind of node, usable and unsable uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) pt.setCell(0, sn1, CellStates.UP_TO_DATE) uuid2 = self.getStorageUUID() server2 = ("127.0.0.2", 19001) sn2 = StorageNode(Mock(), server2, uuid2) pt.setCell(0, sn2, CellStates.OUT_OF_DATE) uuid3 = self.getStorageUUID() server3 = ("127.0.0.3", 19001) sn3 = StorageNode(Mock(), server3, uuid3) pt.setCell(0, sn3, CellStates.FEEDING) uuid4 = self.getStorageUUID() server4 = ("127.0.0.4", 19001) sn4 = StorageNode(Mock(), server4, uuid4) pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added # must get only two node as feeding and discarded not taken # into account self.assertEqual(pt.getNodeSet(True), {sn1, sn3}) self.assertEqual(len(pt.getNodeSet()), 3)
def test_06_clear(self): # add some nodes num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) # add two kind of node, usable and unsable uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) pt.setCell(0, sn1, CellStates.UP_TO_DATE) uuid2 = self.getStorageUUID() server2 = ("127.0.0.2", 19001) sn2 = StorageNode(Mock(), server2, uuid2) pt.setCell(1, sn2, CellStates.OUT_OF_DATE) uuid3 = self.getStorageUUID() server3 = ("127.0.0.3", 19001) sn3 = StorageNode(Mock(), server3, uuid3) pt.setCell(2, sn3, CellStates.FEEDING) # now checks result self.assertEqual(len(pt.partition_list[0]), 1) self.assertEqual(len(pt.partition_list[1]), 1) self.assertEqual(len(pt.partition_list[2]), 1) pt.clear() partition_list = pt.partition_list self.assertEqual(len(partition_list), num_partitions) for x in xrange(num_partitions): part = partition_list[x] self.assertTrue(isinstance(part, list)) self.assertEqual(len(part), 0) self.assertEqual(len(pt.count_dict), 0)
def test_16_make(self): num_partitions = 5 num_replicas = 1 pt = PartitionTable(num_partitions, num_replicas) # add nodes uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1, NodeStates.RUNNING) # add not running node uuid2 = self.getStorageUUID() server2 = ("127.0.0.2", 19001) sn2 = StorageNode(Mock(), server2, uuid2) sn2.setState(NodeStates.TEMPORARILY_DOWN) # add node without uuid server3 = ("127.0.0.3", 19001) sn3 = StorageNode(Mock(), server3, None, NodeStates.RUNNING) # add clear node uuid4 = self.getStorageUUID() server4 = ("127.0.0.4", 19001) sn4 = StorageNode(Mock(), server4, uuid4, NodeStates.RUNNING) uuid5 = self.getStorageUUID() server5 = ("127.0.0.5", 1900) sn5 = StorageNode(Mock(), server5, uuid5, NodeStates.RUNNING) # make the table pt.make([sn1, sn2, sn3, sn4, sn5]) # check it's ok, only running nodes and node with uuid # must be present for x in xrange(num_partitions): cells = pt.getCellList(x) self.assertEqual(len(cells), 2) nodes = [x.getNode() for x in cells] for node in nodes: self.assertTrue(node in (sn1, sn4, sn5)) self.assertTrue(node not in (sn2, sn3)) self.assertTrue(pt.filled()) self.assertTrue(pt.operational()) # create a pt with less nodes pt.clear() self.assertFalse(pt.filled()) self.assertFalse(pt.operational()) pt.make([sn1]) # check it's ok for x in xrange(num_partitions): cells = pt.getCellList(x) self.assertEqual(len(cells), 1) nodes = [x.getNode() for x in cells] for node in nodes: self.assertEqual(node, sn1) self.assertTrue(pt.filled()) self.assertTrue(pt.operational())
def test_15_dropNodeList(self): sn = [ StorageNode(Mock(), None, i + 1, NodeStates.RUNNING) for i in xrange(3) ] pt = PartitionTable(3, 0) pt.setCell(0, sn[0], CellStates.OUT_OF_DATE) pt.setCell(1, sn[1], CellStates.FEEDING) pt.setCell(1, sn[2], CellStates.OUT_OF_DATE) pt.setCell(2, sn[0], CellStates.OUT_OF_DATE) pt.setCell(2, sn[1], CellStates.FEEDING) pt.setCell(2, sn[2], CellStates.UP_TO_DATE) self.assertEqual(sorted(pt.dropNodeList(sn[:1], True)), [(0, 1, CellStates.DISCARDED), (2, 1, CellStates.DISCARDED), (2, 2, CellStates.UP_TO_DATE)]) self.assertEqual(sorted(pt.dropNodeList(sn[2:], True)), [(1, 2, CellStates.UP_TO_DATE), (1, 3, CellStates.DISCARDED), (2, 2, CellStates.UP_TO_DATE), (2, 3, CellStates.DISCARDED)]) self.assertRaises(PartitionTableException, pt.dropNodeList, sn[1:2]) pt.setCell(1, sn[2], CellStates.UP_TO_DATE) self.assertEqual(sorted(pt.dropNodeList(sn[1:2])), [(1, 2, CellStates.DISCARDED), (2, 2, CellStates.DISCARDED)]) self.assertEqual(self.tweak(pt), [(2, 3, CellStates.FEEDING)])
def test_05_getCellList(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) # add two kind of node, usable and unsable uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) pt.setCell(0, sn1, CellStates.UP_TO_DATE) uuid2 = self.getStorageUUID() server2 = ("127.0.0.2", 19001) sn2 = StorageNode(Mock(), server2, uuid2) pt.setCell(0, sn2, CellStates.OUT_OF_DATE) uuid3 = self.getStorageUUID() server3 = ("127.0.0.3", 19001) sn3 = StorageNode(Mock(), server3, uuid3) pt.setCell(0, sn3, CellStates.FEEDING) uuid4 = self.getStorageUUID() server4 = ("127.0.0.4", 19001) sn4 = StorageNode(Mock(), server4, uuid4) pt.setCell(0, sn4, CellStates.DISCARDED) # won't be added # now checks result self.assertEqual(len(pt.partition_list[0]), 3) for x in xrange(num_partitions): if x == 0: # all nodes all_cell = pt.getCellList(0) all_nodes = [x.getNode() for x in all_cell] self.assertEqual(len(all_cell), 3) self.assertTrue(sn1 in all_nodes) self.assertTrue(sn2 in all_nodes) self.assertTrue(sn3 in all_nodes) self.assertTrue(sn4 not in all_nodes) # readable nodes all_cell = pt.getCellList(0, readable=True) all_nodes = [x.getNode() for x in all_cell] self.assertEqual(len(all_cell), 2) self.assertTrue(sn1 in all_nodes) self.assertTrue(sn2 not in all_nodes) self.assertTrue(sn3 in all_nodes) self.assertTrue(sn4 not in all_nodes) else: self.assertEqual(len(pt.getCellList(1, False)), 0) self.assertEqual(len(pt.getCellList(1, True)), 0)
def testStorage(self): """ Check Storage sub class """ node = StorageNode(self.manager) self.assertEqual(node.getType(), protocol.NodeTypes.STORAGE) self.assertTrue(node.isStorage()) self.assertFalse(node.isMaster()) self.assertFalse(node.isClient()) self.assertFalse(node.isAdmin())
def test_09_hasOffset(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) # add two kind of node, usable and unsable uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) pt.setCell(0, sn1, CellStates.UP_TO_DATE) # now test self.assertTrue(pt.hasOffset(0)) self.assertFalse(pt.hasOffset(1)) # unknonw partition self.assertFalse(pt.hasOffset(50))
def test_08_filled(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) self.assertEqual(pt.np, num_partitions) self.assertEqual(pt.num_filled_rows, 0) self.assertFalse(pt.filled()) # adding a node in all partition uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) for x in xrange(num_partitions): pt.setCell(x, sn1, CellStates.UP_TO_DATE) self.assertEqual(pt.num_filled_rows, num_partitions) self.assertTrue(pt.filled())
def test_01_Cell(self): uuid = self.getStorageUUID() server = ("127.0.0.1", 19001) sn = StorageNode(Mock(), server, uuid) cell = Cell(sn) self.assertEqual(cell.node, sn) self.assertEqual(cell.state, CellStates.UP_TO_DATE) cell = Cell(sn, CellStates.OUT_OF_DATE) self.assertEqual(cell.node, sn) self.assertEqual(cell.state, CellStates.OUT_OF_DATE) # check getter self.assertEqual(cell.getNode(), sn) self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE) self.assertEqual(cell.getNodeState(), NodeStates.UNKNOWN) self.assertEqual(cell.getUUID(), uuid) self.assertEqual(cell.getAddress(), server) # check state setter cell.setState(CellStates.FEEDING) self.assertEqual(cell.getState(), CellStates.FEEDING)
def test_04_removeCell(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0) # add a cell to an empty row self.assertFalse(pt.count_dict.has_key(sn1)) pt.setCell(0, sn1, CellStates.UP_TO_DATE) self.assertTrue(pt.count_dict.has_key(sn1)) self.assertEqual(pt.count_dict[sn1], 1) for x in xrange(num_partitions): if x == 0: self.assertEqual(len(pt.partition_list[x]), 1) else: self.assertEqual(len(pt.partition_list[x]), 0) # remove it pt.removeCell(0, sn1) self.assertEqual(pt.count_dict[sn1], 0) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0) # add a feeding cell pt.setCell(0, sn1, CellStates.FEEDING) self.assertTrue(pt.count_dict.has_key(sn1)) self.assertEqual(pt.count_dict[sn1], 0) for x in xrange(num_partitions): if x == 0: self.assertEqual(len(pt.partition_list[x]), 1) else: self.assertEqual(len(pt.partition_list[x]), 0) # remove it pt.removeCell(0, sn1) self.assertEqual(pt.count_dict[sn1], 0) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0)
def test_13_outdate(self): # create nodes uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) uuid2 = self.getStorageUUID() server2 = ("127.0.0.2", 19002) sn2 = StorageNode(Mock(), server2, uuid2) uuid3 = self.getStorageUUID() server3 = ("127.0.0.3", 19003) sn3 = StorageNode(Mock(), server3, uuid3) uuid4 = self.getStorageUUID() server4 = ("127.0.0.4", 19004) sn4 = StorageNode(Mock(), server4, uuid4) uuid5 = self.getStorageUUID() server5 = ("127.0.0.5", 19005) sn5 = StorageNode(Mock(), server5, uuid5) # create partition table num_partitions = 5 num_replicas = 3 pt = PartitionTable(num_partitions, num_replicas) pt.setCell(0, sn1, CellStates.OUT_OF_DATE) sn1.setState(NodeStates.RUNNING) pt.setCell(1, sn2, CellStates.UP_TO_DATE) sn2.setState(NodeStates.TEMPORARILY_DOWN) pt.setCell(2, sn3, CellStates.UP_TO_DATE) sn3.setState(NodeStates.DOWN) pt.setCell(3, sn4, CellStates.UP_TO_DATE) sn4.setState(NodeStates.BROKEN) pt.setCell(4, sn5, CellStates.UP_TO_DATE) sn5.setState(NodeStates.RUNNING) # outdate nodes cells_outdated = pt.outdate() self.assertEqual(len(cells_outdated), 3) for offset, uuid, state in cells_outdated: self.assertTrue(offset in (1, 2, 3)) self.assertTrue(uuid in (uuid2, uuid3, uuid4)) self.assertEqual(state, CellStates.OUT_OF_DATE) # check each cell # part 1, already outdated cells = pt.getCellList(0) self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE) # part 2, must be outdated cells = pt.getCellList(1) self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE) # part 3, must be outdated cells = pt.getCellList(2) self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE) # part 4, already outdated cells = pt.getCellList(3) self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.getState(), CellStates.OUT_OF_DATE) # part 5, remains running cells = pt.getCellList(4) self.assertEqual(len(cells), 1) cell = cells[0] self.assertEqual(cell.getState(), CellStates.UP_TO_DATE)
def test_12_getRow(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) # add nodes uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) pt.setCell(0, sn1, CellStates.UP_TO_DATE) pt.setCell(1, sn1, CellStates.UP_TO_DATE) pt.setCell(2, sn1, CellStates.UP_TO_DATE) uuid2 = self.getStorageUUID() server2 = ("127.0.0.2", 19001) sn2 = StorageNode(Mock(), server2, uuid2) pt.setCell(0, sn2, CellStates.UP_TO_DATE) pt.setCell(1, sn2, CellStates.UP_TO_DATE) uuid3 = self.getStorageUUID() server3 = ("127.0.0.3", 19001) sn3 = StorageNode(Mock(), server3, uuid3) pt.setCell(0, sn3, CellStates.UP_TO_DATE) # test row_0 = pt.getRow(0) self.assertEqual(len(row_0), 3) for uuid, state in row_0: self.assertTrue(uuid in (sn1.getUUID(), sn2.getUUID(), sn3.getUUID())) self.assertEqual(state, CellStates.UP_TO_DATE) row_1 = pt.getRow(1) self.assertEqual(len(row_1), 2) for uuid, state in row_1: self.assertTrue(uuid in (sn1.getUUID(), sn2.getUUID())) self.assertEqual(state, CellStates.UP_TO_DATE) row_2 = pt.getRow(2) self.assertEqual(len(row_2), 1) for uuid, state in row_2: self.assertEqual(uuid, sn1.getUUID()) self.assertEqual(state, CellStates.UP_TO_DATE) row_3 = pt.getRow(3) self.assertEqual(len(row_3), 0) row_4 = pt.getRow(4) self.assertEqual(len(row_4), 0) # unknwon row self.assertRaises(IndexError, pt.getRow, 5)
def test_10_operational(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) self.assertFalse(pt.filled()) self.assertFalse(pt.operational()) # adding a node in all partition uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) for x in xrange(num_partitions): pt.setCell(x, sn1, CellStates.UP_TO_DATE) self.assertTrue(pt.filled()) # it's up to date and running, so operational sn1.setState(NodeStates.RUNNING) self.assertTrue(pt.operational()) # same with feeding state pt.clear() self.assertFalse(pt.filled()) self.assertFalse(pt.operational()) # adding a node in all partition uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) for x in xrange(num_partitions): pt.setCell(x, sn1, CellStates.FEEDING) self.assertTrue(pt.filled()) # it's feeding and running, so operational sn1.setState(NodeStates.RUNNING) self.assertTrue(pt.operational()) # same with feeding state but non running node pt.clear() self.assertFalse(pt.filled()) self.assertFalse(pt.operational()) # adding a node in all partition uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) sn1.setState(NodeStates.TEMPORARILY_DOWN) for x in xrange(num_partitions): pt.setCell(x, sn1, CellStates.FEEDING) self.assertTrue(pt.filled()) # it's up to date and not running, so not operational self.assertFalse(pt.operational()) # same with out of date state and running pt.clear() self.assertFalse(pt.filled()) self.assertFalse(pt.operational()) # adding a node in all partition uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) for x in xrange(num_partitions): pt.setCell(x, sn1, CellStates.OUT_OF_DATE) self.assertTrue(pt.filled()) # it's not up to date and running, so not operational self.assertFalse(pt.operational())
class NodeManagerTests(NeoUnitTestBase): def setUp(self): NeoUnitTestBase.setUp(self) self.manager = NodeManager() def _addStorage(self): self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID()) def _addMaster(self): self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID()) def _addClient(self): self.client = ClientNode(self.manager, None, self.getClientUUID()) def _addAdmin(self): self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID()) def checkNodes(self, node_list): manager = self.manager self.assertEqual(sorted(manager.getList()), sorted(node_list)) def checkMasters(self, master_list): manager = self.manager self.assertEqual(manager.getMasterList(), master_list) def checkStorages(self, storage_list): manager = self.manager self.assertEqual(manager.getStorageList(), storage_list) def checkClients(self, client_list): manager = self.manager self.assertEqual(manager.getClientList(), client_list) def checkByServer(self, node): node_found = self.manager.getByAddress(node.getAddress()) self.assertEqual(node_found, node) def checkByUUID(self, node): node_found = self.manager.getByUUID(node.getUUID()) self.assertEqual(node_found, node) def checkIdentified(self, node_list, pool_set=None): identified_node_list = self.manager.getIdentifiedList(pool_set) self.assertEqual(set(identified_node_list), set(node_list)) def testInit(self): """ Check the manager is empty when started """ manager = self.manager self.checkNodes([]) self.checkMasters([]) self.checkStorages([]) self.checkClients([]) address = ('127.0.0.1', 10000) self.assertEqual(manager.getByAddress(address), None) self.assertEqual(manager.getByAddress(None), None) uuid = self.getNewUUID(None) self.assertEqual(manager.getByUUID(uuid), None) self.assertEqual(manager.getByUUID(None), None) def testAdd(self): """ Check if new nodes are registered in the manager """ manager = self.manager self.checkNodes([]) # storage self._addStorage() self.checkNodes([self.storage]) self.checkStorages([self.storage]) self.checkMasters([]) self.checkClients([]) self.checkByServer(self.storage) self.checkByUUID(self.storage) # master self._addMaster() self.checkNodes([self.storage, self.master]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([]) self.checkByServer(self.master) self.checkByUUID(self.master) # client self._addClient() self.checkNodes([self.storage, self.master, self.client]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) # client node has no address self.assertEqual(manager.getByAddress(self.client.getAddress()), None) self.checkByUUID(self.client) # admin self._addAdmin() self.checkNodes([self.storage, self.master, self.client, self.admin]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) self.checkByServer(self.admin) self.checkByUUID(self.admin) def testUpdate(self): """ Check manager content update """ # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() self.checkNodes([self.master, self.storage, self.client, self.admin]) self.checkMasters([self.master]) self.checkStorages([self.storage]) self.checkClients([self.client]) # build changes old_address = self.master.getAddress() new_address = ('127.0.0.1', 2001) old_uuid = self.storage.getUUID() new_uuid = self.getStorageUUID() node_list = ( (NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN), (NodeTypes.MASTER, new_address, self.master.getUUID(), NodeStates.RUNNING), (NodeTypes.STORAGE, self.storage.getAddress(), new_uuid, NodeStates.RUNNING), (NodeTypes.ADMIN, self.admin.getAddress(), self.admin.getUUID(), NodeStates.UNKNOWN), ) # update manager content manager.update(node_list) # - the client gets down self.checkClients([]) # - master change it's address self.checkMasters([self.master]) self.assertEqual(manager.getByAddress(old_address), None) self.master.setAddress(new_address) self.checkByServer(self.master) # - storage change it's UUID storage_list = manager.getStorageList() self.assertTrue(len(storage_list), 1) new_storage = storage_list[0] self.assertNotEqual(new_storage.getUUID(), old_uuid) self.assertEqual(new_storage.getState(), NodeStates.RUNNING) # admin is still here but in UNKNOWN state self.checkNodes([self.master, self.admin, new_storage]) self.assertEqual(self.admin.getState(), NodeStates.UNKNOWN) def testIdentified(self): # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() # switch node to connected self.checkIdentified([]) self.master.setConnection(Mock()) self.checkIdentified([self.master]) self.storage.setConnection(Mock()) self.checkIdentified([self.master, self.storage]) self.client.setConnection(Mock()) self.checkIdentified([self.master, self.storage, self.client]) self.admin.setConnection(Mock()) self.checkIdentified([self.master, self.storage, self.client, self.admin]) # check the pool_set attribute self.checkIdentified([self.master], pool_set=[self.master.getUUID()]) self.checkIdentified([self.storage], pool_set=[self.storage.getUUID()]) self.checkIdentified([self.client], pool_set=[self.client.getUUID()]) self.checkIdentified([self.admin], pool_set=[self.admin.getUUID()]) self.checkIdentified([self.master, self.storage], pool_set=[ self.master.getUUID(), self.storage.getUUID()])
def _addStorage(self): self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID())
def test_03_setCell(self): num_partitions = 5 num_replicas = 2 pt = PartitionTable(num_partitions, num_replicas) uuid1 = self.getStorageUUID() server1 = ("127.0.0.1", 19001) sn1 = StorageNode(Mock(), server1, uuid1) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0) # add a cell to an empty row self.assertFalse(pt.count_dict.has_key(sn1)) pt.setCell(0, sn1, CellStates.UP_TO_DATE) self.assertTrue(pt.count_dict.has_key(sn1)) self.assertEqual(pt.count_dict[sn1], 1) for x in xrange(num_partitions): if x == 0: self.assertEqual(len(pt.partition_list[x]), 1) cell = pt.partition_list[x][0] self.assertEqual(cell.getState(), CellStates.UP_TO_DATE) else: self.assertEqual(len(pt.partition_list[x]), 0) # try to add to an unexistant partition self.assertRaises(IndexError, pt.setCell, 10, sn1, CellStates.UP_TO_DATE) # if we add in discardes state, must be removed pt.setCell(0, sn1, CellStates.DISCARDED) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0) self.assertEqual(pt.count_dict[sn1], 0) # add a feeding node into empty row pt.setCell(0, sn1, CellStates.FEEDING) self.assertTrue(pt.count_dict.has_key(sn1)) self.assertEqual(pt.count_dict[sn1], 0) for x in xrange(num_partitions): if x == 0: self.assertEqual(len(pt.partition_list[x]), 1) cell = pt.partition_list[x][0] self.assertEqual(cell.getState(), CellStates.FEEDING) else: self.assertEqual(len(pt.partition_list[x]), 0) # re-add it as feeding, nothing change pt.setCell(0, sn1, CellStates.FEEDING) self.assertTrue(pt.count_dict.has_key(sn1)) self.assertEqual(pt.count_dict[sn1], 0) for x in xrange(num_partitions): if x == 0: self.assertEqual(len(pt.partition_list[x]), 1) cell = pt.partition_list[x][0] self.assertEqual(cell.getState(), CellStates.FEEDING) else: self.assertEqual(len(pt.partition_list[x]), 0) # now add it as up to date pt.setCell(0, sn1, CellStates.UP_TO_DATE) self.assertTrue(pt.count_dict.has_key(sn1)) self.assertEqual(pt.count_dict[sn1], 1) for x in xrange(num_partitions): if x == 0: self.assertEqual(len(pt.partition_list[x]), 1) cell = pt.partition_list[x][0] self.assertEqual(cell.getState(), CellStates.UP_TO_DATE) else: self.assertEqual(len(pt.partition_list[x]), 0) # now add broken and down state, must not be taken into account pt.setCell(0, sn1, CellStates.DISCARDED) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0) self.assertEqual(pt.count_dict[sn1], 0) sn1.setState(NodeStates.BROKEN) self.assertRaises(PartitionTableException, pt.setCell, 0, sn1, CellStates.UP_TO_DATE) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0) self.assertEqual(pt.count_dict[sn1], 0) sn1.setState(NodeStates.DOWN) self.assertRaises(PartitionTableException, pt.setCell, 0, sn1, CellStates.UP_TO_DATE) for x in xrange(num_partitions): self.assertEqual(len(pt.partition_list[x]), 0) self.assertEqual(pt.count_dict[sn1], 0)
class NodeManagerTests(NeoUnitTestBase): def setUp(self): NeoUnitTestBase.setUp(self) self.manager = NodeManager() def _addStorage(self): self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID()) def _addMaster(self): self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID()) def _addClient(self): self.client = ClientNode(self.manager, None, self.getClientUUID()) def _addAdmin(self): self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID()) def checkNodes(self, node_list): manager = self.manager self.assertEqual(sorted(manager.getList()), sorted(node_list)) def checkMasters(self, master_list): manager = self.manager self.assertEqual(manager.getMasterList(), master_list) def checkStorages(self, storage_list): manager = self.manager self.assertEqual(manager.getStorageList(), storage_list) def checkClients(self, client_list): manager = self.manager self.assertEqual(manager.getClientList(), client_list) def checkByServer(self, node): node_found = self.manager.getByAddress(node.getAddress()) self.assertEqual(node_found, node) def checkByUUID(self, node): node_found = self.manager.getByUUID(node.getUUID()) self.assertEqual(node_found, node) def checkIdentified(self, node_list, pool_set=None): identified_node_list = self.manager.getIdentifiedList(pool_set) self.assertEqual(set(identified_node_list), set(node_list)) def testInit(self): """ Check the manager is empty when started """ manager = self.manager self.checkNodes([]) self.checkMasters([]) self.checkStorages([]) self.checkClients([]) address = ('127.0.0.1', 10000) self.assertEqual(manager.getByAddress(address), None) self.assertEqual(manager.getByAddress(None), None) uuid = self.getNewUUID(None) self.assertEqual(manager.getByUUID(uuid), None) self.assertEqual(manager.getByUUID(None), None) def testAdd(self): """ Check if new nodes are registered in the manager """ manager = self.manager self.checkNodes([]) # storage self._addStorage() self.checkNodes([self.storage]) self.checkStorages([self.storage]) self.checkMasters([]) self.checkClients([]) self.checkByServer(self.storage) self.checkByUUID(self.storage) # master self._addMaster() self.checkNodes([self.storage, self.master]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([]) self.checkByServer(self.master) self.checkByUUID(self.master) # client self._addClient() self.checkNodes([self.storage, self.master, self.client]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) # client node has no address self.assertEqual(manager.getByAddress(self.client.getAddress()), None) self.checkByUUID(self.client) # admin self._addAdmin() self.checkNodes([self.storage, self.master, self.client, self.admin]) self.checkStorages([self.storage]) self.checkMasters([self.master]) self.checkClients([self.client]) self.checkByServer(self.admin) self.checkByUUID(self.admin) def testUpdate(self): """ Check manager content update """ # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() self.checkNodes([self.master, self.storage, self.client, self.admin]) self.checkMasters([self.master]) self.checkStorages([self.storage]) self.checkClients([self.client]) # build changes old_address = self.master.getAddress() new_address = ('127.0.0.1', 2001) old_uuid = self.storage.getUUID() new_uuid = self.getStorageUUID() node_list = ( (NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN), (NodeTypes.MASTER, new_address, self.master.getUUID(), NodeStates.RUNNING), (NodeTypes.STORAGE, self.storage.getAddress(), new_uuid, NodeStates.RUNNING), (NodeTypes.ADMIN, self.admin.getAddress(), self.admin.getUUID(), NodeStates.UNKNOWN), ) # update manager content manager.update(node_list) # - the client gets down self.checkClients([]) # - master change it's address self.checkMasters([self.master]) self.assertEqual(manager.getByAddress(old_address), None) self.master.setAddress(new_address) self.checkByServer(self.master) # - storage change it's UUID storage_list = manager.getStorageList() self.assertTrue(len(storage_list), 1) new_storage = storage_list[0] self.assertNotEqual(new_storage.getUUID(), old_uuid) self.assertEqual(new_storage.getState(), NodeStates.RUNNING) # admin is still here but in UNKNOWN state self.checkNodes([self.master, self.admin, new_storage]) self.assertEqual(self.admin.getState(), NodeStates.UNKNOWN) def testIdentified(self): # set up four nodes manager = self.manager self._addMaster() self._addStorage() self._addClient() self._addAdmin() # switch node to connected self.checkIdentified([]) self.master.setConnection(Mock()) self.checkIdentified([self.master]) self.storage.setConnection(Mock()) self.checkIdentified([self.master, self.storage]) self.client.setConnection(Mock()) self.checkIdentified([self.master, self.storage, self.client]) self.admin.setConnection(Mock()) self.checkIdentified( [self.master, self.storage, self.client, self.admin]) # check the pool_set attribute self.checkIdentified([self.master], pool_set=[self.master.getUUID()]) self.checkIdentified([self.storage], pool_set=[self.storage.getUUID()]) self.checkIdentified([self.client], pool_set=[self.client.getUUID()]) self.checkIdentified([self.admin], pool_set=[self.admin.getUUID()]) self.checkIdentified( [self.master, self.storage], pool_set=[self.master.getUUID(), self.storage.getUUID()])
def test_17_tweak(self): sn = [ StorageNode(Mock(), None, i + 1, NodeStates.RUNNING) for i in xrange(5) ] pt = PartitionTable(5, 2) # part 0 pt.setCell(0, sn[0], CellStates.DISCARDED) pt.setCell(0, sn[1], CellStates.UP_TO_DATE) # part 1 pt.setCell(1, sn[0], CellStates.FEEDING) pt.setCell(1, sn[1], CellStates.FEEDING) pt.setCell(1, sn[2], CellStates.OUT_OF_DATE) # part 2 pt.setCell(2, sn[0], CellStates.FEEDING) pt.setCell(2, sn[1], CellStates.UP_TO_DATE) pt.setCell(2, sn[2], CellStates.UP_TO_DATE) # part 3 pt.setCell(3, sn[0], CellStates.UP_TO_DATE) pt.setCell(3, sn[1], CellStates.UP_TO_DATE) pt.setCell(3, sn[2], CellStates.UP_TO_DATE) pt.setCell(3, sn[3], CellStates.UP_TO_DATE) # part 4 pt.setCell(4, sn[0], CellStates.UP_TO_DATE) pt.setCell(4, sn[4], CellStates.UP_TO_DATE) count_dict = defaultdict(int) change_list = self.tweak(pt) for offset, uuid, state in change_list: count_dict[state] += 1 self.assertEqual( count_dict, { CellStates.DISCARDED: 3, CellStates.OUT_OF_DATE: 5, CellStates.UP_TO_DATE: 3 }) self.update(pt, change_list) self.checkPT(pt) self.assertRaises(PartitionTableException, pt.dropNodeList, sn[1:4]) self.assertEqual(6, len(pt.dropNodeList(sn[1:3], True))) self.assertEqual(3, len(pt.dropNodeList([sn[1]]))) pt.addNodeList([sn[1]]) change_list = self.tweak(pt) self.assertEqual(3, len(change_list)) self.update(pt, change_list) self.checkPT(pt) for np, i in (12, 0), (12, 1), (13, 2): pt = PartitionTable(np, i) i += 1 pt.make(sn[:i]) for n in sn[i:i + 3]: self.assertEqual([n], pt.addNodeList([n])) self.update(pt, self.tweak(pt)) self.checkPT(pt) pt.clear() pt.make(sn[:i]) for n in sn[i:i + 3]: self.assertEqual([n], pt.addNodeList([n])) self.tweak(pt) self.update(pt) self.checkPT(pt) pt = PartitionTable(7, 0) pt.make(sn[:1]) pt.addNodeList(sn[1:3]) self.update(pt, self.tweak(pt, sn[:1])) self.checkPT(pt, True)