def formatPartitionTable(self, row_list):
nm = NodeManager()
nm.update(dummy_app, 1,
self.neoctl.getNodeList(node_type=NodeTypes.STORAGE))
pt = object.__new__(PartitionTable)
pt._load(None, None, row_list, nm.getByUUID)
pt.addNodeList(nm.getByStateList(NodeStates.RUNNING))
return '\n'.join(line[4:] for line in pt._format())
def setUp(self):
super(MasterHandlerTests, self).setUp()
self.db = Mock()
self.app = Mock({'getDB': self.db, 'txn_contexts': ()})
self.app.nm = NodeManager()
self.app.dispatcher = Mock()
self._next_port = 3000
def __init__(self, app, name, master_addresses):
self.app = weakref.proxy(app)
self.name = name
self.nm = NodeManager()
for master_address in master_addresses:
self.nm.createMaster(address=master_address)
class BackupApplication(object):
pt = None
server = None # like in BaseApplication
uuid = None
def __init__(self, app, name, master_addresses):
self.app = weakref.proxy(app)
self.name = name
self.nm = NodeManager()
for master_address in master_addresses:
self.nm.createMaster(address=master_address)
em = property(lambda self: self.app.em)
ssl = property(lambda self: self.app.ssl)
def close(self):
self.nm.close()
del self.__dict__
def setUUID(self, uuid):
if self.uuid != uuid:
self.uuid = uuid
logging.info('Upstream Node ID: %s', uuid_str(uuid))
def log(self):
self.nm.log()
if self.pt is not None:
self.pt.log()
def provideService(self):
logging.info('provide backup')
poll = self.em.poll
app = self.app
pt = app.pt
while True:
app.changeClusterState(ClusterStates.STARTING_BACKUP)
bootstrap = BootstrapManager(self,
NodeTypes.CLIENT,
backup=app.name)
# {offset -> node}
self.primary_partition_dict = {}
# [[tid]]
self.tid_list = tuple([] for _ in xrange(pt.getPartitions()))
try:
while True:
for node in pt.getNodeSet(readable=True):
if not app.isStorageReady(node.getUUID()):
break
else:
break
poll(1)
node, conn = bootstrap.getPrimaryConnection()
try:
app.changeClusterState(ClusterStates.BACKINGUP)
del bootstrap, node
self.ignore_invalidations = True
conn.setHandler(BackupHandler(self))
conn.ask(Packets.AskLastTransaction())
# debug variable to log how big 'tid_list' can be.
self.debug_tid_count = 0
while True:
poll(1)
except PrimaryFailure, msg:
logging.error('upstream master is down: %s', msg)
finally:
app.backup_tid = pt.getBackupTid()
try:
conn.close()
except PrimaryFailure:
pass
try:
del self.pt
except AttributeError:
pass
for node in app.nm.getClientList(True):
node.getConnection().close()
except StateChangedException, e:
if e.args[0] != ClusterStates.STOPPING_BACKUP:
raise
app.changeClusterState(*e.args)
tid = app.backup_tid
# Wait for non-primary partitions to catch up,
# so that all UP_TO_DATE cells are really UP_TO_DATE.
# XXX: Another possibility could be to outdate such cells, and
# they would be quickly updated at the beginning of the
# RUNNING phase. This may simplify code.
# Any unfinished replication from upstream will be truncated.
while pt.getBackupTid(min) < tid:
poll(1)
last_tid = app.getLastTransaction()
handler = EventHandler(app)
if tid < last_tid:
assert tid != ZERO_TID
logging.warning("Truncating at %s (last_tid was %s)",
dump(app.backup_tid), dump(last_tid))
else:
# We will do a dummy truncation, just to leave backup mode,
# so it's fine to start automatically if there's any
# missing storage.
# XXX: Consider using another method to leave backup mode,
# at least when there's nothing to truncate. Because
# in case of StoppedOperation during VERIFYING state,
# this flag will be wrongly set to False.
app._startup_allowed = True
# If any error happened before reaching this line, we'd go back
# to backup mode, which is the right mode to recover.
del app.backup_tid
# Now back to RECOVERY...
return tid
finally:
def setUp(self):
NeoUnitTestBase.setUp(self)
self.manager = NodeManager()
class NodeManagerTests(NeoUnitTestBase):
def setUp(self):
NeoUnitTestBase.setUp(self)
self.manager = NodeManager()
def _addStorage(self):
self.storage = StorageNode(self.manager, ('127.0.0.1', 1000), self.getStorageUUID())
def _addMaster(self):
self.master = MasterNode(self.manager, ('127.0.0.1', 2000), self.getMasterUUID())
def _addClient(self):
self.client = ClientNode(self.manager, None, self.getClientUUID())
def _addAdmin(self):
self.admin = AdminNode(self.manager, ('127.0.0.1', 4000), self.getAdminUUID())
def checkNodes(self, node_list):
manager = self.manager
self.assertEqual(sorted(manager.getList()), sorted(node_list))
def checkMasters(self, master_list):
manager = self.manager
self.assertEqual(manager.getMasterList(), master_list)
def checkStorages(self, storage_list):
manager = self.manager
self.assertEqual(manager.getStorageList(), storage_list)
def checkClients(self, client_list):
manager = self.manager
self.assertEqual(manager.getClientList(), client_list)
def checkByServer(self, node):
node_found = self.manager.getByAddress(node.getAddress())
self.assertEqual(node_found, node)
def checkByUUID(self, node):
node_found = self.manager.getByUUID(node.getUUID())
self.assertEqual(node_found, node)
def checkIdentified(self, node_list, pool_set=None):
identified_node_list = self.manager.getIdentifiedList(pool_set)
self.assertEqual(set(identified_node_list), set(node_list))
def testInit(self):
""" Check the manager is empty when started """
manager = self.manager
self.checkNodes([])
self.checkMasters([])
self.checkStorages([])
self.checkClients([])
address = ('127.0.0.1', 10000)
self.assertEqual(manager.getByAddress(address), None)
self.assertEqual(manager.getByAddress(None), None)
uuid = self.getNewUUID(None)
self.assertEqual(manager.getByUUID(uuid), None)
self.assertEqual(manager.getByUUID(None), None)
def testAdd(self):
""" Check if new nodes are registered in the manager """
manager = self.manager
self.checkNodes([])
# storage
self._addStorage()
self.checkNodes([self.storage])
self.checkStorages([self.storage])
self.checkMasters([])
self.checkClients([])
self.checkByServer(self.storage)
self.checkByUUID(self.storage)
# master
self._addMaster()
self.checkNodes([self.storage, self.master])
self.checkStorages([self.storage])
self.checkMasters([self.master])
self.checkClients([])
self.checkByServer(self.master)
self.checkByUUID(self.master)
# client
self._addClient()
self.checkNodes([self.storage, self.master, self.client])
self.checkStorages([self.storage])
self.checkMasters([self.master])
self.checkClients([self.client])
# client node has no address
self.assertEqual(manager.getByAddress(self.client.getAddress()), None)
self.checkByUUID(self.client)
# admin
self._addAdmin()
self.checkNodes([self.storage, self.master, self.client, self.admin])
self.checkStorages([self.storage])
self.checkMasters([self.master])
self.checkClients([self.client])
self.checkByServer(self.admin)
self.checkByUUID(self.admin)
def testUpdate(self):
""" Check manager content update """
# set up four nodes
manager = self.manager
self._addMaster()
self._addStorage()
self._addClient()
self._addAdmin()
self.checkNodes([self.master, self.storage, self.client, self.admin])
self.checkMasters([self.master])
self.checkStorages([self.storage])
self.checkClients([self.client])
# build changes
old_address = self.master.getAddress()
new_address = ('127.0.0.1', 2001)
old_uuid = self.storage.getUUID()
new_uuid = self.getStorageUUID()
node_list = (
(NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN),
(NodeTypes.MASTER, new_address, self.master.getUUID(), NodeStates.RUNNING),
(NodeTypes.STORAGE, self.storage.getAddress(), new_uuid,
NodeStates.RUNNING),
(NodeTypes.ADMIN, self.admin.getAddress(), self.admin.getUUID(),
NodeStates.UNKNOWN),
)
# update manager content
manager.update(node_list)
# - the client gets down
self.checkClients([])
# - master change it's address
self.checkMasters([self.master])
self.assertEqual(manager.getByAddress(old_address), None)
self.master.setAddress(new_address)
self.checkByServer(self.master)
# - storage change it's UUID
storage_list = manager.getStorageList()
self.assertTrue(len(storage_list), 1)
new_storage = storage_list[0]
self.assertNotEqual(new_storage.getUUID(), old_uuid)
self.assertEqual(new_storage.getState(), NodeStates.RUNNING)
# admin is still here but in UNKNOWN state
self.checkNodes([self.master, self.admin, new_storage])
self.assertEqual(self.admin.getState(), NodeStates.UNKNOWN)
def testIdentified(self):
# set up four nodes
manager = self.manager
self._addMaster()
self._addStorage()
self._addClient()
self._addAdmin()
# switch node to connected
self.checkIdentified([])
self.master.setConnection(Mock())
self.checkIdentified([self.master])
self.storage.setConnection(Mock())
self.checkIdentified([self.master, self.storage])
self.client.setConnection(Mock())
self.checkIdentified([self.master, self.storage, self.client])
self.admin.setConnection(Mock())
self.checkIdentified([self.master, self.storage, self.client, self.admin])
# check the pool_set attribute
self.checkIdentified([self.master], pool_set=[self.master.getUUID()])
self.checkIdentified([self.storage], pool_set=[self.storage.getUUID()])
self.checkIdentified([self.client], pool_set=[self.client.getUUID()])
self.checkIdentified([self.admin], pool_set=[self.admin.getUUID()])
self.checkIdentified([self.master, self.storage], pool_set=[
self.master.getUUID(), self.storage.getUUID()])
def setUp(self):
NeoUnitTestBase.setUp(self)
self.manager = NodeManager()
class NodeManagerTests(NeoUnitTestBase):
def setUp(self):
NeoUnitTestBase.setUp(self)
self.manager = NodeManager()
def _addStorage(self):
self.storage = StorageNode(self.manager, ('127.0.0.1', 1000),
self.getStorageUUID())
def _addMaster(self):
self.master = MasterNode(self.manager, ('127.0.0.1', 2000),
self.getMasterUUID())
def _addClient(self):
self.client = ClientNode(self.manager, None, self.getClientUUID())
def _addAdmin(self):
self.admin = AdminNode(self.manager, ('127.0.0.1', 4000),
self.getAdminUUID())
def checkNodes(self, node_list):
manager = self.manager
self.assertEqual(sorted(manager.getList()), sorted(node_list))
def checkMasters(self, master_list):
manager = self.manager
self.assertEqual(manager.getMasterList(), master_list)
def checkStorages(self, storage_list):
manager = self.manager
self.assertEqual(manager.getStorageList(), storage_list)
def checkClients(self, client_list):
manager = self.manager
self.assertEqual(manager.getClientList(), client_list)
def checkByServer(self, node):
node_found = self.manager.getByAddress(node.getAddress())
self.assertEqual(node_found, node)
def checkByUUID(self, node):
node_found = self.manager.getByUUID(node.getUUID())
self.assertEqual(node_found, node)
def checkIdentified(self, node_list, pool_set=None):
identified_node_list = self.manager.getIdentifiedList(pool_set)
self.assertEqual(set(identified_node_list), set(node_list))
def testInit(self):
""" Check the manager is empty when started """
manager = self.manager
self.checkNodes([])
self.checkMasters([])
self.checkStorages([])
self.checkClients([])
address = ('127.0.0.1', 10000)
self.assertEqual(manager.getByAddress(address), None)
self.assertEqual(manager.getByAddress(None), None)
uuid = self.getNewUUID(None)
self.assertEqual(manager.getByUUID(uuid), None)
self.assertEqual(manager.getByUUID(None), None)
def testAdd(self):
""" Check if new nodes are registered in the manager """
manager = self.manager
self.checkNodes([])
# storage
self._addStorage()
self.checkNodes([self.storage])
self.checkStorages([self.storage])
self.checkMasters([])
self.checkClients([])
self.checkByServer(self.storage)
self.checkByUUID(self.storage)
# master
self._addMaster()
self.checkNodes([self.storage, self.master])
self.checkStorages([self.storage])
self.checkMasters([self.master])
self.checkClients([])
self.checkByServer(self.master)
self.checkByUUID(self.master)
# client
self._addClient()
self.checkNodes([self.storage, self.master, self.client])
self.checkStorages([self.storage])
self.checkMasters([self.master])
self.checkClients([self.client])
# client node has no address
self.assertEqual(manager.getByAddress(self.client.getAddress()), None)
self.checkByUUID(self.client)
# admin
self._addAdmin()
self.checkNodes([self.storage, self.master, self.client, self.admin])
self.checkStorages([self.storage])
self.checkMasters([self.master])
self.checkClients([self.client])
self.checkByServer(self.admin)
self.checkByUUID(self.admin)
def testUpdate(self):
""" Check manager content update """
# set up four nodes
manager = self.manager
self._addMaster()
self._addStorage()
self._addClient()
self._addAdmin()
self.checkNodes([self.master, self.storage, self.client, self.admin])
self.checkMasters([self.master])
self.checkStorages([self.storage])
self.checkClients([self.client])
# build changes
old_address = self.master.getAddress()
new_address = ('127.0.0.1', 2001)
old_uuid = self.storage.getUUID()
new_uuid = self.getStorageUUID()
node_list = (
(NodeTypes.CLIENT, None, self.client.getUUID(), NodeStates.DOWN),
(NodeTypes.MASTER, new_address, self.master.getUUID(),
NodeStates.RUNNING),
(NodeTypes.STORAGE, self.storage.getAddress(), new_uuid,
NodeStates.RUNNING),
(NodeTypes.ADMIN, self.admin.getAddress(), self.admin.getUUID(),
NodeStates.UNKNOWN),
)
# update manager content
manager.update(node_list)
# - the client gets down
self.checkClients([])
# - master change it's address
self.checkMasters([self.master])
self.assertEqual(manager.getByAddress(old_address), None)
self.master.setAddress(new_address)
self.checkByServer(self.master)
# - storage change it's UUID
storage_list = manager.getStorageList()
self.assertTrue(len(storage_list), 1)
new_storage = storage_list[0]
self.assertNotEqual(new_storage.getUUID(), old_uuid)
self.assertEqual(new_storage.getState(), NodeStates.RUNNING)
# admin is still here but in UNKNOWN state
self.checkNodes([self.master, self.admin, new_storage])
self.assertEqual(self.admin.getState(), NodeStates.UNKNOWN)
def testIdentified(self):
# set up four nodes
manager = self.manager
self._addMaster()
self._addStorage()
self._addClient()
self._addAdmin()
# switch node to connected
self.checkIdentified([])
self.master.setConnection(Mock())
self.checkIdentified([self.master])
self.storage.setConnection(Mock())
self.checkIdentified([self.master, self.storage])
self.client.setConnection(Mock())
self.checkIdentified([self.master, self.storage, self.client])
self.admin.setConnection(Mock())
self.checkIdentified(
[self.master, self.storage, self.client, self.admin])
# check the pool_set attribute
self.checkIdentified([self.master], pool_set=[self.master.getUUID()])
self.checkIdentified([self.storage], pool_set=[self.storage.getUUID()])
self.checkIdentified([self.client], pool_set=[self.client.getUUID()])
self.checkIdentified([self.admin], pool_set=[self.admin.getUUID()])
self.checkIdentified(
[self.master, self.storage],
pool_set=[self.master.getUUID(),
self.storage.getUUID()])
def __init__(self, app, name, master_addresses):
self.app = weakref.proxy(app)
self.name = name
self.nm = NodeManager()
for master_address in master_addresses:
self.nm.createMaster(address=master_address)
class BackupApplication(object):
pt = None
def __init__(self, app, name, master_addresses):
self.app = weakref.proxy(app)
self.name = name
self.nm = NodeManager()
for master_address in master_addresses:
self.nm.createMaster(address=master_address)
em = property(lambda self: self.app.em)
ssl = property(lambda self: self.app.ssl)
def close(self):
self.nm.close()
del self.__dict__
def log(self):
self.nm.log()
if self.pt is not None:
self.pt.log()
def provideService(self):
logging.info('provide backup')
poll = self.em.poll
app = self.app
pt = app.pt
while True:
app.changeClusterState(ClusterStates.STARTING_BACKUP)
bootstrap = BootstrapManager(self, self.name, NodeTypes.CLIENT)
# {offset -> node}
self.primary_partition_dict = {}
# [[tid]]
self.tid_list = tuple([] for _ in xrange(pt.getPartitions()))
try:
while True:
for node in pt.getNodeSet(readable=True):
if not app.isStorageReady(node.getUUID()):
break
else:
break
poll(1)
node, conn, uuid, num_partitions, num_replicas = \
bootstrap.getPrimaryConnection()
try:
app.changeClusterState(ClusterStates.BACKINGUP)
del bootstrap, node
if num_partitions != pt.getPartitions():
raise RuntimeError("inconsistent number of partitions")
self.pt = PartitionTable(num_partitions, num_replicas)
conn.setHandler(BackupHandler(self))
conn.ask(Packets.AskNodeInformation())
conn.ask(Packets.AskPartitionTable())
conn.ask(Packets.AskLastTransaction())
# debug variable to log how big 'tid_list' can be.
self.debug_tid_count = 0
while True:
poll(1)
except PrimaryFailure, msg:
logging.error('upstream master is down: %s', msg)
finally:
app.backup_tid = pt.getBackupTid()
try:
conn.close()
except PrimaryFailure:
pass
try:
del self.pt
except AttributeError:
pass
except StateChangedException, e:
if e.args[0] != ClusterStates.STOPPING_BACKUP:
raise
app.changeClusterState(*e.args)
tid = app.backup_tid
# Wait for non-primary partitions to catch up,
# so that all UP_TO_DATE cells are really UP_TO_DATE.
# XXX: Another possibility could be to outdate such cells, and
# they would be quickly updated at the beginning of the
# RUNNING phase. This may simplify code.
# Any unfinished replication from upstream will be truncated.
while pt.getBackupTid(min) < tid:
poll(1)
last_tid = app.getLastTransaction()
handler = EventHandler(app)
if tid < last_tid:
assert tid != ZERO_TID
logging.warning("Truncating at %s (last_tid was %s)",
dump(app.backup_tid), dump(last_tid))
else:
# We will do a dummy truncation, just to leave backup mode,
# so it's fine to start automatically if there's any
# missing storage.
# XXX: Consider using another method to leave backup mode,
# at least when there's nothing to truncate. Because
# in case of StoppedOperation during VERIFYING state,
# this flag will be wrongly set to False.
app._startup_allowed = True
# If any error happened before reaching this line, we'd go back
# to backup mode, which is the right mode to recover.
del app.backup_tid
# Now back to RECOVERY...
return tid
finally:
