def test_multi_node(self):
base_port = Configuration.get('ovs.ports.arakoon')[0]
cluster = 'one'
nodes = sorted(TestArakoonInstaller.nodes.keys())
nodes = dict((node, SSHClient(node)) for node in nodes)
first_node = nodes.keys()[0]
ArakoonInstaller.create_cluster(cluster, first_node, [])
for node in nodes[1:]:
ArakoonInstaller.extend_cluster(first_node, node, cluster, [])
expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes))
for node in nodes:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
expected = expected.strip()
for node, client in nodes.iteritems():
contents = client.file_read(self._get_config_path(cluster))
self.assertEqual(contents.strip(), expected.strip())
ArakoonInstaller.shrink_cluster(nodes[1], first_node, cluster)
expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:]))
for node in nodes.keys()[1:]:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
expected = expected.strip()
for node, client in nodes.iteritems():
if node == first_node:
continue
contents = client.file_read(self._get_config_path(cluster))
self.assertEqual(contents.strip(), expected.strip())
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:param master_ip: IP of the master node
:param offline_node_ips: IPs of nodes which are offline
"""
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(cluster_ip, username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name('Arakoon')
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv':
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
print '* Shrink StorageDriver cluster'
ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv', offline_node_ips)
if client is not None and ServiceManager.has_service(current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name, client=client)
ServiceManager.remove_service(current_service.name, client=client)
ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver(offline_node_ips)
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:type cluster_ip: str
:param master_ip: IP of the master node
:type master_ip: str
:param offline_node_ips: IPs of nodes which are offline
:type offline_node_ips: list
:return: None
"""
_ = master_ip
if offline_node_ips is None:
offline_node_ips = []
servicetype = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv' and service.is_internal is True: # Externally managed arakoon cluster service does not have storage router
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
if len(remaining_ips) == 0:
raise RuntimeError('Could not find any remaining arakoon nodes for the voldrv cluster')
StorageDriverController._logger.debug('* Shrink StorageDriver cluster')
cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|voldrv'))
ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip,
remaining_node_ips=remaining_ips,
cluster_name=cluster_name,
offline_nodes=offline_node_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def on_demote(cluster_ip, master_ip):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:param master_ip: IP of the master node
"""
client = SSHClient(cluster_ip, username='******')
servicetype = ServiceTypeList.get_by_name('Arakoon')
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv':
if service.storagerouter.ip == cluster_ip:
current_service = service
else:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
print '* Shrink StorageDriver cluster'
ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv')
if ServiceManager.has_service(current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name, client=client)
ServiceManager.remove_service(current_service.name, client=client)
ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips)
current_service.delete()
for storagerouter in StorageRouterList.get_storagerouters():
ArakoonInstaller.deploy_to_slave(master_ip, storagerouter.ip, 'voldrv')
StorageDriverController._configure_arakoon_to_volumedriver()
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:param master_ip: IP of the master node
:param offline_node_ips: IPs of nodes which are offline
"""
_ = master_ip
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(
cluster_ip,
username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name('Arakoon')
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv':
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
print '* Shrink StorageDriver cluster'
ArakoonInstaller.shrink_cluster(cluster_ip, 'voldrv',
offline_node_ips)
if client is not None and ServiceManager.has_service(
current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name,
client=client)
ServiceManager.remove_service(current_service.name,
client=client)
ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver()
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:type cluster_ip: str
:param master_ip: IP of the master node
:type master_ip: str
:param offline_node_ips: IPs of nodes which are offline
:type offline_node_ips: list
:return: None
"""
_ = master_ip
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(
cluster_ip,
username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name(
ServiceType.SERVICE_TYPES.ARAKOON)
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv' and service.is_internal is True: # Externally managed arakoon cluster service does not have storage router
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
StorageDriverController._logger.debug(
'* Shrink StorageDriver cluster')
cluster_name = str(
EtcdConfiguration.get(
'/ovs/framework/arakoon_clusters|voldrv'))
ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip,
cluster_name=cluster_name,
offline_nodes=offline_node_ips)
if client is not None and ServiceManager.has_service(
current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name,
client=client)
ServiceManager.remove_service(current_service.name,
client=client)
ArakoonInstaller.restart_cluster_remove(cluster_name,
remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver(
cluster_name=cluster_name)
def test_multi_node(self):
nodes = sorted(TestArakoonInstaller.nodes.keys())
nodes = dict((node, SSHClient(node)) for node in nodes)
first_node = nodes.keys()[0]
result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, first_node, "/tmp/db")
for node in nodes.keys()[1:]:
ArakoonInstaller.extend_cluster(first_node, node, TestArakoonInstaller.cluster_name, "/tmp/db")
expected = TestArakoonInstaller.expected_global.format(
",".join(TestArakoonInstaller.nodes[node] for node in nodes), TestArakoonInstaller.cluster_name
)
for node in nodes:
expected += TestArakoonInstaller.expected_base.format(
TestArakoonInstaller.nodes[node],
result["client_port"],
result["messaging_port"],
TestArakoonInstaller.cluster_name,
node,
)
expected = expected.strip()
for node, client in nodes.iteritems():
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
ArakoonInstaller.shrink_cluster(first_node, TestArakoonInstaller.cluster_name)
expected = TestArakoonInstaller.expected_global.format(
",".join(TestArakoonInstaller.nodes[node] for node in nodes[1:]), TestArakoonInstaller.cluster_name
)
for node in nodes.keys()[1:]:
expected += TestArakoonInstaller.expected_base.format(
TestArakoonInstaller.nodes[node],
result["client_port"],
result["messaging_port"],
TestArakoonInstaller.cluster_name,
node,
)
expected = expected.strip()
for node, client in nodes.iteritems():
if node == first_node:
continue
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
def on_demote(cluster_ip, master_ip, offline_node_ips=None):
"""
Handles the demote for the StorageDrivers
:param cluster_ip: IP of the node to demote
:type cluster_ip: str
:param master_ip: IP of the master node
:type master_ip: str
:param offline_node_ips: IPs of nodes which are offline
:type offline_node_ips: list
:return: None
"""
_ = master_ip
if offline_node_ips is None:
offline_node_ips = []
client = SSHClient(cluster_ip, username='******') if cluster_ip not in offline_node_ips else None
servicetype = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)
current_service = None
remaining_ips = []
for service in servicetype.services:
if service.name == 'arakoon-voldrv' and service.is_internal is True: # Externally managed arakoon cluster service does not have storage router
if service.storagerouter.ip == cluster_ip:
current_service = service
elif service.storagerouter.ip not in offline_node_ips:
remaining_ips.append(service.storagerouter.ip)
if current_service is not None:
StorageDriverController._logger.debug('* Shrink StorageDriver cluster')
cluster_name = str(EtcdConfiguration.get('/ovs/framework/arakoon_clusters|voldrv'))
ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip,
cluster_name=cluster_name,
offline_nodes=offline_node_ips)
if client is not None and ServiceManager.has_service(current_service.name, client=client) is True:
ServiceManager.stop_service(current_service.name, client=client)
ServiceManager.remove_service(current_service.name, client=client)
ArakoonInstaller.restart_cluster_remove(cluster_name, remaining_ips)
current_service.delete()
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def test_multi_node(self):
nodes = sorted(TestArakoonInstaller.nodes.keys())
nodes = dict((node, SSHClient(node)) for node in nodes)
first_node = nodes.keys()[0]
result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, first_node, '/tmp/db')
for node in nodes.keys()[1:]:
ArakoonInstaller.extend_cluster(first_node, node, TestArakoonInstaller.cluster_name, '/tmp/db')
expected = TestArakoonInstaller.expected_global.format(','.join(TestArakoonInstaller.nodes[node] for node in nodes), TestArakoonInstaller.cluster_name)
for node in nodes:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node)
expected = expected.strip()
for node, client in nodes.iteritems():
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
ArakoonInstaller.shrink_cluster(first_node, TestArakoonInstaller.cluster_name)
expected = TestArakoonInstaller.expected_global.format(','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:]), TestArakoonInstaller.cluster_name)
for node in nodes.keys()[1:]:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node)
expected = expected.strip()
for node, client in nodes.iteritems():
if node == first_node:
continue
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
def test_cluster_maintenance(self):
"""
Validates whether a cluster can be correctly created
"""
Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]})
Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]})
structure = Helper.build_service_structure(
{'storagerouters': [1, 2]}
)
storagerouters = structure['storagerouters']
System._machine_id = {storagerouters[1].ip: '1',
storagerouters[2].ip: '2'}
# Create new cluster
mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir = mountpoint + '/test_create_cluster'
info = ArakoonInstaller.create_cluster('test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir)
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
'files': []},
'db': {'dirs': {},
'files': []}},
'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[1].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))
ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata'])
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
# Extending cluster
mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir2 = mountpoint + '/test_extend_cluster'
ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2)
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
'files': []},
'db': {'dirs': {},
'files': []}},
'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n{2}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1,2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001
),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
SSHClient._run_returns[catchup_command] = None
SSHClient._run_recordings = []
ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False)
self.assertIn(catchup_command, SSHClient._run_recordings)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
# Shrinking cluster
ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test')
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {}, 'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
def ar_0001_validate_create_extend_shrink_delete_cluster_test():
"""
Validate extending and shrinking of arakoon clusters
"""
storagerouters = GeneralStorageRouter.get_storage_routers()
if not len(storagerouters) >= 3:
TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters)))
return
cluster_name = 'ar_0001'
cluster_basedir = '/var/tmp/'
first_sr = storagerouters[0]
second_sr = storagerouters[1]
third_sr = storagerouters[2]
first_root_client = SSHClient(first_sr, username='******')
second_root_client = SSHClient(second_sr, username='******')
third_root_client = SSHClient(third_sr, username='******')
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate single node cluster')
create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, first_sr.ip,
cluster_basedir, filesystem=False)
TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
ArakoonInstaller.start_cluster(cluster_name, first_sr.ip, False)
ArakoonInstaller.claim_cluster(cluster_name, first_sr, False, metadata=create_info['metadata'])
TestArakoon.validate_arakoon_config_files([first_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate two node cluster')
ArakoonInstaller.extend_cluster(first_sr.ip, second_sr.ip, cluster_name, cluster_basedir)
TestArakoon.validate_arakoon_config_files([first_sr, second_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate three node cluster')
ArakoonInstaller.extend_cluster(first_sr.ip, third_sr.ip, cluster_name, cluster_basedir)
TestArakoon.validate_arakoon_config_files([first_sr, second_sr, third_sr], cluster_name)
for client in [first_root_client, second_root_client, third_root_client]:
TestArakoon.verify_arakoon_structure(client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('reduce and validate three node to two node cluster')
ArakoonInstaller.shrink_cluster(second_sr.ip, first_sr.ip, cluster_name)
TestArakoon.validate_arakoon_config_files([first_sr, third_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False)
TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('reduce and validate two node to one node cluster')
ArakoonInstaller.shrink_cluster(first_sr.ip, third_sr.ip, cluster_name)
TestArakoon.validate_arakoon_config_files([third_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, False)
TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False)
TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('remove cluster')
ArakoonInstaller.delete_cluster(cluster_name, third_sr.ip, False)
for client in [first_root_client, second_root_client, third_root_client]:
TestArakoon.verify_arakoon_structure(client, cluster_name, False, False)
GeneralArakoon.delete_config(cluster_name)
def demote_node(cluster_ip, master_ip, ip_client_map, unique_id, unconfigure_memcached, unconfigure_rabbitmq, offline_nodes=None):
"""
Demotes a given node
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
Toolbox.log(logger=NodeTypeController._logger, messages='Demoting node', title=True)
if offline_nodes is None:
offline_nodes = []
if unconfigure_memcached is True and len(offline_nodes) == 0:
if NodeTypeController._validate_local_memcache_servers(ip_client_map) is False:
raise RuntimeError('Not all memcache nodes can be reached which is required for demoting a node.')
# Find other (arakoon) master nodes
arakoon_cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|ovsdb'))
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=arakoon_cluster_name)
config = ArakoonClusterConfig(cluster_id=arakoon_cluster_name, filesystem=False)
config.load_config()
master_node_ips = [node.ip for node in config.nodes]
if cluster_ip in master_node_ips:
master_node_ips.remove(cluster_ip)
if len(master_node_ips) == 0:
raise RuntimeError('There should be at least one other master node')
storagerouter = StorageRouterList.get_by_machine_id(unique_id)
storagerouter.node_type = 'EXTRA'
storagerouter.save()
offline_node_ips = [node.ip for node in offline_nodes]
if arakoon_metadata['internal'] is True:
Toolbox.log(logger=NodeTypeController._logger, messages='Leaving Arakoon {0} cluster'.format(arakoon_cluster_name))
ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip,
remaining_node_ips=master_node_ips,
cluster_name=arakoon_cluster_name,
offline_nodes=offline_node_ips)
try:
external_config = Configuration.get('/ovs/framework/external_config')
if external_config is None:
config_store = Configuration.get_store()
if config_store == 'arakoon':
Toolbox.log(logger=NodeTypeController._logger, messages='Leaving Arakoon config cluster')
ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip,
remaining_node_ips=master_node_ips,
cluster_name='config',
offline_nodes=offline_node_ips,
filesystem=True)
else:
from ovs.extensions.db.etcd.installer import EtcdInstaller
Toolbox.log(logger=NodeTypeController._logger, messages='Leaving Etcd cluster')
EtcdInstaller.shrink_cluster(master_ip, cluster_ip, 'config', offline_node_ips)
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger, messages=['\nFailed to leave configuration cluster', ex], loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger, messages='Update configurations')
try:
if unconfigure_memcached is True:
endpoints = Configuration.get('/ovs/framework/memcache|endpoints')
endpoint = '{0}:{1}'.format(cluster_ip, 11211)
if endpoint in endpoints:
endpoints.remove(endpoint)
Configuration.set('/ovs/framework/memcache|endpoints', endpoints)
if unconfigure_rabbitmq is True:
endpoints = Configuration.get('/ovs/framework/messagequeue|endpoints')
endpoint = '{0}:{1}'.format(cluster_ip, 5672)
if endpoint in endpoints:
endpoints.remove(endpoint)
Configuration.set('/ovs/framework/messagequeue|endpoints', endpoints)
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger, messages=['\nFailed to update configurations', ex], loglevel='exception')
if arakoon_metadata['internal'] is True:
Toolbox.log(logger=NodeTypeController._logger, messages='Restarting master node services')
remaining_nodes = ip_client_map.keys()[:]
if cluster_ip in remaining_nodes:
remaining_nodes.remove(cluster_ip)
PersistentFactory.store = None
VolatileFactory.store = None
for service in storagerouter.services:
if service.name == 'arakoon-ovsdb':
service.delete()
target_client = None
if storagerouter in offline_nodes:
if unconfigure_rabbitmq is True:
Toolbox.log(logger=NodeTypeController._logger, messages='Removing/unconfiguring offline RabbitMQ node')
client = ip_client_map[master_ip]
try:
client.run(['rabbitmqctl', 'forget_cluster_node', 'rabbit@{0}'.format(storagerouter.name)])
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger, messages=['\nFailed to forget RabbitMQ cluster node', ex], loglevel='exception')
else:
target_client = ip_client_map[cluster_ip]
if unconfigure_rabbitmq is True:
Toolbox.log(logger=NodeTypeController._logger, messages='Removing/unconfiguring RabbitMQ')
try:
if ServiceManager.has_service('rabbitmq-server', client=target_client):
Toolbox.change_service_state(target_client, 'rabbitmq-server', 'stop', NodeTypeController._logger)
target_client.run(['rabbitmq-server', '-detached'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop_app'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'reset'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop'])
time.sleep(5)
target_client.file_unlink("/var/lib/rabbitmq/.erlang.cookie")
Toolbox.change_service_state(target_client, 'rabbitmq-server', 'stop', NodeTypeController._logger) # To be sure
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger, messages=['\nFailed to remove/unconfigure RabbitMQ', ex], loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger, messages='Stopping services')
services = ['memcached', 'rabbitmq-server']
if unconfigure_rabbitmq is False:
services.remove('rabbitmq-server')
if unconfigure_memcached is False:
services.remove('memcached')
for service in services:
if ServiceManager.has_service(service, client=target_client):
Toolbox.log(logger=NodeTypeController._logger, messages='Stopping service {0}'.format(service))
try:
Toolbox.change_service_state(target_client, service, 'stop', NodeTypeController._logger)
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger, messages=['\nFailed to stop service'.format(service), ex], loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger, messages='Removing services')
services = ['scheduled-tasks', 'webapp-api', 'volumerouter-consumer']
for service in services:
if ServiceManager.has_service(service, client=target_client):
Toolbox.log(logger=NodeTypeController._logger, messages='Removing service {0}'.format(service))
try:
Toolbox.change_service_state(target_client, service, 'stop', NodeTypeController._logger)
ServiceManager.remove_service(service, client=target_client)
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger, messages=['\nFailed to remove service'.format(service), ex], loglevel='exception')
if ServiceManager.has_service('workers', client=target_client):
ServiceManager.add_service(name='workers',
client=target_client,
params={'WORKER_QUEUE': '{0}'.format(unique_id)})
try:
NodeTypeController._configure_amqp_to_volumedriver()
except Exception as ex:
Toolbox.log(logger=NodeTypeController._logger, messages=['\nFailed to configure AMQP to Storage Driver', ex], loglevel='exception')
Toolbox.log(logger=NodeTypeController._logger, messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(clients=ip_client_map, logger=NodeTypeController._logger, offline_node_ips=offline_node_ips)
if Toolbox.run_hooks(component='nodetype',
sub_component='demote',
logger=NodeTypeController._logger,
cluster_ip=cluster_ip,
master_ip=master_ip,
offline_node_ips=offline_node_ips):
Toolbox.log(logger=NodeTypeController._logger, messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(clients=ip_client_map, logger=NodeTypeController._logger, offline_node_ips=offline_node_ips)
if storagerouter not in offline_nodes:
target_client = ip_client_map[cluster_ip]
node_name, _ = target_client.get_hostname()
if NodeTypeController.avahi_installed(client=target_client, logger=NodeTypeController._logger) is True:
NodeTypeController.configure_avahi(client=target_client, node_name=node_name, node_type='extra', logger=NodeTypeController._logger)
Configuration.set('/ovs/framework/hosts/{0}/type'.format(storagerouter.machine_id), 'EXTRA')
if target_client is not None and target_client.file_exists('/tmp/ovs_rollback'):
target_client.file_write('/tmp/ovs_rollback', 'rollback')
Toolbox.log(logger=NodeTypeController._logger, messages='Demote complete', title=True)
def test_cluster_maintenance(self):
"""
Validates whether a cluster can be correctly created
"""
Configuration.set('/ovs/framework/hosts/1/ports',
{'arakoon': [10000, 10100]})
Configuration.set('/ovs/framework/hosts/2/ports',
{'arakoon': [20000, 20100]})
structure = Helper.build_service_structure({'storagerouters': [1, 2]})
storagerouters = structure['storagerouters']
System._machine_id = {
storagerouters[1].ip: '1',
storagerouters[2].ip: '2'
}
# Create new cluster
mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir = mountpoint + '/test_create_cluster'
info = ArakoonInstaller.create_cluster(
'test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK,
storagerouters[1].ip, base_dir)
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {
'tlogs': {
'dirs': {},
'files': []
},
'db': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'1', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.start_cluster('test',
storagerouters[1].ip,
filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[1].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))
ArakoonInstaller.claim_cluster('test',
storagerouters[1].ip,
filesystem=False,
metadata=info['metadata'])
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
# Extending cluster
mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir2 = mountpoint + '/test_extend_cluster'
ArakoonInstaller.extend_cluster(storagerouters[1].ip,
storagerouters[2].ip, 'test',
base_dir2)
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {
'tlogs': {
'dirs': {},
'files': []
},
'db': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n{2}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'1,2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
SSHClient._run_returns[catchup_command] = None
SSHClient._run_recordings = []
ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip],
storagerouters[2].ip,
filesystem=False)
self.assertIn(catchup_command, SSHClient._run_recordings)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
# Shrinking cluster
ArakoonInstaller.shrink_cluster(storagerouters[1].ip,
storagerouters[2].ip, 'test')
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip],
filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
