def test_multi_node(self):
base_port = Configuration.get('ovs.ports.arakoon')[0]
cluster = 'one'
nodes = sorted(TestArakoonInstaller.nodes.keys())
nodes = dict((node, SSHClient(node)) for node in nodes)
first_node = nodes.keys()[0]
ArakoonInstaller.create_cluster(cluster, first_node, [])
for node in nodes[1:]:
ArakoonInstaller.extend_cluster(first_node, node, cluster, [])
expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes))
for node in nodes:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
expected = expected.strip()
for node, client in nodes.iteritems():
contents = client.file_read(self._get_config_path(cluster))
self.assertEqual(contents.strip(), expected.strip())
ArakoonInstaller.shrink_cluster(nodes[1], first_node, cluster)
expected = TestArakoonInstaller.expected_global.format(cluster, ','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:]))
for node in nodes.keys()[1:]:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], node, base_port, base_port + 1)
expected = expected.strip()
for node, client in nodes.iteritems():
if node == first_node:
continue
contents = client.file_read(self._get_config_path(cluster))
self.assertEqual(contents.strip(), expected.strip())
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_result):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_result: Port information
:type arakoon_result: Dictionary
:return: The newly created and added service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']]
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
cluster_name = 'voldrv'
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name('Arakoon')
current_services = []
current_ips = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
current_ips.append(service.storagerouter.ip)
all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0:
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
ip=storagerouter.ip,
base_dir=partition.folder)
current_services.append(add_service(storagerouter, result))
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
current_ips.append(storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
if 0 < len(current_services) < len(available_storagerouters):
for storagerouter, partition in available_storagerouters.iteritems():
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(
current_services[0].storagerouter.ip,
storagerouter.ip,
cluster_name,
partition.folder
)
add_service(storagerouter, result)
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
def test_multi_node(self):
nodes = sorted(TestArakoonInstaller.nodes.keys())
nodes = dict((node, SSHClient(node)) for node in nodes)
first_node = nodes.keys()[0]
result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, first_node, "/tmp/db")
for node in nodes.keys()[1:]:
ArakoonInstaller.extend_cluster(first_node, node, TestArakoonInstaller.cluster_name, "/tmp/db")
expected = TestArakoonInstaller.expected_global.format(
",".join(TestArakoonInstaller.nodes[node] for node in nodes), TestArakoonInstaller.cluster_name
)
for node in nodes:
expected += TestArakoonInstaller.expected_base.format(
TestArakoonInstaller.nodes[node],
result["client_port"],
result["messaging_port"],
TestArakoonInstaller.cluster_name,
node,
)
expected = expected.strip()
for node, client in nodes.iteritems():
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
ArakoonInstaller.shrink_cluster(first_node, TestArakoonInstaller.cluster_name)
expected = TestArakoonInstaller.expected_global.format(
",".join(TestArakoonInstaller.nodes[node] for node in nodes[1:]), TestArakoonInstaller.cluster_name
)
for node in nodes.keys()[1:]:
expected += TestArakoonInstaller.expected_base.format(
TestArakoonInstaller.nodes[node],
result["client_port"],
result["messaging_port"],
TestArakoonInstaller.cluster_name,
node,
)
expected = expected.strip()
for node, client in nodes.iteritems():
if node == first_node:
continue
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
def test_multi_node(self):
nodes = sorted(TestArakoonInstaller.nodes.keys())
nodes = dict((node, SSHClient(node)) for node in nodes)
first_node = nodes.keys()[0]
result = ArakoonInstaller.create_cluster(TestArakoonInstaller.cluster_name, first_node, '/tmp/db')
for node in nodes.keys()[1:]:
ArakoonInstaller.extend_cluster(first_node, node, TestArakoonInstaller.cluster_name, '/tmp/db')
expected = TestArakoonInstaller.expected_global.format(','.join(TestArakoonInstaller.nodes[node] for node in nodes), TestArakoonInstaller.cluster_name)
for node in nodes:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node)
expected = expected.strip()
for node, client in nodes.iteritems():
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
ArakoonInstaller.shrink_cluster(first_node, TestArakoonInstaller.cluster_name)
expected = TestArakoonInstaller.expected_global.format(','.join(TestArakoonInstaller.nodes[node] for node in nodes[1:]), TestArakoonInstaller.cluster_name)
for node in nodes.keys()[1:]:
expected += TestArakoonInstaller.expected_base.format(TestArakoonInstaller.nodes[node], result['client_port'], result['messaging_port'], TestArakoonInstaller.cluster_name, node)
expected = expected.strip()
for node, client in nodes.iteritems():
if node == first_node:
continue
contents = client.file_read(TestArakoonInstaller.cluster_config_file)
self.assertEqual(contents.strip(), expected.strip())
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_ports):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_ports: Port information
:type arakoon_ports: list
:return: The newly created and added service
:rtype: Service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = arakoon_ports
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)
current_ips = []
current_services = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
if service.is_internal is True:
current_ips.append(service.storagerouter.ip)
all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(current_services) == 0: # Create new cluster
metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD)
if metadata is None: # No externally managed cluster found, we create 1 ourselves
if not available_storagerouters:
raise RuntimeError('Could not find any Storage Router with a DB role')
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name='voldrv',
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
ip=storagerouter.ip,
base_dir=partition.folder,
filesystem=False)
ports = [result['client_port'], result['messaging_port']]
metadata = result['metadata']
ArakoonInstaller.restart_cluster_add(cluster_name='voldrv',
current_ips=current_ips,
new_ip=storagerouter.ip,
filesystem=False)
ArakoonInstaller.claim_cluster(cluster_name='voldrv',
master_ip=storagerouter.ip,
filesystem=False,
metadata=metadata)
current_ips.append(storagerouter.ip)
else:
ports = []
storagerouter = None
cluster_name = metadata['cluster_name']
Configuration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name)
StorageDriverController._logger.info('Claiming {0} managed arakoon cluster: {1}'.format('externally' if storagerouter is None else 'internally', cluster_name))
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
current_services.append(add_service(service_storagerouter=storagerouter, arakoon_ports=ports))
cluster_name = Configuration.get('/ovs/framework/arakoon_clusters').get('voldrv')
if cluster_name is None:
return
metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)
if 0 < len(current_services) < len(available_storagerouters) and metadata['internal'] is True:
for storagerouter, partition in available_storagerouters.iteritems():
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(master_ip=current_services[0].storagerouter.ip,
new_ip=storagerouter.ip,
cluster_name=cluster_name,
base_dir=partition.folder)
add_service(storagerouter, [result['client_port'], result['messaging_port']])
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name=cluster_name,
current_ips=current_ips,
new_ip=storagerouter.ip,
filesystem=False)
StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def test_cluster_maintenance(self):
"""
Validates whether a cluster can be correctly created
"""
Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]})
Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]})
structure = Helper.build_service_structure(
{'storagerouters': [1, 2]}
)
storagerouters = structure['storagerouters']
System._machine_id = {storagerouters[1].ip: '1',
storagerouters[2].ip: '2'}
# Create new cluster
mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir = mountpoint + '/test_create_cluster'
info = ArakoonInstaller.create_cluster('test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir)
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
'files': []},
'db': {'dirs': {},
'files': []}},
'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[1].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))
ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata'])
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
# Extending cluster
mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir2 = mountpoint + '/test_extend_cluster'
ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2)
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {},
'files': []},
'db': {'dirs': {},
'files': []}},
'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n{2}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1,2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001
),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
SSHClient._run_returns[catchup_command] = None
SSHClient._run_recordings = []
ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False)
self.assertIn(catchup_command, SSHClient._run_recordings)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
# Shrinking cluster
ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test')
reality = Helper.extract_dir_structure(base_dir)
expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {}, 'files': []}},
'files': []}},
'files': []}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001
))
self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True}
self.assertDictEqual(reality, expected)
def ovs_3671_validate_archiving_of_existing_arakoon_data_on_create_and_extend_test():
"""
Validate arakoon archiving when creating and extending an arakoon cluster
"""
storagerouters = GeneralStorageRouter.get_storage_routers()
storagerouters.sort(key=lambda k: k.ip)
if len(storagerouters) < 2:
TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters)))
return
cluster_name = 'OVS_3671-multi-node-cluster'
cluster_basedir = '/var/tmp'
archived_files = []
files_to_create = []
for index, sr in enumerate(storagerouters):
root_client = SSHClient(sr, username='******')
for directory in ['/'.join([cluster_basedir, 'arakoon']), '/var/log/arakoon']:
root_client.dir_create(os.path.dirname(directory))
root_client.dir_chmod(os.path.dirname(directory), 0755, recursive=True)
root_client.dir_chown(os.path.dirname(directory), 'ovs', 'ovs', recursive=True)
files_to_create = ['/'.join([cluster_basedir, 'arakoon', cluster_name, 'db', 'one.db']),
'/'.join([cluster_basedir, 'arakoon', cluster_name, 'tlogs', 'one.tlog'])]
client = SSHClient(sr, username='******')
for filename in files_to_create:
client.dir_create(os.path.dirname(filename))
client.dir_chmod(os.path.dirname(filename), 0755, recursive=True)
client.dir_chown(os.path.dirname(filename), 'ovs', 'ovs', recursive=True)
client.file_create(files_to_create)
for filename in files_to_create:
assert client.file_exists(filename) is True, 'File {0} not present'.format(filename)
archived_files = ['/'.join(['/var/log/arakoon', cluster_name, 'archive', 'one.log'])]
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate single node cluster')
if index == 0:
create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK,
sr.ip, cluster_basedir, filesystem=False)
TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
ArakoonInstaller.start_cluster(cluster_name, sr.ip, False)
ArakoonInstaller.claim_cluster(cluster_name, sr, False, metadata=create_info['metadata'])
else:
ArakoonInstaller.extend_cluster(storagerouters[0].ip, sr.ip, cluster_name, cluster_basedir)
TestArakoon.validate_arakoon_config_files(storagerouters[:index + 1], cluster_name)
TestArakoon.verify_arakoon_structure(root_client, cluster_name, True, True)
TestArakoon.check_archived_directory(client, archived_files)
for filename in files_to_create:
assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('remove cluster')
ArakoonInstaller.delete_cluster(cluster_name, storagerouters[0].ip, False)
for sr in storagerouters:
client = SSHClient(sr, username='******')
TestArakoon.check_archived_directory(client, archived_files)
for filename in files_to_create:
assert client.file_exists(filename) is False, 'File {0} is missing'.format(filename)
TestArakoon.verify_arakoon_structure(client, cluster_name, False, False)
def ar_0001_validate_create_extend_shrink_delete_cluster_test():
"""
Validate extending and shrinking of arakoon clusters
"""
storagerouters = GeneralStorageRouter.get_storage_routers()
if not len(storagerouters) >= 3:
TestArakoon.logger.info('Environment has only {0} node(s)'.format(len(storagerouters)))
return
cluster_name = 'ar_0001'
cluster_basedir = '/var/tmp/'
first_sr = storagerouters[0]
second_sr = storagerouters[1]
third_sr = storagerouters[2]
first_root_client = SSHClient(first_sr, username='******')
second_root_client = SSHClient(second_sr, username='******')
third_root_client = SSHClient(third_sr, username='******')
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate single node cluster')
create_info = ArakoonInstaller.create_cluster(cluster_name, ServiceType.ARAKOON_CLUSTER_TYPES.FWK, first_sr.ip,
cluster_basedir, filesystem=False)
TestArakoon.logger.info('create_info: \n{0}'.format(create_info))
ArakoonInstaller.start_cluster(cluster_name, first_sr.ip, False)
ArakoonInstaller.claim_cluster(cluster_name, first_sr, False, metadata=create_info['metadata'])
TestArakoon.validate_arakoon_config_files([first_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate two node cluster')
ArakoonInstaller.extend_cluster(first_sr.ip, second_sr.ip, cluster_name, cluster_basedir)
TestArakoon.validate_arakoon_config_files([first_sr, second_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('setup and validate three node cluster')
ArakoonInstaller.extend_cluster(first_sr.ip, third_sr.ip, cluster_name, cluster_basedir)
TestArakoon.validate_arakoon_config_files([first_sr, second_sr, third_sr], cluster_name)
for client in [first_root_client, second_root_client, third_root_client]:
TestArakoon.verify_arakoon_structure(client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('reduce and validate three node to two node cluster')
ArakoonInstaller.shrink_cluster(second_sr.ip, first_sr.ip, cluster_name)
TestArakoon.validate_arakoon_config_files([first_sr, third_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, True)
TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False)
TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('reduce and validate two node to one node cluster')
ArakoonInstaller.shrink_cluster(first_sr.ip, third_sr.ip, cluster_name)
TestArakoon.validate_arakoon_config_files([third_sr], cluster_name)
TestArakoon.verify_arakoon_structure(first_root_client, cluster_name, True, False)
TestArakoon.verify_arakoon_structure(second_root_client, cluster_name, True, False)
TestArakoon.verify_arakoon_structure(third_root_client, cluster_name, True, True)
TestArakoon.logger.info('===================================================')
TestArakoon.logger.info('remove cluster')
ArakoonInstaller.delete_cluster(cluster_name, third_sr.ip, False)
for client in [first_root_client, second_root_client, third_root_client]:
TestArakoon.verify_arakoon_structure(client, cluster_name, False, False)
GeneralArakoon.delete_config(cluster_name)
def promote_node(cluster_ip, master_ip, ip_client_map, unique_id, configure_memcached, configure_rabbitmq):
"""
Promotes a given node
"""
from ovs.dal.lists.storagerouterlist import StorageRouterList
from ovs.dal.lists.servicetypelist import ServiceTypeList
from ovs.dal.lists.servicelist import ServiceList
from ovs.dal.hybrids.service import Service
Toolbox.log(logger=NodeTypeController._logger, messages='Promoting node', title=True)
if configure_memcached is True:
if NodeTypeController._validate_local_memcache_servers(ip_client_map) is False:
raise RuntimeError('Not all memcache nodes can be reached which is required for promoting a node.')
target_client = ip_client_map[cluster_ip]
machine_id = System.get_my_machine_id(target_client)
node_name, _ = target_client.get_hostname()
master_client = ip_client_map[master_ip]
storagerouter = StorageRouterList.get_by_machine_id(unique_id)
storagerouter.node_type = 'MASTER'
storagerouter.save()
external_config = Configuration.get('/ovs/framework/external_config')
if external_config is None:
config_store = Configuration.get_store()
if config_store == 'arakoon':
Toolbox.log(logger=NodeTypeController._logger, messages='Joining Arakoon configuration cluster')
metadata = ArakoonInstaller.extend_cluster(master_ip=master_ip,
new_ip=cluster_ip,
cluster_name='config',
base_dir=Configuration.get('/ovs/framework/paths|ovsdb'),
ports=[26400, 26401],
filesystem=True)
ArakoonInstaller.restart_cluster_add(cluster_name='config',
current_ips=metadata['ips'],
new_ip=cluster_ip,
filesystem=True)
ServiceManager.register_service(node_name=machine_id,
service_metadata=metadata['service_metadata'])
else:
from ovs.extensions.db.etcd.installer import EtcdInstaller
Toolbox.log(logger=NodeTypeController._logger, messages='Joining Etcd cluster')
EtcdInstaller.extend_cluster(master_ip, cluster_ip, 'config')
# Find other (arakoon) master nodes
arakoon_cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|ovsdb'))
arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=arakoon_cluster_name)
config = ArakoonClusterConfig(cluster_id=arakoon_cluster_name, filesystem=False)
config.load_config()
master_node_ips = [node.ip for node in config.nodes]
if cluster_ip in master_node_ips:
master_node_ips.remove(cluster_ip)
if len(master_node_ips) == 0:
raise RuntimeError('There should be at least one other master node')
arakoon_ports = []
if arakoon_metadata['internal'] is True:
Toolbox.log(logger=NodeTypeController._logger, messages='Joining Arakoon OVS DB cluster')
result = ArakoonInstaller.extend_cluster(master_ip=master_ip,
new_ip=cluster_ip,
cluster_name=arakoon_cluster_name,
base_dir=Configuration.get('/ovs/framework/paths|ovsdb'))
ArakoonInstaller.restart_cluster_add(cluster_name=arakoon_cluster_name,
current_ips=result['ips'],
new_ip=cluster_ip, filesystem=False)
arakoon_ports = [result['client_port'], result['messaging_port']]
if configure_memcached is True:
NodeTypeController.configure_memcached(client=target_client, logger=NodeTypeController._logger)
NodeTypeController.add_services(client=target_client, node_type='master', logger=NodeTypeController._logger)
Toolbox.log(logger=NodeTypeController._logger, messages='Update configurations')
if configure_memcached is True:
endpoints = Configuration.get('/ovs/framework/memcache|endpoints')
endpoint = '{0}:11211'.format(cluster_ip)
if endpoint not in endpoints:
endpoints.append(endpoint)
Configuration.set('/ovs/framework/memcache|endpoints', endpoints)
if configure_rabbitmq is True:
endpoints = Configuration.get('/ovs/framework/messagequeue|endpoints')
endpoint = '{0}:5672'.format(cluster_ip)
if endpoint not in endpoints:
endpoints.append(endpoint)
Configuration.set('/ovs/framework/messagequeue|endpoints', endpoints)
if arakoon_metadata['internal'] is True:
Toolbox.log(logger=NodeTypeController._logger, messages='Restarting master node services')
ArakoonInstaller.restart_cluster_add(cluster_name=arakoon_cluster_name,
current_ips=master_node_ips,
new_ip=cluster_ip,
filesystem=False)
PersistentFactory.store = None
VolatileFactory.store = None
if 'arakoon-ovsdb' not in [s.name for s in ServiceList.get_services() if s.is_internal is False or s.storagerouter.ip == cluster_ip]:
service = Service()
service.name = 'arakoon-ovsdb'
service.type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON)
service.ports = arakoon_ports
service.storagerouter = storagerouter
service.save()
if configure_rabbitmq is True:
NodeTypeController.configure_rabbitmq(client=target_client, logger=NodeTypeController._logger)
# Copy rabbitmq cookie
rabbitmq_cookie_file = '/var/lib/rabbitmq/.erlang.cookie'
Toolbox.log(logger=NodeTypeController._logger, messages='Copying Rabbit MQ cookie')
contents = master_client.file_read(rabbitmq_cookie_file)
master_hostname, _ = master_client.get_hostname()
target_client.dir_create(os.path.dirname(rabbitmq_cookie_file))
target_client.file_write(rabbitmq_cookie_file, contents)
target_client.file_chmod(rabbitmq_cookie_file, mode=400)
target_client.run(['rabbitmq-server', '-detached'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop_app'])
time.sleep(5)
target_client.run(['rabbitmqctl', 'join_cluster', 'rabbit@{0}'.format(master_hostname)])
time.sleep(5)
target_client.run(['rabbitmqctl', 'stop'])
time.sleep(5)
# Enable HA for the rabbitMQ queues
Toolbox.change_service_state(target_client, 'rabbitmq-server', 'start', NodeTypeController._logger)
NodeTypeController.check_rabbitmq_and_enable_ha_mode(client=target_client, logger=NodeTypeController._logger)
NodeTypeController._configure_amqp_to_volumedriver()
Toolbox.log(logger=NodeTypeController._logger, messages='Starting services')
services = ['memcached', 'arakoon-ovsdb', 'rabbitmq-server', 'etcd-config']
if arakoon_metadata['internal'] is True:
services.remove('arakoon-ovsdb')
for service in services:
if ServiceManager.has_service(service, client=target_client):
Toolbox.change_service_state(target_client, service, 'start', NodeTypeController._logger)
Toolbox.log(logger=NodeTypeController._logger, messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(clients=ip_client_map, logger=NodeTypeController._logger)
if Toolbox.run_hooks(component='nodetype',
sub_component='promote',
logger=NodeTypeController._logger,
cluster_ip=cluster_ip,
master_ip=master_ip):
Toolbox.log(logger=NodeTypeController._logger, messages='Restarting services')
NodeTypeController.restart_framework_and_memcache_services(clients=ip_client_map, logger=NodeTypeController._logger)
if NodeTypeController.avahi_installed(client=target_client, logger=NodeTypeController._logger) is True:
NodeTypeController.configure_avahi(client=target_client, node_name=node_name, node_type='master', logger=NodeTypeController._logger)
Configuration.set('/ovs/framework/hosts/{0}/type'.format(machine_id), 'MASTER')
target_client.run(['chown', '-R', 'ovs:ovs', '/opt/OpenvStorage/config'])
Configuration.set('/ovs/framework/hosts/{0}/promotecompleted'.format(machine_id), True)
if target_client.file_exists('/tmp/ovs_rollback'):
target_client.file_delete('/tmp/ovs_rollback')
Toolbox.log(logger=NodeTypeController._logger, messages='Promote complete')
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_result):
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']]
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
cluster_name = 'voldrv'
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name('Arakoon')
current_services = []
current_ips = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
current_ips.append(service.storagerouter.ip)
all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0:
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
ip=storagerouter.ip,
exclude_ports=ServiceList.get_ports_for_ip(storagerouter.ip),
base_dir=partition.folder)
current_services.append(add_service(storagerouter, result))
for sr_ip in all_sr_ips:
if sr_ip not in current_ips:
ArakoonInstaller.deploy_to_slave(storagerouter.ip, sr_ip, cluster_name)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
current_ips.append(storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
if 0 < len(current_services) < len(available_storagerouters):
distributed = False
for storagerouter, partition in available_storagerouters.iteritems():
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(
current_services[0].storagerouter.ip,
storagerouter.ip,
cluster_name,
ServiceList.get_ports_for_ip(storagerouter.ip),
partition.folder
)
add_service(storagerouter, result)
current_ips.append(storagerouter.ip)
if distributed is False:
distributed = True
for sr_ip in all_sr_ips:
if sr_ip not in current_ips:
ArakoonInstaller.deploy_to_slave(current_services[0].storagerouter.ip, sr_ip, cluster_name)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
def test_cluster_maintenance(self):
"""
Validates whether a cluster can be correctly created
"""
Configuration.set('/ovs/framework/hosts/1/ports',
{'arakoon': [10000, 10100]})
Configuration.set('/ovs/framework/hosts/2/ports',
{'arakoon': [20000, 20100]})
structure = Helper.build_service_structure({'storagerouters': [1, 2]})
storagerouters = structure['storagerouters']
System._machine_id = {
storagerouters[1].ip: '1',
storagerouters[2].ip: '2'
}
# Create new cluster
mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir = mountpoint + '/test_create_cluster'
info = ArakoonInstaller.create_cluster(
'test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK,
storagerouters[1].ip, base_dir)
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {
'tlogs': {
'dirs': {},
'files': []
},
'db': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'1', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.start_cluster('test',
storagerouters[1].ip,
filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[1].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY))
ArakoonInstaller.claim_cluster('test',
storagerouters[1].ip,
filesystem=False,
metadata=info['metadata'])
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
# Extending cluster
mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint
if os.path.exists(mountpoint) and mountpoint != '/':
shutil.rmtree(mountpoint)
base_dir2 = mountpoint + '/test_extend_cluster'
ArakoonInstaller.extend_cluster(storagerouters[1].ip,
storagerouters[2].ip, 'test',
base_dir2)
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {
'tlogs': {
'dirs': {},
'files': []
},
'db': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n{2}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'1,2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'1', storagerouters[1].ip, 10000, base_dir, '1', 10001),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only'
SSHClient._run_returns[catchup_command] = None
SSHClient._run_recordings = []
ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip],
storagerouters[2].ip,
filesystem=False)
self.assertIn(catchup_command, SSHClient._run_recordings)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
# Shrinking cluster
ArakoonInstaller.shrink_cluster(storagerouters[1].ip,
storagerouters[2].ip, 'test')
reality = Helper.extract_dir_structure(base_dir)
expected = {
'dirs': {
'arakoon': {
'dirs': {
'test': {
'dirs': {},
'files': []
}
},
'files': []
}
},
'files': []
}
self.assertDictEqual(reality, expected)
expected = '{0}\n\n{1}\n\n'.format(
ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format(
'2', 'test', ''),
ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format(
'2', storagerouters[2].ip, 20000, base_dir2, '2', 20001))
self.assertEqual(
Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'),
raw=True), expected)
# @TODO: assert service availability here. It should be stopped
ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip],
filesystem=False)
# @TODO: assert the service is running
config = ArakoonClusterConfig('test', filesystem=False)
config.load_config(storagerouters[2].ip)
client = ArakoonInstaller.build_client(config)
reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY)
self.assertEqual(reality, expected)
reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY))
expected = {
'cluster_name': 'test',
'cluster_type': 'FWK',
'in_use': True,
'internal': True
}
self.assertDictEqual(reality, expected)
def _voldrv_arakoon_checkup(create_cluster):
def add_service(service_storagerouter, arakoon_result):
"""
Add a service to the storage router
:param service_storagerouter: Storage Router to add the service to
:type service_storagerouter: StorageRouter
:param arakoon_result: Port information
:type arakoon_result: Dictionary
:return: The newly created and added service
"""
new_service = Service()
new_service.name = service_name
new_service.type = service_type
new_service.ports = [
arakoon_result['client_port'], arakoon_result['messaging_port']
]
new_service.storagerouter = service_storagerouter
new_service.save()
return new_service
cluster_name = 'voldrv'
service_name = 'arakoon-voldrv'
service_type = ServiceTypeList.get_by_name('Arakoon')
current_services = []
current_ips = []
for service in service_type.services:
if service.name == service_name:
current_services.append(service)
current_ips.append(service.storagerouter.ip)
all_sr_ips = [
storagerouter.ip
for storagerouter in StorageRouterList.get_slaves()
]
available_storagerouters = {}
for storagerouter in StorageRouterList.get_masters():
storagerouter.invalidate_dynamics(['partition_config'])
if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0:
available_storagerouters[storagerouter] = DiskPartition(
storagerouter.partition_config[DiskPartition.ROLES.DB][0])
all_sr_ips.append(storagerouter.ip)
if create_cluster is True and len(
current_services) == 0 and len(available_storagerouters) > 0:
storagerouter, partition = available_storagerouters.items()[0]
result = ArakoonInstaller.create_cluster(cluster_name=cluster_name,
ip=storagerouter.ip,
base_dir=partition.folder)
current_services.append(add_service(storagerouter, result))
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips,
storagerouter.ip)
current_ips.append(storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
if 0 < len(current_services) < len(available_storagerouters):
for storagerouter, partition in available_storagerouters.iteritems(
):
if storagerouter.ip in current_ips:
continue
result = ArakoonInstaller.extend_cluster(
current_services[0].storagerouter.ip, storagerouter.ip,
cluster_name, partition.folder)
add_service(storagerouter, result)
current_ips.append(storagerouter.ip)
ArakoonInstaller.restart_cluster_add(cluster_name, current_ips,
storagerouter.ip)
StorageDriverController._configure_arakoon_to_volumedriver()
