def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_result): """ Add a service to the storage router :param service_storagerouter: Storage Router to add the service to :type service_storagerouter: StorageRouter :param arakoon_result: Port information :type arakoon_result: Dictionary :return: The newly created and added service """ new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']] new_service.storagerouter = service_storagerouter new_service.save() return new_service cluster_name = 'voldrv' service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name('Arakoon') current_services = [] current_ips = [] for service in service_type.services: if service.name == service_name: current_services.append(service) current_ips.append(service.storagerouter.ip) all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0: storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name=cluster_name, ip=storagerouter.ip, base_dir=partition.folder) current_services.append(add_service(storagerouter, result)) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) current_ips.append(storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver() if 0 < len(current_services) < len(available_storagerouters): for storagerouter, partition in available_storagerouters.iteritems(): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster( current_services[0].storagerouter.ip, storagerouter.ip, cluster_name, partition.folder ) add_service(storagerouter, result) current_ips.append(storagerouter.ip) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver()
def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_ports): """ Add a service to the storage router :param service_storagerouter: Storage Router to add the service to :type service_storagerouter: StorageRouter :param arakoon_ports: Port information :type arakoon_ports: list :return: The newly created and added service :rtype: Service """ new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = arakoon_ports new_service.storagerouter = service_storagerouter new_service.save() return new_service service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON) current_ips = [] current_services = [] for service in service_type.services: if service.name == service_name: current_services.append(service) if service.is_internal is True: current_ips.append(service.storagerouter.ip) all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len(current_services) == 0: # Create new cluster metadata = ArakoonInstaller.get_unused_arakoon_metadata_and_claim(cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD) if metadata is None: # No externally managed cluster found, we create 1 ourselves if not available_storagerouters: raise RuntimeError('Could not find any Storage Router with a DB role') storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name='voldrv', cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD, ip=storagerouter.ip, base_dir=partition.folder, filesystem=False) ports = [result['client_port'], result['messaging_port']] metadata = result['metadata'] ArakoonInstaller.restart_cluster_add(cluster_name='voldrv', current_ips=current_ips, new_ip=storagerouter.ip, filesystem=False) ArakoonInstaller.claim_cluster(cluster_name='voldrv', master_ip=storagerouter.ip, filesystem=False, metadata=metadata) current_ips.append(storagerouter.ip) else: ports = [] storagerouter = None cluster_name = metadata['cluster_name'] Configuration.set('/ovs/framework/arakoon_clusters|voldrv', cluster_name) StorageDriverController._logger.info('Claiming {0} managed arakoon cluster: {1}'.format('externally' if storagerouter is None else 'internally', cluster_name)) StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name) current_services.append(add_service(service_storagerouter=storagerouter, arakoon_ports=ports)) cluster_name = Configuration.get('/ovs/framework/arakoon_clusters').get('voldrv') if cluster_name is None: return metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name) if 0 < len(current_services) < len(available_storagerouters) and metadata['internal'] is True: for storagerouter, partition in available_storagerouters.iteritems(): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster(master_ip=current_services[0].storagerouter.ip, new_ip=storagerouter.ip, cluster_name=cluster_name, base_dir=partition.folder) add_service(storagerouter, [result['client_port'], result['messaging_port']]) current_ips.append(storagerouter.ip) ArakoonInstaller.restart_cluster_add(cluster_name=cluster_name, current_ips=current_ips, new_ip=storagerouter.ip, filesystem=False) StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def test_cluster_maintenance(self): """ Validates whether a cluster can be correctly created """ Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]}) Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]}) structure = Helper.build_service_structure( {'storagerouters': [1, 2]} ) storagerouters = structure['storagerouters'] System._machine_id = {storagerouters[1].ip: '1', storagerouters[2].ip: '2'} # Create new cluster mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir = mountpoint + '/test_create_cluster' info = ArakoonInstaller.create_cluster('test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir) reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {}, 'files': []}, 'db': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[1].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY)) ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata']) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected) # Extending cluster mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir2 = mountpoint + '/test_extend_cluster' ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2) reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {}, 'files': []}, 'db': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n{2}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1,2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001 ), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only' SSHClient._run_returns[catchup_command] = None SSHClient._run_recordings = [] ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False) self.assertIn(catchup_command, SSHClient._run_recordings) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected) # Shrinking cluster ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test') reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected)
def promote_node(cluster_ip, master_ip, ip_client_map, unique_id, configure_memcached, configure_rabbitmq): """ Promotes a given node """ from ovs.dal.lists.storagerouterlist import StorageRouterList from ovs.dal.lists.servicetypelist import ServiceTypeList from ovs.dal.lists.servicelist import ServiceList from ovs.dal.hybrids.service import Service Toolbox.log(logger=NodeTypeController._logger, messages='Promoting node', title=True) if configure_memcached is True: if NodeTypeController._validate_local_memcache_servers(ip_client_map) is False: raise RuntimeError('Not all memcache nodes can be reached which is required for promoting a node.') target_client = ip_client_map[cluster_ip] machine_id = System.get_my_machine_id(target_client) node_name, _ = target_client.get_hostname() master_client = ip_client_map[master_ip] storagerouter = StorageRouterList.get_by_machine_id(unique_id) storagerouter.node_type = 'MASTER' storagerouter.save() external_config = Configuration.get('/ovs/framework/external_config') if external_config is None: config_store = Configuration.get_store() if config_store == 'arakoon': Toolbox.log(logger=NodeTypeController._logger, messages='Joining Arakoon configuration cluster') metadata = ArakoonInstaller.extend_cluster(master_ip=master_ip, new_ip=cluster_ip, cluster_name='config', base_dir=Configuration.get('/ovs/framework/paths|ovsdb'), ports=[26400, 26401], filesystem=True) ArakoonInstaller.restart_cluster_add(cluster_name='config', current_ips=metadata['ips'], new_ip=cluster_ip, filesystem=True) ServiceManager.register_service(node_name=machine_id, service_metadata=metadata['service_metadata']) else: from ovs.extensions.db.etcd.installer import EtcdInstaller Toolbox.log(logger=NodeTypeController._logger, messages='Joining Etcd cluster') EtcdInstaller.extend_cluster(master_ip, cluster_ip, 'config') # Find other (arakoon) master nodes arakoon_cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|ovsdb')) arakoon_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name(cluster_name=arakoon_cluster_name) config = ArakoonClusterConfig(cluster_id=arakoon_cluster_name, filesystem=False) config.load_config() master_node_ips = [node.ip for node in config.nodes] if cluster_ip in master_node_ips: master_node_ips.remove(cluster_ip) if len(master_node_ips) == 0: raise RuntimeError('There should be at least one other master node') arakoon_ports = [] if arakoon_metadata['internal'] is True: Toolbox.log(logger=NodeTypeController._logger, messages='Joining Arakoon OVS DB cluster') result = ArakoonInstaller.extend_cluster(master_ip=master_ip, new_ip=cluster_ip, cluster_name=arakoon_cluster_name, base_dir=Configuration.get('/ovs/framework/paths|ovsdb')) ArakoonInstaller.restart_cluster_add(cluster_name=arakoon_cluster_name, current_ips=result['ips'], new_ip=cluster_ip, filesystem=False) arakoon_ports = [result['client_port'], result['messaging_port']] if configure_memcached is True: NodeTypeController.configure_memcached(client=target_client, logger=NodeTypeController._logger) NodeTypeController.add_services(client=target_client, node_type='master', logger=NodeTypeController._logger) Toolbox.log(logger=NodeTypeController._logger, messages='Update configurations') if configure_memcached is True: endpoints = Configuration.get('/ovs/framework/memcache|endpoints') endpoint = '{0}:11211'.format(cluster_ip) if endpoint not in endpoints: endpoints.append(endpoint) Configuration.set('/ovs/framework/memcache|endpoints', endpoints) if configure_rabbitmq is True: endpoints = Configuration.get('/ovs/framework/messagequeue|endpoints') endpoint = '{0}:5672'.format(cluster_ip) if endpoint not in endpoints: endpoints.append(endpoint) Configuration.set('/ovs/framework/messagequeue|endpoints', endpoints) if arakoon_metadata['internal'] is True: Toolbox.log(logger=NodeTypeController._logger, messages='Restarting master node services') ArakoonInstaller.restart_cluster_add(cluster_name=arakoon_cluster_name, current_ips=master_node_ips, new_ip=cluster_ip, filesystem=False) PersistentFactory.store = None VolatileFactory.store = None if 'arakoon-ovsdb' not in [s.name for s in ServiceList.get_services() if s.is_internal is False or s.storagerouter.ip == cluster_ip]: service = Service() service.name = 'arakoon-ovsdb' service.type = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON) service.ports = arakoon_ports service.storagerouter = storagerouter service.save() if configure_rabbitmq is True: NodeTypeController.configure_rabbitmq(client=target_client, logger=NodeTypeController._logger) # Copy rabbitmq cookie rabbitmq_cookie_file = '/var/lib/rabbitmq/.erlang.cookie' Toolbox.log(logger=NodeTypeController._logger, messages='Copying Rabbit MQ cookie') contents = master_client.file_read(rabbitmq_cookie_file) master_hostname, _ = master_client.get_hostname() target_client.dir_create(os.path.dirname(rabbitmq_cookie_file)) target_client.file_write(rabbitmq_cookie_file, contents) target_client.file_chmod(rabbitmq_cookie_file, mode=400) target_client.run(['rabbitmq-server', '-detached']) time.sleep(5) target_client.run(['rabbitmqctl', 'stop_app']) time.sleep(5) target_client.run(['rabbitmqctl', 'join_cluster', 'rabbit@{0}'.format(master_hostname)]) time.sleep(5) target_client.run(['rabbitmqctl', 'stop']) time.sleep(5) # Enable HA for the rabbitMQ queues Toolbox.change_service_state(target_client, 'rabbitmq-server', 'start', NodeTypeController._logger) NodeTypeController.check_rabbitmq_and_enable_ha_mode(client=target_client, logger=NodeTypeController._logger) NodeTypeController._configure_amqp_to_volumedriver() Toolbox.log(logger=NodeTypeController._logger, messages='Starting services') services = ['memcached', 'arakoon-ovsdb', 'rabbitmq-server', 'etcd-config'] if arakoon_metadata['internal'] is True: services.remove('arakoon-ovsdb') for service in services: if ServiceManager.has_service(service, client=target_client): Toolbox.change_service_state(target_client, service, 'start', NodeTypeController._logger) Toolbox.log(logger=NodeTypeController._logger, messages='Restarting services') NodeTypeController.restart_framework_and_memcache_services(clients=ip_client_map, logger=NodeTypeController._logger) if Toolbox.run_hooks(component='nodetype', sub_component='promote', logger=NodeTypeController._logger, cluster_ip=cluster_ip, master_ip=master_ip): Toolbox.log(logger=NodeTypeController._logger, messages='Restarting services') NodeTypeController.restart_framework_and_memcache_services(clients=ip_client_map, logger=NodeTypeController._logger) if NodeTypeController.avahi_installed(client=target_client, logger=NodeTypeController._logger) is True: NodeTypeController.configure_avahi(client=target_client, node_name=node_name, node_type='master', logger=NodeTypeController._logger) Configuration.set('/ovs/framework/hosts/{0}/type'.format(machine_id), 'MASTER') target_client.run(['chown', '-R', 'ovs:ovs', '/opt/OpenvStorage/config']) Configuration.set('/ovs/framework/hosts/{0}/promotecompleted'.format(machine_id), True) if target_client.file_exists('/tmp/ovs_rollback'): target_client.file_delete('/tmp/ovs_rollback') Toolbox.log(logger=NodeTypeController._logger, messages='Promote complete')
def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_result): new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = [arakoon_result['client_port'], arakoon_result['messaging_port']] new_service.storagerouter = service_storagerouter new_service.save() return new_service cluster_name = 'voldrv' service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name('Arakoon') current_services = [] current_ips = [] for service in service_type.services: if service.name == service_name: current_services.append(service) current_ips.append(service.storagerouter.ip) all_sr_ips = [storagerouter.ip for storagerouter in StorageRouterList.get_slaves()] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition(storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len(current_services) == 0 and len(available_storagerouters) > 0: storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name=cluster_name, ip=storagerouter.ip, exclude_ports=ServiceList.get_ports_for_ip(storagerouter.ip), base_dir=partition.folder) current_services.append(add_service(storagerouter, result)) for sr_ip in all_sr_ips: if sr_ip not in current_ips: ArakoonInstaller.deploy_to_slave(storagerouter.ip, sr_ip, cluster_name) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) current_ips.append(storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver() if 0 < len(current_services) < len(available_storagerouters): distributed = False for storagerouter, partition in available_storagerouters.iteritems(): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster( current_services[0].storagerouter.ip, storagerouter.ip, cluster_name, ServiceList.get_ports_for_ip(storagerouter.ip), partition.folder ) add_service(storagerouter, result) current_ips.append(storagerouter.ip) if distributed is False: distributed = True for sr_ip in all_sr_ips: if sr_ip not in current_ips: ArakoonInstaller.deploy_to_slave(current_services[0].storagerouter.ip, sr_ip, cluster_name) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver()
def test_cluster_maintenance(self): """ Validates whether a cluster can be correctly created """ Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]}) Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]}) structure = Helper.build_service_structure({'storagerouters': [1, 2]}) storagerouters = structure['storagerouters'] System._machine_id = { storagerouters[1].ip: '1', storagerouters[2].ip: '2' } # Create new cluster mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir = mountpoint + '/test_create_cluster' info = ArakoonInstaller.create_cluster( 'test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir) reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': { 'tlogs': { 'dirs': {}, 'files': [] }, 'db': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '1', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[1].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY)) ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata']) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected) # Extending cluster mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir2 = mountpoint + '/test_extend_cluster' ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2) reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': { 'tlogs': { 'dirs': {}, 'files': [] }, 'db': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n{2}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '1,2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only' SSHClient._run_returns[catchup_command] = None SSHClient._run_recordings = [] ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False) self.assertIn(catchup_command, SSHClient._run_recordings) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected) # Shrinking cluster ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test') reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected)
def _voldrv_arakoon_checkup(create_cluster): def add_service(service_storagerouter, arakoon_result): """ Add a service to the storage router :param service_storagerouter: Storage Router to add the service to :type service_storagerouter: StorageRouter :param arakoon_result: Port information :type arakoon_result: Dictionary :return: The newly created and added service """ new_service = Service() new_service.name = service_name new_service.type = service_type new_service.ports = [ arakoon_result['client_port'], arakoon_result['messaging_port'] ] new_service.storagerouter = service_storagerouter new_service.save() return new_service cluster_name = 'voldrv' service_name = 'arakoon-voldrv' service_type = ServiceTypeList.get_by_name('Arakoon') current_services = [] current_ips = [] for service in service_type.services: if service.name == service_name: current_services.append(service) current_ips.append(service.storagerouter.ip) all_sr_ips = [ storagerouter.ip for storagerouter in StorageRouterList.get_slaves() ] available_storagerouters = {} for storagerouter in StorageRouterList.get_masters(): storagerouter.invalidate_dynamics(['partition_config']) if len(storagerouter.partition_config[DiskPartition.ROLES.DB]) > 0: available_storagerouters[storagerouter] = DiskPartition( storagerouter.partition_config[DiskPartition.ROLES.DB][0]) all_sr_ips.append(storagerouter.ip) if create_cluster is True and len( current_services) == 0 and len(available_storagerouters) > 0: storagerouter, partition = available_storagerouters.items()[0] result = ArakoonInstaller.create_cluster(cluster_name=cluster_name, ip=storagerouter.ip, base_dir=partition.folder) current_services.append(add_service(storagerouter, result)) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) current_ips.append(storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver() if 0 < len(current_services) < len(available_storagerouters): for storagerouter, partition in available_storagerouters.iteritems( ): if storagerouter.ip in current_ips: continue result = ArakoonInstaller.extend_cluster( current_services[0].storagerouter.ip, storagerouter.ip, cluster_name, partition.folder) add_service(storagerouter, result) current_ips.append(storagerouter.ip) ArakoonInstaller.restart_cluster_add(cluster_name, current_ips, storagerouter.ip) StorageDriverController._configure_arakoon_to_volumedriver()