def on_demote(cluster_ip, master_ip, offline_node_ips=None): """ Handles the demote for the StorageDrivers :param cluster_ip: IP of the node to demote :param master_ip: IP of the master node :param offline_node_ips: IPs of nodes which are offline """ if offline_node_ips is None: offline_node_ips = [] client = SSHClient(cluster_ip, username='******') if cluster_ip not in offline_node_ips else None servicetype = ServiceTypeList.get_by_name('Arakoon') current_service = None remaining_ips = [] for service in servicetype.services: if service.name == 'arakoon-voldrv': if service.storagerouter.ip == cluster_ip: current_service = service elif service.storagerouter.ip not in offline_node_ips: remaining_ips.append(service.storagerouter.ip) if current_service is not None: print '* Shrink StorageDriver cluster' ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv', offline_node_ips) if client is not None and ServiceManager.has_service(current_service.name, client=client) is True: ServiceManager.stop_service(current_service.name, client=client) ServiceManager.remove_service(current_service.name, client=client) ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips) current_service.delete() StorageDriverController._configure_arakoon_to_volumedriver(offline_node_ips)
def on_demote(cluster_ip, master_ip): """ Handles the demote for the StorageDrivers :param cluster_ip: IP of the node to demote :param master_ip: IP of the master node """ client = SSHClient(cluster_ip, username='******') servicetype = ServiceTypeList.get_by_name('Arakoon') current_service = None remaining_ips = [] for service in servicetype.services: if service.name == 'arakoon-voldrv': if service.storagerouter.ip == cluster_ip: current_service = service else: remaining_ips.append(service.storagerouter.ip) if current_service is not None: print '* Shrink StorageDriver cluster' ArakoonInstaller.shrink_cluster(master_ip, cluster_ip, 'voldrv') if ServiceManager.has_service(current_service.name, client=client) is True: ServiceManager.stop_service(current_service.name, client=client) ServiceManager.remove_service(current_service.name, client=client) ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips) current_service.delete() for storagerouter in StorageRouterList.get_storagerouters(): ArakoonInstaller.deploy_to_slave(master_ip, storagerouter.ip, 'voldrv') StorageDriverController._configure_arakoon_to_volumedriver()
def on_demote(cluster_ip, master_ip, offline_node_ips=None): """ Handles the demote for the StorageDrivers :param cluster_ip: IP of the node to demote :param master_ip: IP of the master node :param offline_node_ips: IPs of nodes which are offline """ _ = master_ip if offline_node_ips is None: offline_node_ips = [] client = SSHClient( cluster_ip, username='******') if cluster_ip not in offline_node_ips else None servicetype = ServiceTypeList.get_by_name('Arakoon') current_service = None remaining_ips = [] for service in servicetype.services: if service.name == 'arakoon-voldrv': if service.storagerouter.ip == cluster_ip: current_service = service elif service.storagerouter.ip not in offline_node_ips: remaining_ips.append(service.storagerouter.ip) if current_service is not None: print '* Shrink StorageDriver cluster' ArakoonInstaller.shrink_cluster(cluster_ip, 'voldrv', offline_node_ips) if client is not None and ServiceManager.has_service( current_service.name, client=client) is True: ServiceManager.stop_service(current_service.name, client=client) ServiceManager.remove_service(current_service.name, client=client) ArakoonInstaller.restart_cluster_remove('voldrv', remaining_ips) current_service.delete() StorageDriverController._configure_arakoon_to_volumedriver()
def on_demote(cluster_ip, master_ip, offline_node_ips=None): """ Handles the demote for the StorageDrivers :param cluster_ip: IP of the node to demote :type cluster_ip: str :param master_ip: IP of the master node :type master_ip: str :param offline_node_ips: IPs of nodes which are offline :type offline_node_ips: list :return: None """ _ = master_ip if offline_node_ips is None: offline_node_ips = [] client = SSHClient( cluster_ip, username='******') if cluster_ip not in offline_node_ips else None servicetype = ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.ARAKOON) current_service = None remaining_ips = [] for service in servicetype.services: if service.name == 'arakoon-voldrv' and service.is_internal is True: # Externally managed arakoon cluster service does not have storage router if service.storagerouter.ip == cluster_ip: current_service = service elif service.storagerouter.ip not in offline_node_ips: remaining_ips.append(service.storagerouter.ip) if current_service is not None: StorageDriverController._logger.debug( '* Shrink StorageDriver cluster') cluster_name = str( EtcdConfiguration.get( '/ovs/framework/arakoon_clusters|voldrv')) ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip, cluster_name=cluster_name, offline_nodes=offline_node_ips) if client is not None and ServiceManager.has_service( current_service.name, client=client) is True: ServiceManager.stop_service(current_service.name, client=client) ServiceManager.remove_service(current_service.name, client=client) ArakoonInstaller.restart_cluster_remove(cluster_name, remaining_ips) current_service.delete() StorageDriverController._configure_arakoon_to_volumedriver( cluster_name=cluster_name)
def on_demote(cluster_ip, master_ip, offline_node_ips=None): """ Handles the demote for the StorageDrivers :param cluster_ip: IP of the node to demote :type cluster_ip: str :param master_ip: IP of the master node :type master_ip: str :param offline_node_ips: IPs of nodes which are offline :type offline_node_ips: list :return: None """ _ = master_ip if offline_node_ips is None: offline_node_ips = [] client = SSHClient(cluster_ip, username='******') if cluster_ip not in offline_node_ips else None servicetype = ServiceTypeList.get_by_name(ServiceType.SERVICE_TYPES.ARAKOON) current_service = None remaining_ips = [] for service in servicetype.services: if service.name == 'arakoon-voldrv' and service.is_internal is True: # Externally managed arakoon cluster service does not have storage router if service.storagerouter.ip == cluster_ip: current_service = service elif service.storagerouter.ip not in offline_node_ips: remaining_ips.append(service.storagerouter.ip) if current_service is not None: if len(remaining_ips) == 0: raise RuntimeError('Could not find any remaining arakoon nodes for the voldrv cluster') StorageDriverController._logger.debug('* Shrink StorageDriver cluster') cluster_name = str(Configuration.get('/ovs/framework/arakoon_clusters|voldrv')) ArakoonInstaller.shrink_cluster(deleted_node_ip=cluster_ip, remaining_node_ip=remaining_ips[0], cluster_name=cluster_name, offline_nodes=offline_node_ips) if client is not None and ServiceManager.has_service(current_service.name, client=client) is True: ServiceManager.stop_service(current_service.name, client=client) ServiceManager.remove_service(current_service.name, client=client) ArakoonInstaller.restart_cluster_remove(cluster_name, remaining_ips, filesystem=False) current_service.delete() StorageDriverController._configure_arakoon_to_volumedriver(cluster_name=cluster_name)
def test_cluster_maintenance(self): """ Validates whether a cluster can be correctly created """ Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]}) Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]}) structure = Helper.build_service_structure( {'storagerouters': [1, 2]} ) storagerouters = structure['storagerouters'] System._machine_id = {storagerouters[1].ip: '1', storagerouters[2].ip: '2'} # Create new cluster mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir = mountpoint + '/test_create_cluster' info = ArakoonInstaller.create_cluster('test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir) reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {}, 'files': []}, 'db': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[1].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY)) ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata']) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected) # Extending cluster mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir2 = mountpoint + '/test_extend_cluster' ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2) reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {'tlogs': {'dirs': {}, 'files': []}, 'db': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n{2}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('1,2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001 ), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only' SSHClient._run_returns[catchup_command] = None SSHClient._run_recordings = [] ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False) self.assertIn(catchup_command, SSHClient._run_recordings) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected) # Shrinking cluster ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test') reality = Helper.extract_dir_structure(base_dir) expected = {'dirs': {'arakoon': {'dirs': {'test': {'dirs': {}, 'files': []}}, 'files': []}}, 'files': []} self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format(ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format('2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001 )) self.assertEqual(Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = {'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True} self.assertDictEqual(reality, expected)
def test_cluster_maintenance(self): """ Validates whether a cluster can be correctly created """ Configuration.set('/ovs/framework/hosts/1/ports', {'arakoon': [10000, 10100]}) Configuration.set('/ovs/framework/hosts/2/ports', {'arakoon': [20000, 20100]}) structure = Helper.build_service_structure({'storagerouters': [1, 2]}) storagerouters = structure['storagerouters'] System._machine_id = { storagerouters[1].ip: '1', storagerouters[2].ip: '2' } # Create new cluster mountpoint = storagerouters[1].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir = mountpoint + '/test_create_cluster' info = ArakoonInstaller.create_cluster( 'test', ServiceType.ARAKOON_CLUSTER_TYPES.FWK, storagerouters[1].ip, base_dir) reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': { 'tlogs': { 'dirs': {}, 'files': [] }, 'db': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '1', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.start_cluster('test', storagerouters[1].ip, filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[1].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) self.assertFalse(client.exists(ArakoonInstaller.METADATA_KEY)) ArakoonInstaller.claim_cluster('test', storagerouters[1].ip, filesystem=False, metadata=info['metadata']) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected) # Extending cluster mountpoint = storagerouters[2].disks[0].partitions[0].mountpoint if os.path.exists(mountpoint) and mountpoint != '/': shutil.rmtree(mountpoint) base_dir2 = mountpoint + '/test_extend_cluster' ArakoonInstaller.extend_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test', base_dir2) reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': { 'tlogs': { 'dirs': {}, 'files': [] }, 'db': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n{2}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '1,2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '1', storagerouters[1].ip, 10000, base_dir, '1', 10001), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped catchup_command = 'arakoon --node 2 -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/test/config -catchup-only' SSHClient._run_returns[catchup_command] = None SSHClient._run_recordings = [] ArakoonInstaller.restart_cluster_add('test', [storagerouters[1].ip], storagerouters[2].ip, filesystem=False) self.assertIn(catchup_command, SSHClient._run_recordings) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected) # Shrinking cluster ArakoonInstaller.shrink_cluster(storagerouters[1].ip, storagerouters[2].ip, 'test') reality = Helper.extract_dir_structure(base_dir) expected = { 'dirs': { 'arakoon': { 'dirs': { 'test': { 'dirs': {}, 'files': [] } }, 'files': [] } }, 'files': [] } self.assertDictEqual(reality, expected) expected = '{0}\n\n{1}\n\n'.format( ArakoonInstallerTester.EXPECTED_CLUSTER_CONFIG.format( '2', 'test', ''), ArakoonInstallerTester.EXPECTED_NODE_CONFIG.format( '2', storagerouters[2].ip, 20000, base_dir2, '2', 20001)) self.assertEqual( Configuration.get(ArakoonInstaller.CONFIG_KEY.format('test'), raw=True), expected) # @TODO: assert service availability here. It should be stopped ArakoonInstaller.restart_cluster_remove('test', [storagerouters[2].ip], filesystem=False) # @TODO: assert the service is running config = ArakoonClusterConfig('test', filesystem=False) config.load_config(storagerouters[2].ip) client = ArakoonInstaller.build_client(config) reality = client.get(ArakoonInstaller.INTERNAL_CONFIG_KEY) self.assertEqual(reality, expected) reality = json.loads(client.get(ArakoonInstaller.METADATA_KEY)) expected = { 'cluster_name': 'test', 'cluster_type': 'FWK', 'in_use': True, 'internal': True } self.assertDictEqual(reality, expected)