def test_alba_arakoon_checkup(self): """ Validates whether the ALBA Arakoon checkup works (Manual and Scheduled) """ ovs_structure = DalHelper.build_dal_structure( structure={'storagerouters': [1]}) alba_structure = AlbaDalHelper.build_dal_structure( structure={'alba_backends': [[1, 'LOCAL']]}) ############################# # SCHEDULED_ARAKOON_CHECKUP # ############################# # Create an ABM and NSM cluster for ALBA Backend 1 and do some basic validations sr_1 = ovs_structure['storagerouters'][1] ab_1 = alba_structure['alba_backends'][1] MockedSSHClient._run_returns[sr_1.ip] = {} MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/nsm_host_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-nsm_0/db'] = None AlbaController.add_cluster(ab_1.guid) abm_cluster_name = '{0}-abm'.format(ab_1.name) nsm_cluster_name = '{0}-nsm_0'.format(ab_1.name) arakoon_clusters = sorted(Configuration.list('/ovs/arakoon')) self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) abm_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name( cluster_name=abm_cluster_name) nsm_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name( cluster_name=nsm_cluster_name) self.assertTrue(expr=abm_metadata['in_use']) self.assertTrue(expr=nsm_metadata['in_use']) # Run scheduled Arakoon checkup and validate amount of Arakoon clusters did not change AlbaArakoonController.scheduled_alba_arakoon_checkup() self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=1) self.assertEqual(first=len(ab_1.nsm_clusters), second=1) self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services), second=1) # Create 2 additional StorageRouters srs = DalHelper.build_dal_structure( structure={'storagerouters': [2, 3]}, previous_structure=ovs_structure)['storagerouters'] sr_2 = srs[2] sr_3 = srs[3] # Run scheduled checkup again and do some validations MockedSSHClient._run_returns[sr_2.ip] = {} MockedSSHClient._run_returns[sr_3.ip] = {} MockedSSHClient._run_returns[sr_2.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_2/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_3.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_3/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_2.ip][ 'arakoon --node {0} -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/backend_1-abm/config -catchup-only' .format(sr_2.machine_id)] = None MockedSSHClient._run_returns[sr_3.ip][ 'arakoon --node {0} -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/backend_1-abm/config -catchup-only' .format(sr_3.machine_id)] = None AlbaArakoonController.scheduled_alba_arakoon_checkup() self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=3) # Gone up from 1 to 3 self.assertEqual(first=len(ab_1.nsm_clusters), second=1) self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services), second=1) # Still 1 since NSM checkup hasn't ran yet # Make sure 1 StorageRouter is unreachable SSHClient._raise_exceptions[sr_3.ip] = { 'users': ['ovs'], 'exception': UnableToConnectException('No route to host') } AlbaArakoonController.scheduled_alba_arakoon_checkup() alba_logs = Logger._logs.get('lib', []) self.assertIn( member='Storage Router with IP {0} is not reachable'.format( sr_3.ip), container=alba_logs) ########################## # MANUAL_ARAKOON_CHECKUP # ########################## AlbaDalHelper.setup() # Clear everything ovs_structure = DalHelper.build_dal_structure( structure={'storagerouters': [1]}) alba_structure = AlbaDalHelper.build_dal_structure( structure={'alba_backends': [[1, 'LOCAL']]}) sr_1 = ovs_structure['storagerouters'][1] ab_1 = alba_structure['alba_backends'][1] MockedSSHClient._run_returns[sr_1.ip] = {} MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/nsm_host_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-nsm_0/db'] = None AlbaController.add_cluster(ab_1.guid) # Run manual Arakoon checkup and validate amount of Arakoon clusters did not change AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=[], abm_cluster=None) self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=1) self.assertEqual(first=len(ab_1.nsm_clusters), second=1) self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services), second=1) # Test some error paths with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=['no_abm_cluster_passed']) self.assertEqual( first=raise_info.exception.message, second='Both ABM cluster and NSM clusters must be provided') with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=[], abm_cluster='no_nsm_clusters_passed') self.assertEqual( first=raise_info.exception.message, second='Both ABM cluster and NSM clusters must be provided') with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=[nsm_cluster_name], abm_cluster=abm_cluster_name) self.assertEqual(first=raise_info.exception.message, second='Cluster {0} has already been claimed'.format( abm_cluster_name)) with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=['non-existing-nsm-cluster'], abm_cluster='non-existing-abm-cluster') self.assertEqual( first=raise_info.exception.message, second= 'Could not find an Arakoon cluster with name: non-existing-abm-cluster' ) # Recreate ALBA Backend with Arakoon clusters AlbaDalHelper.setup() # Clear everything ovs_structure = DalHelper.build_dal_structure( structure={'storagerouters': [1]}) alba_structure = AlbaDalHelper.build_dal_structure( structure={'alba_backends': [[1, 'LOCAL']]}) sr_1 = ovs_structure['storagerouters'][1] ab_1 = alba_structure['alba_backends'][1] # Create some Arakoon clusters to be claimed by the manual checkup for cluster_name, cluster_type in { 'manual-abm-1': ServiceType.ARAKOON_CLUSTER_TYPES.ABM, 'manual-abm-2': ServiceType.ARAKOON_CLUSTER_TYPES.ABM, 'manual-nsm-1': ServiceType.ARAKOON_CLUSTER_TYPES.NSM, 'manual-nsm-2': ServiceType.ARAKOON_CLUSTER_TYPES.NSM, 'manual-nsm-3': ServiceType.ARAKOON_CLUSTER_TYPES.NSM }.iteritems(): arakoon_installer = ArakoonInstaller(cluster_name=cluster_name) arakoon_installer.create_cluster( cluster_type=cluster_type, ip=sr_1.ip, base_dir=DalHelper.CLUSTER_DIR.format(cluster_name), internal=False) arakoon_installer.start_cluster() arakoon_installer.unclaim_cluster() AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=['manual-nsm-1', 'manual-nsm-3'], abm_cluster='manual-abm-2') # Validate the correct clusters have been claimed by the manual checkup unused_abms = ArakoonInstaller.get_unused_arakoon_clusters( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.ABM) unused_nsms = ArakoonInstaller.get_unused_arakoon_clusters( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM) self.assertEqual(first=len(unused_abms), second=1) self.assertEqual(first=len(unused_nsms), second=1) self.assertEqual(first=unused_abms[0]['cluster_name'], second='manual-abm-1') self.assertEqual(first=unused_nsms[0]['cluster_name'], second='manual-nsm-2')
def nsm_checkup(alba_backend_guid=None, min_internal_nsms=1, external_nsm_cluster_names=None): # type: (Optional[str], Optional[int], Optional[List[str]]) -> None """ Validates the current NSM setup/configuration and takes actions where required. Assumptions: * A 2 node NSM is considered safer than a 1 node NSM. * When adding an NSM, the nodes with the least amount of NSM participation are preferred :param alba_backend_guid: Run for a specific ALBA Backend :type alba_backend_guid: str :param min_internal_nsms: Minimum amount of NSM hosts that need to be provided :type min_internal_nsms: int :param external_nsm_cluster_names: Information about the additional clusters to claim (only for externally managed Arakoon clusters) :type external_nsm_cluster_names: list :return: None :rtype: NoneType """ ############### # Validations # ############### if external_nsm_cluster_names is None: external_nsm_cluster_names = [] AlbaArakoonController._logger.info('NSM checkup started') if min_internal_nsms < 1: raise ValueError( 'Minimum amount of NSM clusters must be 1 or more') if not isinstance(external_nsm_cluster_names, list): raise ValueError( "'external_nsm_cluster_names' must be of type 'list'") if len(external_nsm_cluster_names) > 0: if alba_backend_guid is None: raise ValueError( 'Additional NSMs can only be configured for a specific ALBA Backend' ) if min_internal_nsms > 1: raise ValueError( "'min_internal_nsms' and 'external_nsm_cluster_names' are mutually exclusive" ) external_nsm_cluster_names = list(set( external_nsm_cluster_names)) # Remove duplicate cluster names for cluster_name in external_nsm_cluster_names: try: ArakoonInstaller.get_arakoon_metadata_by_cluster_name( cluster_name=cluster_name) except NotFoundException: raise ValueError( 'Arakoon cluster with name {0} does not exist'.format( cluster_name)) if alba_backend_guid is None: alba_backends = [ alba_backend for alba_backend in AlbaBackendList.get_albabackends() if alba_backend.backend.status == 'RUNNING' ] else: alba_backends = [AlbaBackend(alba_backend_guid)] masters = StorageRouterList.get_masters() storagerouters = set() for alba_backend in alba_backends: if alba_backend.abm_cluster is None: raise ValueError( 'No ABM cluster found for ALBA Backend {0}'.format( alba_backend.name)) if len(alba_backend.abm_cluster.abm_services) == 0: raise ValueError( 'ALBA Backend {0} does not have any registered ABM services' .format(alba_backend.name)) if len(alba_backend.nsm_clusters) + len( external_nsm_cluster_names) > MAX_NSM_AMOUNT: raise ValueError( 'The maximum of {0} NSM Arakoon clusters will be exceeded. Amount of clusters that can be deployed for this ALBA Backend: {1}' .format(MAX_NSM_AMOUNT, MAX_NSM_AMOUNT - len(alba_backend.nsm_clusters))) # Validate enough externally managed Arakoon clusters are available if alba_backend.abm_cluster.abm_services[ 0].service.is_internal is False: unused_cluster_names = set([ cluster_info['cluster_name'] for cluster_info in ArakoonInstaller.get_unused_arakoon_clusters( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM) ]) if set(external_nsm_cluster_names).difference( unused_cluster_names): raise ValueError( 'Some of the provided cluster_names have already been claimed before' ) storagerouters.update( set(masters) ) # For externally managed we need an available master node else: for abm_service in alba_backend.abm_cluster.abm_services: # For internally managed we need all StorageRouters online storagerouters.add(abm_service.service.storagerouter) for nsm_cluster in alba_backend.nsm_clusters: # For internally managed we need all StorageRouters online for nsm_service in nsm_cluster.nsm_services: storagerouters.add(nsm_service.service.storagerouter) ssh_clients = {} for storagerouter in storagerouters: try: ssh_clients[storagerouter] = SSHClient(endpoint=storagerouter) except UnableToConnectException: raise RuntimeError( 'StorageRouter {0} with IP {1} is not reachable'.format( storagerouter.name, storagerouter.ip)) version_str = AlbaArakoonInstaller.get_alba_version_string() nsm_installer = NSMInstaller(version_str=version_str, ssh_clients=ssh_clients) ################## # Check Clusters # ################## safety = Configuration.get( '/ovs/framework/plugins/alba/config|nsm.safety') maxload = Configuration.get( '/ovs/framework/plugins/alba/config|nsm.maxload') AlbaArakoonController._logger.debug( 'NSM safety is configured at: {0}'.format(safety)) AlbaArakoonController._logger.debug( 'NSM max load is configured at: {0}'.format(maxload)) master_client = None failed_backends = [] for alba_backend in alba_backends: try: # Gather information AlbaArakoonController._logger.info( 'ALBA Backend {0} - Ensuring NSM safety'.format( alba_backend.name)) internal = AlbaArakoonInstaller.is_internally_managed( alba_backend) nsm_loads = AlbaArakoonController.get_nsm_loads(alba_backend) nsm_storagerouters = AlbaArakoonController.get_nsms_per_storagerouter( alba_backend) sorted_nsm_clusters = sorted(alba_backend.nsm_clusters, key=lambda k: k.number) if not internal and len(external_nsm_cluster_names) > 0: for sr, cl in ssh_clients.iteritems(): if sr.node_type == 'MASTER': master_client = cl break if master_client is None: # Internal is False and we specified the NSM clusters to claim, but no MASTER nodes online raise ValueError( 'Could not find an online master node') AlbaArakoonController._logger.debug( 'ALBA Backend {0} - Arakoon clusters are {1} managed'. format(alba_backend.name, 'internally' if internal is True else 'externally')) for nsm_number, nsm_load in nsm_loads.iteritems(): AlbaArakoonController._logger.debug( 'ALBA Backend {0} - NSM Cluster {1} - Load {2}'.format( alba_backend.name, nsm_number, nsm_load)) for sr, count in nsm_storagerouters.iteritems(): AlbaArakoonController._logger.debug( 'ALBA Backend {0} - StorageRouter {1} - NSM Services {2}' .format(alba_backend.name, sr.name, count)) if internal: # Extend existing NSM clusters if safety not met for nsm_cluster in sorted_nsm_clusters: AlbaArakoonController._logger.debug( 'ALBA Backend {0} - Processing NSM {1} - Expected safety {2} - Current safety {3}' .format(alba_backend.name, nsm_cluster.number, safety, len(nsm_cluster.nsm_services))) AlbaArakoonController.ensure_nsm_cluster_safety( nsm_cluster, nsm_storagerouters, nsm_installer=nsm_installer) AlbaArakoonController.ensure_nsm_clusters_load( alba_backend, nsms_per_storagerouter=nsm_storagerouters, ssh_clients=ssh_clients, version_str=version_str, min_internal_nsms=min_internal_nsms, external_nsm_cluster_names=external_nsm_cluster_names) except Exception: AlbaArakoonController._logger.exception( 'NSM Checkup failed for Backend {0}'.format( alba_backend.name)) failed_backends.append(alba_backend.name)