def test_alba_arakoon_checkup(self): """ Validates whether the ALBA Arakoon checkup works (Manual and Scheduled) """ ovs_structure = DalHelper.build_dal_structure( structure={'storagerouters': [1]}) alba_structure = AlbaDalHelper.build_dal_structure( structure={'alba_backends': [[1, 'LOCAL']]}) ############################# # SCHEDULED_ARAKOON_CHECKUP # ############################# # Create an ABM and NSM cluster for ALBA Backend 1 and do some basic validations sr_1 = ovs_structure['storagerouters'][1] ab_1 = alba_structure['alba_backends'][1] MockedSSHClient._run_returns[sr_1.ip] = {} MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/nsm_host_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-nsm_0/db'] = None AlbaController.add_cluster(ab_1.guid) abm_cluster_name = '{0}-abm'.format(ab_1.name) nsm_cluster_name = '{0}-nsm_0'.format(ab_1.name) arakoon_clusters = sorted(Configuration.list('/ovs/arakoon')) self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) abm_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name( cluster_name=abm_cluster_name) nsm_metadata = ArakoonInstaller.get_arakoon_metadata_by_cluster_name( cluster_name=nsm_cluster_name) self.assertTrue(expr=abm_metadata['in_use']) self.assertTrue(expr=nsm_metadata['in_use']) # Run scheduled Arakoon checkup and validate amount of Arakoon clusters did not change AlbaArakoonController.scheduled_alba_arakoon_checkup() self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=1) self.assertEqual(first=len(ab_1.nsm_clusters), second=1) self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services), second=1) # Create 2 additional StorageRouters srs = DalHelper.build_dal_structure( structure={'storagerouters': [2, 3]}, previous_structure=ovs_structure)['storagerouters'] sr_2 = srs[2] sr_3 = srs[3] # Run scheduled checkup again and do some validations MockedSSHClient._run_returns[sr_2.ip] = {} MockedSSHClient._run_returns[sr_3.ip] = {} MockedSSHClient._run_returns[sr_2.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_2/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_3.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_3/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_2.ip][ 'arakoon --node {0} -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/backend_1-abm/config -catchup-only' .format(sr_2.machine_id)] = None MockedSSHClient._run_returns[sr_3.ip][ 'arakoon --node {0} -config file://opt/OpenvStorage/config/framework.json?key=/ovs/arakoon/backend_1-abm/config -catchup-only' .format(sr_3.machine_id)] = None AlbaArakoonController.scheduled_alba_arakoon_checkup() self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=3) # Gone up from 1 to 3 self.assertEqual(first=len(ab_1.nsm_clusters), second=1) self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services), second=1) # Still 1 since NSM checkup hasn't ran yet # Make sure 1 StorageRouter is unreachable SSHClient._raise_exceptions[sr_3.ip] = { 'users': ['ovs'], 'exception': UnableToConnectException('No route to host') } AlbaArakoonController.scheduled_alba_arakoon_checkup() alba_logs = Logger._logs.get('lib', []) self.assertIn( member='Storage Router with IP {0} is not reachable'.format( sr_3.ip), container=alba_logs) ########################## # MANUAL_ARAKOON_CHECKUP # ########################## AlbaDalHelper.setup() # Clear everything ovs_structure = DalHelper.build_dal_structure( structure={'storagerouters': [1]}) alba_structure = AlbaDalHelper.build_dal_structure( structure={'alba_backends': [[1, 'LOCAL']]}) sr_1 = ovs_structure['storagerouters'][1] ab_1 = alba_structure['alba_backends'][1] MockedSSHClient._run_returns[sr_1.ip] = {} MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/albamgr_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-abm/db'] = None MockedSSHClient._run_returns[sr_1.ip][ 'ln -s /usr/lib/alba/nsm_host_plugin.cmxs /tmp/unittest/sr_1/disk_1/partition_1/arakoon/backend_1-nsm_0/db'] = None AlbaController.add_cluster(ab_1.guid) # Run manual Arakoon checkup and validate amount of Arakoon clusters did not change AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=[], abm_cluster=None) self.assertListEqual(list1=[abm_cluster_name, nsm_cluster_name], list2=arakoon_clusters) self.assertEqual(first=len(ab_1.abm_cluster.abm_services), second=1) self.assertEqual(first=len(ab_1.nsm_clusters), second=1) self.assertEqual(first=len(ab_1.nsm_clusters[0].nsm_services), second=1) # Test some error paths with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=['no_abm_cluster_passed']) self.assertEqual( first=raise_info.exception.message, second='Both ABM cluster and NSM clusters must be provided') with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=[], abm_cluster='no_nsm_clusters_passed') self.assertEqual( first=raise_info.exception.message, second='Both ABM cluster and NSM clusters must be provided') with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=[nsm_cluster_name], abm_cluster=abm_cluster_name) self.assertEqual(first=raise_info.exception.message, second='Cluster {0} has already been claimed'.format( abm_cluster_name)) with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=['non-existing-nsm-cluster'], abm_cluster='non-existing-abm-cluster') self.assertEqual( first=raise_info.exception.message, second= 'Could not find an Arakoon cluster with name: non-existing-abm-cluster' ) # Recreate ALBA Backend with Arakoon clusters AlbaDalHelper.setup() # Clear everything ovs_structure = DalHelper.build_dal_structure( structure={'storagerouters': [1]}) alba_structure = AlbaDalHelper.build_dal_structure( structure={'alba_backends': [[1, 'LOCAL']]}) sr_1 = ovs_structure['storagerouters'][1] ab_1 = alba_structure['alba_backends'][1] # Create some Arakoon clusters to be claimed by the manual checkup for cluster_name, cluster_type in { 'manual-abm-1': ServiceType.ARAKOON_CLUSTER_TYPES.ABM, 'manual-abm-2': ServiceType.ARAKOON_CLUSTER_TYPES.ABM, 'manual-nsm-1': ServiceType.ARAKOON_CLUSTER_TYPES.NSM, 'manual-nsm-2': ServiceType.ARAKOON_CLUSTER_TYPES.NSM, 'manual-nsm-3': ServiceType.ARAKOON_CLUSTER_TYPES.NSM }.iteritems(): arakoon_installer = ArakoonInstaller(cluster_name=cluster_name) arakoon_installer.create_cluster( cluster_type=cluster_type, ip=sr_1.ip, base_dir=DalHelper.CLUSTER_DIR.format(cluster_name), internal=False) arakoon_installer.start_cluster() arakoon_installer.unclaim_cluster() AlbaArakoonController.manual_alba_arakoon_checkup( alba_backend_guid=ab_1.guid, nsm_clusters=['manual-nsm-1', 'manual-nsm-3'], abm_cluster='manual-abm-2') # Validate the correct clusters have been claimed by the manual checkup unused_abms = ArakoonInstaller.get_unused_arakoon_clusters( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.ABM) unused_nsms = ArakoonInstaller.get_unused_arakoon_clusters( cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.NSM) self.assertEqual(first=len(unused_abms), second=1) self.assertEqual(first=len(unused_nsms), second=1) self.assertEqual(first=unused_abms[0]['cluster_name'], second='manual-abm-1') self.assertEqual(first=unused_nsms[0]['cluster_name'], second='manual-nsm-2')
def add_arakoon(cluster_name, storagerouter_ip, cluster_basedir, service_type=ServiceType.ARAKOON_CLUSTER_TYPES.FWK): """ Adds a external arakoon to a storagerouter :param cluster_name: name of the new arakoon cluster :type cluster_name: str :param service_type: type of plugin for arakoon (DEFAULT=ServiceType.ARAKOON_CLUSTER_TYPES.FWK) * FWK * ABM * NSM :type service_type: ovs.dal.hybrids.ServiceType.ARAKOON_CLUSTER_TYPES :param storagerouter_ip: ip of a storagerouter :type storagerouter_ip: str :param cluster_basedir: absolute path for the new arakoon cluster :type cluster_basedir: str :return: """ client = SSHClient(storagerouter_ip, username='******') # create required directories if not client.dir_exists(cluster_basedir): client.dir_create(cluster_basedir) # determine plugin if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.FWK: plugins = None elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: plugins = { AlbaController.ABM_PLUGIN: AlbaController.ALBA_VERSION_GET } elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: plugins = { AlbaController.NSM_PLUGIN: AlbaController.ALBA_VERSION_GET } else: raise RuntimeError( "Incompatible Arakoon cluster type selected: {0}".format( service_type)) ArakoonSetup.LOGGER.info( "Starting creation of new arakoon cluster with name `{0}`, servicetype `{1}`, ip `{2}`, base_dir `{3}`" .format(cluster_name, service_type, storagerouter_ip, cluster_basedir)) arakoon_installer = ArakoonInstaller(cluster_name) arakoon_installer.create_cluster( cluster_type=service_type, ip=storagerouter_ip, base_dir=cluster_basedir, plugins=plugins, locked=False, internal=False, log_sinks=Logger.get_sink_path('automation_lib_arakoon_server'), crash_log_sinks=Logger.get_sink_path( 'automation_lib_arakoon_server_crash')) if service_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: client.run([ 'ln', '-s', '/usr/lib/alba/albamgr_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name) ]) elif service_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: client.run([ 'ln', '-s', '/usr/lib/alba/nsm_host_plugin.cmxs', '{0}/arakoon/{1}/db'.format(cluster_basedir, cluster_name) ]) arakoon_installer.start_cluster() arakoon_installer.unclaim_cluster() ArakoonSetup.LOGGER.info( "Finished creation of new arakoon cluster with name `{0}`, servicetype `{1}`, ip `{2}`, base_dir `{3}`" .format(cluster_name, service_type, storagerouter_ip, cluster_basedir))
def test_nsm_checkup_external(self): """ Validates whether the NSM checkup works for externally managed Arakoon clusters """ Configuration.set('/ovs/framework/plugins/alba/config|nsm.safety', 1) Configuration.set('/ovs/framework/plugins/alba/config|nsm.maxload', 10) structure = DalHelper.build_dal_structure(structure={'storagerouters': [1, 2, 3]}) alba_structure = AlbaDalHelper.build_dal_structure(structure={'alba_backends': [[1, 'LOCAL']]}) alba_backend = alba_structure['alba_backends'][1] storagerouter_1 = structure['storagerouters'][1] storagerouter_2 = structure['storagerouters'][2] # Validate some logic for externally managed arakoons during NSM checkup with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.nsm_checkup(external_nsm_cluster_names=['test']) # No ALBA Backend specified self.assertEqual(first=str(raise_info.exception), second='Additional NSMs can only be configured for a specific ALBA Backend') with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, min_internal_nsms=2, external_nsm_cluster_names=['test']) self.assertEqual(first=str(raise_info.exception), second="'min_internal_nsms' and 'external_nsm_cluster_names' are mutually exclusive") with self.assertRaises(ValueError) as raise_info: # noinspection PyTypeChecker AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names={}) # NSM cluster names must be a list self.assertEqual(first=str(raise_info.exception), second="'external_nsm_cluster_names' must be of type 'list'") with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=['non-existing-cluster']) # non-existing cluster names should raise self.assertEqual(first=str(raise_info.exception), second="Arakoon cluster with name non-existing-cluster does not exist") # Create an external ABM and NSM Arakoon cluster external_abm_1 = 'backend_1-abm' external_nsm_1 = 'backend_1-nsm_0' external_nsm_2 = 'backend_1-nsm_1' for cluster_name, cluster_type in {external_abm_1: 'ABM', external_nsm_1: 'NSM', external_nsm_2: 'NSM'}.iteritems(): arakoon_installer = ArakoonInstaller(cluster_name=cluster_name) arakoon_installer.create_cluster(cluster_type=cluster_type, ip=storagerouter_1.ip, base_dir='/tmp', internal=False) arakoon_installer.extend_cluster(new_ip=storagerouter_2.ip, base_dir='/tmp') arakoon_installer.start_cluster() arakoon_installer.unclaim_cluster() self.assertDictEqual(d1={'cluster_name': cluster_name, 'cluster_type': cluster_type, 'internal': False, 'in_use': False}, d2=arakoon_installer.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)) # Let the 'add_cluster` claim the externally managed clusters and model the services Logger._logs = {} AlbaController.add_cluster(alba_backend_guid=alba_backend.guid, abm_cluster=external_abm_1, nsm_clusters=[external_nsm_1]) # Only claim external_nsm_1 for cluster_name, cluster_type in {external_abm_1: 'ABM', external_nsm_1: 'NSM', external_nsm_2: 'NSM'}.iteritems(): arakoon_installer = ArakoonInstaller(cluster_name=cluster_name) self.assertDictEqual(d1={'cluster_name': cluster_name, 'cluster_type': cluster_type, 'internal': False, 'in_use': False if cluster_name == external_nsm_2 else True}, d2=arakoon_installer.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name)) log_found = False for log_record in Logger._logs.get('lib', []): if 'NSM load OK' in log_record: log_found = True break self.assertTrue(expr=log_found) self.assertEqual(first=1, second=len(alba_backend.abm_cluster.abm_services)) self.assertEqual(first=1, second=len(alba_backend.nsm_clusters)) self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[0].nsm_services)) self.assertIsNone(obj=alba_backend.abm_cluster.abm_services[0].service.storagerouter) self.assertIsNone(obj=alba_backend.nsm_clusters[0].nsm_services[0].service.storagerouter) self.assertListEqual(VirtualAlbaBackend.run_log['backend_1-abm'], [['update_abm_client_config'], ['add_nsm_host', 'backend_1-nsm_0'], ['update_maintenance_config','--eviction-type-random'], ['update_maintenance_config','enable-auto-cleanup-deleted-namespaces-days']]) # Add cluster already invokes a NSM checkup, so nothing should have changed VirtualAlbaBackend.run_log['backend_1-abm'] = [] AlbaArakoonController.nsm_checkup() self.assertListEqual(list1=[], list2=VirtualAlbaBackend.run_log['backend_1-abm']) # Overload the only NSM and run NSM checkup. This should log a critical message, but change nothing VirtualAlbaBackend.data['backend_1-abm']['nsms'][0]['namespaces_count'] = 25 Logger._logs = {} AlbaArakoonController.nsm_checkup() log_found = False for log_record in Logger._logs.get('lib', []): if 'All NSM clusters are overloaded' in log_record: log_found = True break self.assertTrue(expr=log_found) self.assertEqual(first=1, second=len(alba_backend.abm_cluster.abm_services)) self.assertEqual(first=1, second=len(alba_backend.nsm_clusters)) self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[0].nsm_services)) self.assertIsNone(obj=alba_backend.abm_cluster.abm_services[0].service.storagerouter) self.assertIsNone(obj=alba_backend.nsm_clusters[0].nsm_services[0].service.storagerouter) self.assertListEqual(list1=[], list2=VirtualAlbaBackend.run_log['backend_1-abm']) # Validate a maximum of 50 NSMs can be deployed current_nsms = [nsm_cluster.number for nsm_cluster in alba_backend.nsm_clusters] alba_structure = AlbaDalHelper.build_dal_structure( structure={'alba_nsm_clusters': [(1, 50)]}, # (<abackend_id>, <amount_of_nsm_clusters>) previous_structure=alba_structure ) # Try to add 1 additional NSM with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=[external_nsm_2]) self.assertEqual(first=str(raise_info.exception), second='The maximum of 50 NSM Arakoon clusters will be exceeded. Amount of clusters that can be deployed for this ALBA Backend: 0') # Remove the unused NSM clusters again for nsm_cluster in alba_structure['alba_nsm_clusters'][1][len(current_nsms):]: for nsm_service in nsm_cluster.nsm_services: nsm_service.delete() nsm_service.service.delete() nsm_cluster.delete() # Try to add a previously claimed NSM cluster with self.assertRaises(ValueError) as raise_info: AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=[external_nsm_1]) # The provided cluster_name to claim has already been claimed self.assertEqual(first=str(raise_info.exception), second='Some of the provided cluster_names have already been claimed before') # Add a 2nd NSM cluster AlbaArakoonController.nsm_checkup(alba_backend_guid=alba_backend.guid, external_nsm_cluster_names=[external_nsm_2]) self.assertEqual(first=1, second=len(alba_backend.abm_cluster.abm_services)) self.assertEqual(first=2, second=len(alba_backend.nsm_clusters)) self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[0].nsm_services)) self.assertEqual(first=1, second=len(alba_backend.nsm_clusters[1].nsm_services)) self.assertIsNone(obj=alba_backend.abm_cluster.abm_services[0].service.storagerouter) self.assertIsNone(obj=alba_backend.nsm_clusters[0].nsm_services[0].service.storagerouter) self.assertIsNone(obj=alba_backend.nsm_clusters[1].nsm_services[0].service.storagerouter) self.assertListEqual(list1=[['add_nsm_host', 'backend_1-nsm_1']], list2=VirtualAlbaBackend.run_log['backend_1-abm']) for cluster_name, cluster_type in {external_abm_1: 'ABM', external_nsm_1: 'NSM', external_nsm_2: 'NSM'}.iteritems(): arakoon_installer = ArakoonInstaller(cluster_name=cluster_name) self.assertDictEqual(d1={'cluster_name': cluster_name, 'cluster_type': cluster_type, 'internal': False, 'in_use': True}, d2=arakoon_installer.get_arakoon_metadata_by_cluster_name(cluster_name=cluster_name))