def ovs_4509_validate_arakoon_collapse_test(): """ Validate arakoon collapse """ node_ips = [sr.ip for sr in GeneralStorageRouter.get_storage_routers()] node_ips.sort() for node_ip in node_ips: root_client = SSHClient(node_ip, username='******') arakoon_clusters = [] for service in ServiceList.get_services(): if service.is_internal is True and service.storagerouter.ip == node_ip and \ service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON, ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR): arakoon_clusters.append(service.name.replace('arakoon-', '')) for arakoon_cluster in arakoon_clusters: arakoon_config_path = Configuration.get_configuration_path('/ovs/arakoon/{0}/config'.format(arakoon_cluster)) tlog_location = '/opt/OpenvStorage/db/arakoon/{0}/tlogs'.format(arakoon_cluster) # read_tlog_dir with remote(node_ip, [Configuration]) as rem: config_contents = rem.Configuration.get('/ovs/arakoon/{0}/config'.format(arakoon_cluster), raw=True) for line in config_contents.splitlines(): if 'tlog_dir' in line: tlog_location = line.split()[-1] nr_of_tlogs = TestArakoon.get_nr_of_tlogs_in_folder(root_client, tlog_location) old_headdb_timestamp = 0 if root_client.file_exists('/'.join([tlog_location, 'head.db'])): old_headdb_timestamp = root_client.run(['stat', '--format=%Y', tlog_location + '/head.db']) if nr_of_tlogs <= 2: benchmark_command = ['arakoon', '--benchmark', '-n_clients', '1', '-max_n', '5_000', '-config', arakoon_config_path] root_client.run(benchmark_command) GenericController.collapse_arakoon() nr_of_tlogs = TestArakoon.get_nr_of_tlogs_in_folder(root_client, tlog_location) new_headdb_timestamp = root_client.run(['stat', '--format=%Y', tlog_location + '/head.db']) assert nr_of_tlogs <= 2,\ 'Arakoon collapse left {0} tlogs on the environment, expecting less than 2'.format(nr_of_tlogs) assert old_headdb_timestamp != new_headdb_timestamp,\ 'Timestamp of the head_db file was not changed in the process of collapsing tlogs'
def test_arakoon_collapse(self): """ Test the Arakoon collapse functionality """ # Set up the test structure = DalHelper.build_dal_structure( structure={'storagerouters': [1, 2]}) storagerouter_1 = structure['storagerouters'][1] storagerouter_2 = structure['storagerouters'][2] MockedSSHClient._run_returns[storagerouter_1.ip] = {} MockedSSHClient._run_returns[storagerouter_2.ip] = {} # Make sure we cover all Arakoon cluster types clusters_to_create = { ServiceType.ARAKOON_CLUSTER_TYPES.SD: [{ 'name': 'unittest-voldrv', 'internal': True, 'success': True }], ServiceType.ARAKOON_CLUSTER_TYPES.CFG: [{ 'name': 'unittest-cacc', 'internal': True, 'success': True }], ServiceType.ARAKOON_CLUSTER_TYPES.FWK: [{ 'name': 'unittest-ovsdb', 'internal': True, 'success': False }], ServiceType.ARAKOON_CLUSTER_TYPES.ABM: [{ 'name': 'unittest-cluster-1-abm', 'internal': True, 'success': False }, { 'name': 'unittest-random-abm-name', 'internal': False, 'success': True }], ServiceType.ARAKOON_CLUSTER_TYPES.NSM: [{ 'name': 'unittest-cluster-1-nsm_0', 'internal': True, 'success': True }] } self.assertEqual( first=sorted(clusters_to_create.keys()), second=sorted(ServiceType.ARAKOON_CLUSTER_TYPES.keys()), msg= 'An Arakoon cluster type has been removed or added, please update this test accordingly' ) # Create all Arakoon clusters and related services failed_clusters = [] external_clusters = [] successful_clusters = [] for cluster_type, cluster_infos in clusters_to_create.iteritems(): filesystem = cluster_type == ServiceType.ARAKOON_CLUSTER_TYPES.CFG for cluster_info in cluster_infos: internal = cluster_info['internal'] cluster_name = cluster_info['name'] base_dir = DalHelper.CLUSTER_DIR.format(cluster_name) arakoon_installer = ArakoonInstaller(cluster_name=cluster_name) arakoon_installer.create_cluster(cluster_type=cluster_type, ip=storagerouter_1.ip, base_dir=base_dir, internal=internal) arakoon_installer.start_cluster() arakoon_installer.extend_cluster(new_ip=storagerouter_2.ip, base_dir=base_dir) service_name = ArakoonInstaller.get_service_name_for_cluster( cluster_name=cluster_name) if cluster_type == ServiceType.ARAKOON_CLUSTER_TYPES.ABM: service_type = ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.ALBA_MGR) elif cluster_type == ServiceType.ARAKOON_CLUSTER_TYPES.NSM: service_type = ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.NS_MGR) else: service_type = ServiceTypeList.get_by_name( ServiceType.SERVICE_TYPES.ARAKOON) if internal is True: DalHelper.create_service( service_name=service_name, service_type=service_type, storagerouter=storagerouter_1, ports=arakoon_installer.ports[storagerouter_1.ip]) DalHelper.create_service( service_name=service_name, service_type=service_type, storagerouter=storagerouter_2, ports=arakoon_installer.ports[storagerouter_2.ip]) else: DalHelper.create_service(service_name=service_name, service_type=service_type) external_clusters.append(cluster_name) continue if cluster_info['success'] is True: if filesystem is True: config_path = ArakoonClusterConfig.CONFIG_FILE.format( cluster_name) else: config_path = Configuration.get_configuration_path( ArakoonClusterConfig.CONFIG_KEY.format( cluster_name)) MockedSSHClient._run_returns[storagerouter_1.ip][ 'arakoon --collapse-local 1 2 -config {0}'.format( config_path)] = None MockedSSHClient._run_returns[storagerouter_2.ip][ 'arakoon --collapse-local 2 2 -config {0}'.format( config_path)] = None successful_clusters.append(cluster_name) else: # For successful False clusters we don't emulate the collapse, thus making it fail failed_clusters.append(cluster_name) # Start collapse and make it fail for all clusters on StorageRouter 2 SSHClient._raise_exceptions[storagerouter_2.ip] = { 'users': ['ovs'], 'exception': UnableToConnectException('No route to host') } GenericController.collapse_arakoon() # Verify all log messages for each type of cluster generic_logs = Logger._logs.get('lib', {}) for cluster_name in successful_clusters + failed_clusters + external_clusters: collect_msg = ( 'DEBUG', 'Collecting info for cluster {0}'.format(cluster_name)) unreachable_msg = ( 'ERROR', 'Could not collapse any cluster on {0} (not reachable)'.format( storagerouter_2.name)) end_collapse_msg = ( 'DEBUG', 'Collapsing cluster {0} on {1} completed'.format( cluster_name, storagerouter_1.ip)) start_collapse_msg = ('DEBUG', 'Collapsing cluster {0} on {1}'.format( cluster_name, storagerouter_1.ip)) failed_collapse_msg = ( 'ERROR', 'Collapsing cluster {0} on {1} failed'.format( cluster_name, storagerouter_1.ip)) messages_to_validate = [] if cluster_name in successful_clusters: assert_function = self.assertIn messages_to_validate.append(collect_msg) messages_to_validate.append(unreachable_msg) messages_to_validate.append(start_collapse_msg) messages_to_validate.append(end_collapse_msg) elif cluster_name in failed_clusters: assert_function = self.assertIn messages_to_validate.append(collect_msg) messages_to_validate.append(unreachable_msg) messages_to_validate.append(start_collapse_msg) messages_to_validate.append(failed_collapse_msg) else: assert_function = self.assertNotIn messages_to_validate.append(collect_msg) messages_to_validate.append(start_collapse_msg) messages_to_validate.append(end_collapse_msg) for severity, message in messages_to_validate: if assert_function == self.assertIn: assert_message = 'Expected to find log message: {0}'.format( message) else: assert_message = 'Did not expect to find log message: {0}'.format( message) assert_function(member=message, container=generic_logs, msg=assert_message) if assert_function == self.assertIn: self.assertEqual( first=severity, second=generic_logs[message], msg='Log message {0} is of severity {1} expected {2}'. format(message, generic_logs[message], severity)) # Collapse should always have a 'finished' message since each cluster should be attempted to be collapsed for general_message in [ 'Arakoon collapse started', 'Arakoon collapse finished' ]: self.assertIn(member=general_message, container=generic_logs, msg='Expected to find log message: {0}'.format( general_message))
def test_collapse(): """ Test the arakoon collapsing :return: """ ArakoonCollapse.LOGGER.info("Starting validating arakoon collapse") node_ips = StoragerouterHelper.get_storagerouter_ips() node_ips.sort() for node_ip in node_ips: ArakoonCollapse.LOGGER.info( "Fetching arakoons on node `{0}`".format(node_ip)) arakoon_clusters = [] root_client = SSHClient(node_ip, username='******') # fetch arakoon clusters for service in ServiceList.get_services(): if service.is_internal is True and service.storagerouter.ip == node_ip and \ service.type.name in (ServiceType.SERVICE_TYPES.ARAKOON, ServiceType.SERVICE_TYPES.NS_MGR, ServiceType.SERVICE_TYPES.ALBA_MGR): arakoon_clusters.append( service.name.replace('arakoon-', '')) # perform collapse ArakoonCollapse.LOGGER.info( "Starting arakoon collapse on node `{0}`".format(node_ip)) for arakoon_cluster in arakoon_clusters: ArakoonCollapse.LOGGER.info( "Fetching `{0}` arakoon on node `{1}`".format( arakoon_cluster, node_ip)) arakoon_config_path = Configuration.get_configuration_path( '/ovs/arakoon/{0}/config'.format(arakoon_cluster)) tlog_location = '/opt/OpenvStorage/db/arakoon/{0}/tlogs'.format( arakoon_cluster) # read_tlog_dir with remote(node_ip, [Configuration]) as rem: config_contents = rem.Configuration.get( '/ovs/arakoon/{0}/config'.format(arakoon_cluster), raw=True) for line in config_contents.splitlines(): if 'tlog_dir' in line: tlog_location = line.split()[-1] nr_of_tlogs = ArakoonCollapse.get_nr_of_tlogs_in_folder( root_client, tlog_location) old_headdb_timestamp = 0 if root_client.file_exists('/'.join([tlog_location, 'head.db'])): old_headdb_timestamp = root_client.run([ 'stat', '--format=%Y', '{0}/{1}'.format(tlog_location, 'head.db') ]) if nr_of_tlogs <= 2: benchmark_command = [ 'arakoon', '--benchmark', '-n_clients', '1', '-max_n', '5_000', '-config', arakoon_config_path ] root_client.run(benchmark_command) ArakoonCollapse.LOGGER.info( "Collapsing arakoon `{0}` on node `{1}` ...".format( arakoon_cluster, node_ip)) GenericController.collapse_arakoon() nr_of_tlogs = ArakoonCollapse.get_nr_of_tlogs_in_folder( root_client, tlog_location) new_headdb_timestamp = root_client.run([ 'stat', '--format=%Y', '{0}/{1}'.format(tlog_location, 'head.db') ]) # perform assertion assert nr_of_tlogs <= 2,\ 'Arakoon collapse left {0} tlogs on the environment, expecting less than 2 in `{1}` on node `{1}`'\ .format(nr_of_tlogs, arakoon_cluster, node_ip) assert old_headdb_timestamp != new_headdb_timestamp,\ 'Timestamp of the head_db file was not changed ' \ 'in the process of collapsing tlogs of arakoon `{0}` on node `{1}`'\ .format(arakoon_cluster, node_ip) ArakoonCollapse.LOGGER.info( "Successfully collapsed arakoon `{0}` on node `{1}`". format(arakoon_cluster, node_ip)) ArakoonCollapse.LOGGER.info("Finished validating arakoon collapsing")