def setup(): """ Setup for Arakoon package, will be executed when any test in this package is being executed Make necessary changes before being able to run the tests :return: None """ autotest_config = General.get_config() backend_name = autotest_config.get('backend', 'name') assert backend_name, 'Please fill out a backend name in the autotest.cfg file' backend = GeneralBackend.get_by_name(backend_name) if backend is not None: GeneralAlba.remove_alba_backend(backend.alba_backend) for storagerouter in GeneralStorageRouter.get_masters(): root_client = SSHClient(storagerouter, username='******') if GeneralService.get_service_status(name='ovs-scheduled-tasks', client=root_client) is True: GeneralService.stop_service(name='ovs-scheduled-tasks', client=root_client) storagerouters = GeneralStorageRouter.get_storage_routers() for sr in storagerouters: root_client = SSHClient(sr, username='******') GeneralDisk.add_db_role(sr) for location in TEST_CLEANUP: root_client.run('rm -rf {0}'.format(location)) GeneralAlba.add_alba_backend(backend_name) GeneralArakoon.voldrv_arakoon_checkup()
def teardown(): """ Teardown for Arakoon package, will be executed when all started tests in this package have ended Removal actions of possible things left over after the test-run :return: None """ autotest_config = General.get_config() backend_name = autotest_config.get('backend', 'name') backend = GeneralBackend.get_by_name(backend_name) if backend is not None: GeneralAlba.remove_alba_backend(backend.alba_backend) for storagerouter in GeneralStorageRouter.get_masters(): root_client = SSHClient(storagerouter, username='******') if GeneralService.get_service_status(name='ovs-scheduled-tasks', client=root_client) is False: GeneralService.start_service(name='ovs-scheduled-tasks', client=root_client) for location in TEST_CLEANUP: root_client.run('rm -rf {0}'.format(location)) for key in KEY_CLEANUP: if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True): EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))
def validate_alba_backend_removal(alba_backend_info): """ Validate whether the backend has been deleted properly alba_backend_info should be a dictionary containing: - guid - name - maintenance_service_names :param alba_backend_info: Information about the backend :return: None """ Toolbox.verify_required_params(actual_params=alba_backend_info, required_params={'name': (str, None), 'guid': (str, Toolbox.regex_guid), 'maintenance_service_names': (list, None)}, exact_match=True) alba_backend_guid = alba_backend_info['guid'] alba_backend_name = alba_backend_info['name'] backend = GeneralBackend.get_by_name(alba_backend_name) assert backend is None,\ 'Still found a backend in the model with name {0}'.format(alba_backend_name) # Validate services removed from model for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.ALBA_MGR): assert service.name != '{0}-abm'.format(alba_backend_name),\ 'An AlbaManager service has been found with name {0}'.format(alba_backend_name) for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.NS_MGR): assert service.name.startswith('{0}-nsm_'.format(alba_backend_name)) is False,\ 'An NamespaceManager service has been found with name {0}'.format(alba_backend_name) # Validate ALBA backend configuration structure alba_backend_key = '/ovs/alba/backends' actual_configuration_keys = [key for key in Configuration.list(alba_backend_key)] assert alba_backend_guid not in actual_configuration_keys,\ 'Configuration still contains an entry in {0} with guid {1}'.format(alba_backend_key, alba_backend_guid) # Validate Arakoon configuration structure arakoon_keys = [key for key in Configuration.list('/ovs/arakoon') if key.startswith(alba_backend_name)] assert len(arakoon_keys) == 0,\ 'Configuration still contains configurations for clusters: {0}'.format(', '.join(arakoon_keys)) # Validate services for storagerouter in GeneralStorageRouter.get_storage_routers(): root_client = SSHClient(endpoint=storagerouter, username='******') maintenance_services = alba_backend_info['maintenance_service_names'] abm_arakoon_service_name = 'ovs-arakoon-{0}-abm'.format(alba_backend_name) nsm_arakoon_service_name = 'ovs-arakoon-{0}-nsm_0'.format(alba_backend_name) for service_name in [abm_arakoon_service_name, nsm_arakoon_service_name] + maintenance_services: assert GeneralService.has_service(name=service_name, client=root_client) is False,\ 'Service {0} still deployed on Storage Router {1}'.format(service_name, storagerouter.name)
def setup(): """ Setup for Arakoon package, will be executed when any test in this package is being executed Make necessary changes before being able to run the tests :return: None """ for storagerouter in GeneralStorageRouter.get_masters(): root_client = SSHClient(storagerouter, username='******') if GeneralService.get_service_status(name='ovs-scheduled-tasks', client=root_client) is True: GeneralService.stop_service(name='ovs-scheduled-tasks', client=root_client) for sr in GeneralStorageRouter.get_storage_routers(): root_client = SSHClient(sr, username='******') for location in TEST_CLEANUP: root_client.run(['rm', '-rf', location])
def teardown(): """ Teardown for Arakoon package, will be executed when all started tests in this package have ended Removal actions of possible things left over after the test-run :return: None """ for storagerouter in GeneralStorageRouter.get_masters(): root_client = SSHClient(storagerouter, username='******') if GeneralService.get_service_status(name='ovs-scheduled-tasks', client=root_client) is False: GeneralService.start_service(name='ovs-scheduled-tasks', client=root_client) for location in TEST_CLEANUP: root_client.run(['rm', '-rf', location]) for key in KEY_CLEANUP: if Configuration.exists('{0}/{1}'.format(GeneralArakoon.CONFIG_ROOT, key), raw=True): Configuration.delete('{0}/{1}'.format(GeneralArakoon.CONFIG_ROOT, key))
def ovs_2703_kill_various_services_test(): """ Kill various services and see if they recover """ # @TODO 1: This test does not belong in the vPool tests, its a service test which happens to create a vPool # @TODO 2: Make test smarter to test all required services on all node types vpool = GeneralVPool.get_vpool_by_name(General.get_config().get('vpool', 'name')) if vpool is None: vpool, _ = GeneralVPool.add_vpool() errors = [] root_client = SSHClient(GeneralStorageRouter.get_local_storagerouter(), username='******') for service_name in GeneralService.get_all_service_templates(): if GeneralService.has_service(name=service_name, client=root_client) is False: continue if GeneralService.get_service_status(name=service_name, client=root_client) is False: errors.append('Service {0} not found in running state'.format(service_name)) continue pid_before = GeneralService.get_service_pid(name=service_name, client=root_client) if pid_before == -1: errors.append('Service {0} has unknown PID before being killed'.format(service_name)) continue GeneralService.kill_service(name=service_name, client=root_client) time.sleep(5) if GeneralService.get_service_status(name=service_name, client=root_client) is False: errors.append('Service {0} not found in running state after killing it'.format(service_name)) continue pid_after = GeneralService.get_service_pid(name=service_name, client=root_client) if pid_after == -1: errors.append('Service {0} has unknown PID after being killed'.format(service_name)) continue if pid_before == pid_after: errors.append('Kill command did not work on service {0}'.format(service_name)) GeneralVPool.remove_vpool(vpool) assert len(errors) == 0, "Following issues were found with the services:\n - {0}".format('\n - '.join(errors))
def check_vpool_cleanup(vpool_info, storagerouters=None): """ Check if everything related to a vPool has been cleaned up on the storagerouters provided vpool_info should be a dictionary containing: - type - guid - files - directories - name (optional) - vpool (optional) If vpool is provided: - storagerouters need to be provided, because on these Storage Routers, we check whether the vPool has been cleaned up If name is provided: - If storagerouters is NOT provided, all Storage Routers will be checked for a correct vPool removal - If storagerouters is provided, only these Storage Routers will be checked for a correct vPool removal :param vpool_info: Information about the vPool :param storagerouters: Storage Routers to check if vPool has been cleaned up :return: None """ for required_param in ['type', 'guid', 'files', 'directories']: if required_param not in vpool_info: raise ValueError('Incorrect vpool_info provided') if 'vpool' in vpool_info and 'name' in vpool_info: raise ValueError('vpool and name are mutually exclusive') if 'vpool' not in vpool_info and 'name' not in vpool_info: raise ValueError('Either vpool or vpool_name needs to be provided') vpool = vpool_info.get('vpool') vpool_name = vpool_info.get('name') vpool_guid = vpool_info['guid'] vpool_type = vpool_info['type'] files = vpool_info['files'] directories = vpool_info['directories'] supported_backend_types = GeneralBackend.get_valid_backendtypes() if vpool_type not in supported_backend_types: raise ValueError('Unsupported Backend Type provided. Please choose from: {0}'.format(', '.join(supported_backend_types))) if storagerouters is None: storagerouters = GeneralStorageRouter.get_storage_routers() if vpool_name is not None: assert GeneralVPool.get_vpool_by_name(vpool_name=vpool_name) is None, 'A vPool with name {0} still exists'.format(vpool_name) # Prepare some fields to check vpool_name = vpool.name if vpool else vpool_name vpool_services = ['ovs-dtl_{0}'.format(vpool_name), 'ovs-volumedriver_{0}'.format(vpool_name)] if vpool_type == 'alba': vpool_services.append('ovs-albaproxy_{0}'.format(vpool_name)) # Check etcd if vpool is None: assert EtcdConfiguration.exists('/ovs/vpools/{0}'.format(vpool_guid), raw=True) is False, 'vPool config still found in etcd' else: remaining_sd_ids = set([storagedriver.storagedriver_id for storagedriver in vpool.storagedrivers]) current_sd_ids = set([item for item in EtcdConfiguration.list('/ovs/vpools/{0}/hosts'.format(vpool_guid))]) assert not remaining_sd_ids.difference(current_sd_ids), 'There are more storagedrivers modelled than present in etcd' assert not current_sd_ids.difference(remaining_sd_ids), 'There are more storagedrivers in etcd than present in model' # Perform checks on all storagerouters where vpool was removed for storagerouter in storagerouters: # Check management center mgmt_center = GeneralManagementCenter.get_mgmt_center(pmachine=storagerouter.pmachine) if mgmt_center is not None: assert GeneralManagementCenter.is_host_configured(pmachine=storagerouter.pmachine) is False, 'Management Center is still configured on Storage Router {0}'.format(storagerouter.ip) # Check MDS services mds_services = GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.MD_SERVER) assert len([mds_service for mds_service in mds_services if mds_service.storagerouter_guid == storagerouter.guid]) == 0, 'There are still MDS services present for Storage Router {0}'.format(storagerouter.ip) # Check services root_client = SSHClient(storagerouter, username='******') for service in vpool_services: if ServiceManager.has_service(service, client=root_client): raise RuntimeError('Service {0} is still configured on Storage Router {1}'.format(service, storagerouter.ip)) # Check KVM vpool if storagerouter.pmachine.hvtype == 'KVM': vpool_overview = root_client.run('virsh pool-list --all').splitlines() vpool_overview.pop(1) vpool_overview.pop(0) for vpool_info in vpool_overview: kvm_vpool_name = vpool_info.split()[0].strip() if vpool_name == kvm_vpool_name: raise ValueError('vPool {0} is still defined on Storage Router {1}'.format(vpool_name, storagerouter.ip)) # Check file and directory existence if storagerouter.guid not in directories: raise ValueError('Could not find directory information for Storage Router {0}'.format(storagerouter.ip)) if storagerouter.guid not in files: raise ValueError('Could not find file information for Storage Router {0}'.format(storagerouter.ip)) for directory in directories[storagerouter.guid]: assert root_client.dir_exists(directory) is False, 'Directory {0} still exists on Storage Router {1}'.format(directory, storagerouter.ip) for file_name in files[storagerouter.guid]: assert root_client.file_exists(file_name) is False, 'File {0} still exists on Storage Router {1}'.format(file_name, storagerouter.ip) # Look for errors in storagedriver log for error_type in ['error', 'fatal']: cmd = "cat -vet /var/log/ovs/volumedriver/{0}.log | tail -1000 | grep ' {1} '; echo true > /dev/null".format(vpool_name, error_type) errors = [] for line in root_client.run(cmd).splitlines(): if "HierarchicalArakoon" in line: continue errors.append(line) if len(errors) > 0: if error_type == 'error': print 'Volumedriver log file contains errors on Storage Router {0}\n - {1}'.format(storagerouter.ip, '\n - '.join(errors)) else: raise RuntimeError('Fatal errors found in volumedriver log file on Storage Router {0}\n - {1}'.format(storagerouter.ip, '\n - '.join(errors)))
def check_vpool_cleanup(vpool_info, storagerouters=None): """ Check if everything related to a vPool has been cleaned up on the storagerouters provided vpool_info should be a dictionary containing: - type - guid - files - directories - name (optional) - vpool (optional) If vpool is provided: - storagerouters need to be provided, because on these Storage Routers, we check whether the vPool has been cleaned up If name is provided: - If storagerouters is NOT provided, all Storage Routers will be checked for a correct vPool removal - If storagerouters is provided, only these Storage Routers will be checked for a correct vPool removal :param vpool_info: Information about the vPool :param storagerouters: Storage Routers to check if vPool has been cleaned up :return: None """ for required_param in ["type", "guid", "files", "directories"]: if required_param not in vpool_info: raise ValueError("Incorrect vpool_info provided") if "vpool" in vpool_info and "name" in vpool_info: raise ValueError("vpool and name are mutually exclusive") if "vpool" not in vpool_info and "name" not in vpool_info: raise ValueError("Either vpool or vpool_name needs to be provided") vpool = vpool_info.get("vpool") vpool_name = vpool_info.get("name") vpool_guid = vpool_info["guid"] vpool_type = vpool_info["type"] files = vpool_info["files"] directories = vpool_info["directories"] supported_backend_types = GeneralBackend.get_valid_backendtypes() if vpool_type not in supported_backend_types: raise ValueError( "Unsupported Backend Type provided. Please choose from: {0}".format(", ".join(supported_backend_types)) ) if storagerouters is None: storagerouters = GeneralStorageRouter.get_storage_routers() if vpool_name is not None: assert ( GeneralVPool.get_vpool_by_name(vpool_name=vpool_name) is None ), "A vPool with name {0} still exists".format(vpool_name) # Prepare some fields to check vpool_name = vpool.name if vpool else vpool_name vpool_services = ["ovs-dtl_{0}".format(vpool_name), "ovs-volumedriver_{0}".format(vpool_name)] if vpool_type == "alba": vpool_services.append("ovs-albaproxy_{0}".format(vpool_name)) # Check configuration if vpool is None: assert ( Configuration.exists("/ovs/vpools/{0}".format(vpool_guid), raw=True) is False ), "vPool config still found in etcd" else: remaining_sd_ids = set([storagedriver.storagedriver_id for storagedriver in vpool.storagedrivers]) current_sd_ids = set([item for item in Configuration.list("/ovs/vpools/{0}/hosts".format(vpool_guid))]) assert not remaining_sd_ids.difference( current_sd_ids ), "There are more storagedrivers modelled than present in etcd" assert not current_sd_ids.difference( remaining_sd_ids ), "There are more storagedrivers in etcd than present in model" # Perform checks on all storagerouters where vpool was removed for storagerouter in storagerouters: # Check MDS services mds_services = GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.MD_SERVER) assert ( len( [ mds_service for mds_service in mds_services if mds_service.storagerouter_guid == storagerouter.guid ] ) == 0 ), "There are still MDS services present for Storage Router {0}".format(storagerouter.ip) # Check services root_client = SSHClient(storagerouter, username="******") for service in vpool_services: if ServiceManager.has_service(service, client=root_client): raise RuntimeError( "Service {0} is still configured on Storage Router {1}".format(service, storagerouter.ip) ) # Check KVM vpool if GeneralHypervisor.get_hypervisor_type() == "KVM": vpool_overview = root_client.run(["virsh", "pool-list", "--all"]).splitlines() vpool_overview.pop(1) vpool_overview.pop(0) for vpool_info in vpool_overview: kvm_vpool_name = vpool_info.split()[0].strip() if vpool_name == kvm_vpool_name: raise ValueError( "vPool {0} is still defined on Storage Router {1}".format(vpool_name, storagerouter.ip) ) # Check file and directory existence if storagerouter.guid not in directories: raise ValueError("Could not find directory information for Storage Router {0}".format(storagerouter.ip)) if storagerouter.guid not in files: raise ValueError("Could not find file information for Storage Router {0}".format(storagerouter.ip)) for directory in directories[storagerouter.guid]: assert ( root_client.dir_exists(directory) is False ), "Directory {0} still exists on Storage Router {1}".format(directory, storagerouter.ip) for file_name in files[storagerouter.guid]: assert ( root_client.file_exists(file_name) is False ), "File {0} still exists on Storage Router {1}".format(file_name, storagerouter.ip) # Look for errors in storagedriver log for error_type in ["error", "fatal"]: cmd = "cat -vet /var/log/ovs/volumedriver/{0}.log | tail -1000 | grep ' {1} '; echo true > /dev/null".format( vpool_name, error_type ) errors = [] for line in root_client.run(cmd, allow_insecure=True).splitlines(): if "HierarchicalArakoon" in line: continue errors.append(line) if len(errors) > 0: if error_type == "error": print "Volumedriver log file contains errors on Storage Router {0}\n - {1}".format( storagerouter.ip, "\n - ".join(errors) ) else: raise RuntimeError( "Fatal errors found in volumedriver log file on Storage Router {0}\n - {1}".format( storagerouter.ip, "\n - ".join(errors) ) )
def validate_alba_backend_sanity_without_claimed_disks(alba_backend): """ Validate whether the ALBA backend is configured correctly :param alba_backend: ALBA backend :return: None """ # Attribute validation assert alba_backend.available is True,\ 'ALBA backend {0} is not available'.format(alba_backend.backend.name) assert len(alba_backend.presets) >= 1,\ 'No preset found for ALBA backend {0}'.format(alba_backend.backend.name) assert len([default for default in alba_backend.presets if default['is_default'] is True]) == 1,\ 'Could not find default preset for backend {0}'.format(alba_backend.backend.name) assert alba_backend.backend.backend_type.code == 'alba',\ 'Backend type for ALBA backend is {0}'.format(alba_backend.backend.backend_type.code) assert alba_backend.backend.status == 'RUNNING',\ 'Status for ALBA backend is {0}'.format(alba_backend.backend.status) # Validate ABM and NSM services storagerouters = GeneralStorageRouter.get_storage_routers() storagerouters_with_db_role = [sr for sr in storagerouters if GeneralStorageRouter.has_roles(storagerouter=sr, roles='DB') is True and sr.node_type == 'MASTER'] assert len(alba_backend.abm_services) == len(storagerouters_with_db_role),\ 'Not enough ABM services found' assert len(alba_backend.nsm_services) == len(storagerouters_with_db_role),\ 'Not enough NSM services found' # Validate ALBA backend configuration structure alba_backend_key = '/ovs/alba/backends' assert Configuration.dir_exists(key=alba_backend_key) is True,\ 'Configuration does not contain key {0}'.format(alba_backend_key) actual_config_keys = [key for key in Configuration.list(alba_backend_key)] expected_config_keys = ['global_gui_error_interval', alba_backend.guid, 'default_nsm_hosts'] optional_config_keys = ['verification_factor'] expected_keys_amount = 0 for optional_key in optional_config_keys: if optional_key in actual_config_keys: expected_keys_amount += 1 for expected_key in expected_config_keys: if not re.match(Toolbox.regex_guid, expected_key): expected_keys_amount += 1 assert expected_key in actual_config_keys,\ 'Key {0} was not found in tree {1}'.format(expected_key, alba_backend_key) for actual_key in list(actual_config_keys): if re.match(Toolbox.regex_guid, actual_key): actual_config_keys.remove(actual_key) # Remove all alba backend keys assert len(actual_config_keys) == expected_keys_amount,\ 'Another key was added to the {0} tree'.format(alba_backend_key) this_alba_backend_key = '{0}/{1}'.format(alba_backend_key, alba_backend.guid) actual_keys = [key for key in Configuration.list(this_alba_backend_key)] expected_keys = ['maintenance'] assert actual_keys == expected_keys,\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_keys, expected_keys) maintenance_key = '{0}/maintenance'.format(this_alba_backend_key) actual_keys = [key for key in Configuration.list(maintenance_key)] expected_keys = ['nr_of_agents', 'config'] assert set(actual_keys) == set(expected_keys),\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_keys, expected_keys) # @TODO: Add validation for config values # Validate ASD node configuration structure alba_nodes = GeneralAlba.get_alba_nodes() assert len(alba_nodes) > 0,\ 'Could not find any ALBA nodes in the model' alba_node_key = '/ovs/alba/asdnodes' actual_keys = [key for key in Configuration.list(alba_node_key)] assert len(alba_nodes) == len(actual_keys),\ 'Amount of ALBA nodes in model: {0} >< amount of ALBA nodes in configuration: {1}.'.format(len(alba_nodes), len(actual_keys)) for alba_node in alba_nodes: assert alba_node.node_id in actual_keys,\ 'ALBA node with ID {0} not present in configuration'.format(alba_node.node_id) actual_asdnode_keys = [key for key in Configuration.list('{0}/{1}'.format(alba_node_key, alba_node.node_id))] expected_asdnode_keys = ['config', 'services'] assert actual_asdnode_keys == expected_asdnode_keys,\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_asdnode_keys, expected_asdnode_keys) actual_config_keys = [key for key in Configuration.list('{0}/{1}/config'.format(alba_node_key, alba_node.node_id))] expected_config_keys = ['main', 'network'] assert set(actual_config_keys) == set(expected_config_keys),\ 'Actual keys: {0} - Expected keys: {1}'.format(actual_config_keys, expected_config_keys) # @TODO: Add validation for main and network values # Validate Arakoon configuration structure arakoon_abm_key = '/ovs/arakoon/{0}/config'.format(alba_backend.abm_services[0].service.name).replace('arakoon-', '') arakoon_nsm_key = '/ovs/arakoon/{0}/config'.format(alba_backend.nsm_services[0].service.name).replace('arakoon-', '') assert Configuration.exists(key=arakoon_abm_key, raw=True) is True,\ 'Configuration key {0} does not exist'.format(arakoon_abm_key) assert Configuration.exists(key=arakoon_nsm_key, raw=True) is True,\ 'Configuration key {0} does not exist'.format(arakoon_nsm_key) # @TODO: Add validation for config values # Validate maintenance agents actual_amount_agents = len([service for node_services in [alba_node.client.list_maintenance_services() for alba_node in alba_nodes] for service in node_services]) expected_amount_agents = 1 assert actual_amount_agents == expected_amount_agents,\ 'Amount of maintenance agents is incorrect. Found {0} - Expected {1}'.format(actual_amount_agents, expected_amount_agents) # Validate arakoon services machine_ids = [sr.machine_id for sr in storagerouters_with_db_role] abm_service_name = alba_backend.abm_services[0].service.name nsm_service_name = alba_backend.nsm_services[0].service.name for storagerouter in storagerouters_with_db_role: root_client = SSHClient(endpoint=storagerouter, username='******') for service_name in [abm_service_name, nsm_service_name]: assert GeneralService.has_service(name=service_name, client=root_client) is True,\ 'Service {0} not deployed on Storage Router {1}'.format(service_name, storagerouter.name) exitcode, output = GeneralService.get_service_status(name=service_name, client=root_client) assert exitcode is True,\ 'Service {0} not running on Storage Router {1} - {2}'.format(service_name, storagerouter.name, output) out, err, _ = General.execute_command('arakoon --who-master -config {0}'.format(Configuration.get_configuration_path('/ovs/arakoon/{0}/config'.format(abm_service_name.replace('arakoon-', ''))))) assert out.strip() in machine_ids,\ 'Arakoon master is {0}, but should be 1 of "{1}"'.format(out.strip(), ', '.join(machine_ids))