def validate_alba_backend_removal(alba_backend_info):
"""
Validate whether the backend has been deleted properly
alba_backend_info should be a dictionary containing:
- guid
- name
- maintenance_service_names
:param alba_backend_info: Information about the backend
:return: None
"""
Toolbox.verify_required_params(actual_params=alba_backend_info,
required_params={'name': (str, None),
'guid': (str, Toolbox.regex_guid),
'maintenance_service_names': (list, None)},
exact_match=True)
alba_backend_guid = alba_backend_info['guid']
alba_backend_name = alba_backend_info['name']
backend = GeneralBackend.get_by_name(alba_backend_name)
assert backend is None,\
'Still found a backend in the model with name {0}'.format(alba_backend_name)
# Validate services removed from model
for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.ALBA_MGR):
assert service.name != '{0}-abm'.format(alba_backend_name),\
'An AlbaManager service has been found with name {0}'.format(alba_backend_name)
for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.NS_MGR):
assert service.name.startswith('{0}-nsm_'.format(alba_backend_name)) is False,\
'An NamespaceManager service has been found with name {0}'.format(alba_backend_name)
# Validate ALBA backend configuration structure
alba_backend_key = '/ovs/alba/backends'
actual_configuration_keys = [key for key in Configuration.list(alba_backend_key)]
assert alba_backend_guid not in actual_configuration_keys,\
'Configuration still contains an entry in {0} with guid {1}'.format(alba_backend_key, alba_backend_guid)
# Validate Arakoon configuration structure
arakoon_keys = [key for key in Configuration.list('/ovs/arakoon') if key.startswith(alba_backend_name)]
assert len(arakoon_keys) == 0,\
'Configuration still contains configurations for clusters: {0}'.format(', '.join(arakoon_keys))
# Validate services
for storagerouter in GeneralStorageRouter.get_storage_routers():
root_client = SSHClient(endpoint=storagerouter, username='******')
maintenance_services = alba_backend_info['maintenance_service_names']
abm_arakoon_service_name = 'ovs-arakoon-{0}-abm'.format(alba_backend_name)
nsm_arakoon_service_name = 'ovs-arakoon-{0}-nsm_0'.format(alba_backend_name)
for service_name in [abm_arakoon_service_name, nsm_arakoon_service_name] + maintenance_services:
assert GeneralService.has_service(name=service_name, client=root_client) is False,\
'Service {0} still deployed on Storage Router {1}'.format(service_name, storagerouter.name)
def ovs_2703_kill_various_services_test():
"""
Kill various services and see if they recover
"""
# @TODO 1: This test does not belong in the vPool tests, its a service test which happens to create a vPool
# @TODO 2: Make test smarter to test all required services on all node types
vpool = GeneralVPool.get_vpool_by_name(General.get_config().get('vpool', 'name'))
if vpool is None:
vpool, _ = GeneralVPool.add_vpool()
errors = []
root_client = SSHClient(GeneralStorageRouter.get_local_storagerouter(), username='******')
for service_name in GeneralService.get_all_service_templates():
if GeneralService.has_service(name=service_name,
client=root_client) is False:
continue
if GeneralService.get_service_status(name=service_name,
client=root_client) is False:
errors.append('Service {0} not found in running state'.format(service_name))
continue
pid_before = GeneralService.get_service_pid(name=service_name,
client=root_client)
if pid_before == -1:
errors.append('Service {0} has unknown PID before being killed'.format(service_name))
continue
GeneralService.kill_service(name=service_name,
client=root_client)
time.sleep(5)
if GeneralService.get_service_status(name=service_name,
client=root_client) is False:
errors.append('Service {0} not found in running state after killing it'.format(service_name))
continue
pid_after = GeneralService.get_service_pid(name=service_name,
client=root_client)
if pid_after == -1:
errors.append('Service {0} has unknown PID after being killed'.format(service_name))
continue
if pid_before == pid_after:
errors.append('Kill command did not work on service {0}'.format(service_name))
GeneralVPool.remove_vpool(vpool)
assert len(errors) == 0, "Following issues were found with the services:\n - {0}".format('\n - '.join(errors))
def validate_alba_backend_sanity_without_claimed_disks(alba_backend):
"""
Validate whether the ALBA backend is configured correctly
:param alba_backend: ALBA backend
:return: None
"""
# Attribute validation
assert alba_backend.available is True,\
'ALBA backend {0} is not available'.format(alba_backend.backend.name)
assert len(alba_backend.presets) >= 1,\
'No preset found for ALBA backend {0}'.format(alba_backend.backend.name)
assert len([default for default in alba_backend.presets if default['is_default'] is True]) == 1,\
'Could not find default preset for backend {0}'.format(alba_backend.backend.name)
assert alba_backend.backend.backend_type.code == 'alba',\
'Backend type for ALBA backend is {0}'.format(alba_backend.backend.backend_type.code)
assert alba_backend.backend.status == 'RUNNING',\
'Status for ALBA backend is {0}'.format(alba_backend.backend.status)
# Validate ABM and NSM services
storagerouters = GeneralStorageRouter.get_storage_routers()
storagerouters_with_db_role = [sr for sr in storagerouters if GeneralStorageRouter.has_roles(storagerouter=sr, roles='DB') is True and sr.node_type == 'MASTER']
assert len(alba_backend.abm_services) == len(storagerouters_with_db_role),\
'Not enough ABM services found'
assert len(alba_backend.nsm_services) == len(storagerouters_with_db_role),\
'Not enough NSM services found'
# Validate ALBA backend configuration structure
alba_backend_key = '/ovs/alba/backends'
assert Configuration.dir_exists(key=alba_backend_key) is True,\
'Configuration does not contain key {0}'.format(alba_backend_key)
actual_config_keys = [key for key in Configuration.list(alba_backend_key)]
expected_config_keys = ['global_gui_error_interval', alba_backend.guid, 'default_nsm_hosts']
optional_config_keys = ['verification_factor']
expected_keys_amount = 0
for optional_key in optional_config_keys:
if optional_key in actual_config_keys:
expected_keys_amount += 1
for expected_key in expected_config_keys:
if not re.match(Toolbox.regex_guid, expected_key):
expected_keys_amount += 1
assert expected_key in actual_config_keys,\
'Key {0} was not found in tree {1}'.format(expected_key, alba_backend_key)
for actual_key in list(actual_config_keys):
if re.match(Toolbox.regex_guid, actual_key):
actual_config_keys.remove(actual_key) # Remove all alba backend keys
assert len(actual_config_keys) == expected_keys_amount,\
'Another key was added to the {0} tree'.format(alba_backend_key)
this_alba_backend_key = '{0}/{1}'.format(alba_backend_key, alba_backend.guid)
actual_keys = [key for key in Configuration.list(this_alba_backend_key)]
expected_keys = ['maintenance']
assert actual_keys == expected_keys,\
'Actual keys: {0} - Expected keys: {1}'.format(actual_keys, expected_keys)
maintenance_key = '{0}/maintenance'.format(this_alba_backend_key)
actual_keys = [key for key in Configuration.list(maintenance_key)]
expected_keys = ['nr_of_agents', 'config']
assert set(actual_keys) == set(expected_keys),\
'Actual keys: {0} - Expected keys: {1}'.format(actual_keys, expected_keys)
# @TODO: Add validation for config values
# Validate ASD node configuration structure
alba_nodes = GeneralAlba.get_alba_nodes()
assert len(alba_nodes) > 0,\
'Could not find any ALBA nodes in the model'
alba_node_key = '/ovs/alba/asdnodes'
actual_keys = [key for key in Configuration.list(alba_node_key)]
assert len(alba_nodes) == len(actual_keys),\
'Amount of ALBA nodes in model: {0} >< amount of ALBA nodes in configuration: {1}.'.format(len(alba_nodes),
len(actual_keys))
for alba_node in alba_nodes:
assert alba_node.node_id in actual_keys,\
'ALBA node with ID {0} not present in configuration'.format(alba_node.node_id)
actual_asdnode_keys = [key for key in Configuration.list('{0}/{1}'.format(alba_node_key, alba_node.node_id))]
expected_asdnode_keys = ['config', 'services']
assert actual_asdnode_keys == expected_asdnode_keys,\
'Actual keys: {0} - Expected keys: {1}'.format(actual_asdnode_keys, expected_asdnode_keys)
actual_config_keys = [key for key in Configuration.list('{0}/{1}/config'.format(alba_node_key, alba_node.node_id))]
expected_config_keys = ['main', 'network']
assert set(actual_config_keys) == set(expected_config_keys),\
'Actual keys: {0} - Expected keys: {1}'.format(actual_config_keys, expected_config_keys)
# @TODO: Add validation for main and network values
# Validate Arakoon configuration structure
arakoon_abm_key = '/ovs/arakoon/{0}/config'.format(alba_backend.abm_services[0].service.name).replace('arakoon-', '')
arakoon_nsm_key = '/ovs/arakoon/{0}/config'.format(alba_backend.nsm_services[0].service.name).replace('arakoon-', '')
assert Configuration.exists(key=arakoon_abm_key, raw=True) is True,\
'Configuration key {0} does not exist'.format(arakoon_abm_key)
assert Configuration.exists(key=arakoon_nsm_key, raw=True) is True,\
'Configuration key {0} does not exist'.format(arakoon_nsm_key)
# @TODO: Add validation for config values
# Validate maintenance agents
actual_amount_agents = len([service for node_services in [alba_node.client.list_maintenance_services() for alba_node in alba_nodes] for service in node_services])
expected_amount_agents = 1
assert actual_amount_agents == expected_amount_agents,\
'Amount of maintenance agents is incorrect. Found {0} - Expected {1}'.format(actual_amount_agents,
expected_amount_agents)
# Validate arakoon services
machine_ids = [sr.machine_id for sr in storagerouters_with_db_role]
abm_service_name = alba_backend.abm_services[0].service.name
nsm_service_name = alba_backend.nsm_services[0].service.name
for storagerouter in storagerouters_with_db_role:
root_client = SSHClient(endpoint=storagerouter, username='******')
for service_name in [abm_service_name, nsm_service_name]:
assert GeneralService.has_service(name=service_name, client=root_client) is True,\
'Service {0} not deployed on Storage Router {1}'.format(service_name, storagerouter.name)
exitcode, output = GeneralService.get_service_status(name=service_name, client=root_client)
assert exitcode is True,\
'Service {0} not running on Storage Router {1} - {2}'.format(service_name, storagerouter.name,
output)
out, err, _ = General.execute_command('arakoon --who-master -config {0}'.format(Configuration.get_configuration_path('/ovs/arakoon/{0}/config'.format(abm_service_name.replace('arakoon-', '')))))
assert out.strip() in machine_ids,\
'Arakoon master is {0}, but should be 1 of "{1}"'.format(out.strip(), ', '.join(machine_ids))
