def be_0001_add_and_remove_backend_test():
"""
Create an ALBA backend and verify its status
Validate services, etcd, arakoon without claiming disks
Claim some disks and validate whether backend can be used for storing objects in namespaces
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is not None:
raise ValueError('A backend has already been deployed, cannot execute test')
alba_backend = GeneralAlba.add_alba_backend(TestALBA.backend_name)
GeneralAlba.validate_alba_backend_sanity_without_claimed_disks(alba_backend=alba_backend)
GeneralAlba.claim_asds(alba_backend, 3, 'SATA')
GeneralAlba.validate_alba_backend_sanity_with_claimed_disks(alba_backend=alba_backend)
guid = alba_backend.guid
name = TestALBA.backend_name
service_names = GeneralAlba.get_maintenance_services_for_alba_backend(alba_backend=alba_backend)
GeneralAlba.unclaim_disks(alba_backend)
GeneralAlba.remove_alba_backend(alba_backend)
GeneralAlba.validate_alba_backend_removal(alba_backend_info={'name': name,
'guid': guid,
'maintenance_service_names': service_names})
def ovs_3188_verify_namespace_test():
"""
Verify namespaces
"""
nr_of_disks_to_create = 5
namespace_prefix = 'ovs_3188-'
compression = 'none'
encryption = 'none'
preset_name = 'be_preset_02'
policies = [[1, 1, 1, 2]]
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
GeneralAlba.add_preset(backend.alba_backend, preset_name, policies, compression, encryption)
for x in range(nr_of_disks_to_create):
namespace_name = namespace_prefix + str(x)
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'create-namespace', [namespace_name, preset_name], False)
GeneralAlba.upload_file(backend.alba_backend, namespace_name, 1024 * 1024 * 1)
AlbaScheduledTaskController.verify_namespaces()
GeneralAlba.remove_alba_namespaces(backend.alba_backend)
GeneralAlba.remove_preset(backend.alba_backend, preset_name)
def prepare_alba_backend(name=None):
"""
Create an ALBA backend and claim disks
:param name: Name for the backend
:return: None
"""
# @TODO: Fix this, because backend_type should not be configurable if you always create an ALBA backend
# @TODO 2: Get rid of these asserts, any test (or testsuite) should verify the required params first before starting execution
autotest_config = General.get_config()
if name is None:
name = autotest_config.get('backend', 'name')
nr_of_disks_to_claim = autotest_config.getint('backend', 'nr_of_disks_to_claim')
type_of_disks_to_claim = autotest_config.get('backend', 'type_of_disks_to_claim')
assert name,\
"Please fill out a valid backend name in autotest.cfg file"
storage_routers = GeneralStorageRouter.get_storage_routers()
for sr in storage_routers:
if GeneralStorageRouter.has_roles(storagerouter=sr, roles='DB') is False:
GeneralDisk.add_db_role(sr)
if GeneralStorageRouter.has_roles(storagerouter=sr, roles=['SCRUB', 'WRITE']) is False:
GeneralDisk.add_write_scrub_roles(sr)
backend = GeneralBackend.get_by_name(name)
if not backend:
alba_backend = GeneralAlba.add_alba_backend(name)
else:
alba_backend = backend.alba_backend
GeneralAlba.claim_asds(alba_backend, nr_of_disks_to_claim, type_of_disks_to_claim)
if GeneralAlba.has_preset(alba_backend=alba_backend,
preset_name=GeneralAlba.ONE_DISK_PRESET) is False:
GeneralAlba.add_preset(alba_backend=alba_backend,
name=GeneralAlba.ONE_DISK_PRESET,
policies=[[1, 1, 1, 2]])
def setup():
"""
Setup for Arakoon package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
assert backend_name, 'Please fill out a backend name in the autotest.cfg file'
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is True:
GeneralService.stop_service(name='ovs-scheduled-tasks',
client=root_client)
storagerouters = GeneralStorageRouter.get_storage_routers()
for sr in storagerouters:
root_client = SSHClient(sr, username='******')
GeneralDisk.add_db_role(sr)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
GeneralAlba.add_alba_backend(backend_name)
GeneralArakoon.voldrv_arakoon_checkup()
def teardown():
"""
Teardown for Arakoon package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is False:
GeneralService.start_service(name='ovs-scheduled-tasks',
client=root_client)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
for key in KEY_CLEANUP:
if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True):
EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))
def verify_policies_for_preset(preset_name, policies, compression, encryption):
"""
Verify the policies of a preset
:param preset_name: Name of preset
:param policies: Policies to verify
:param compression: Compression for preset
:param encryption: Encryption for preset
:return: True is valid
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
valid = False
for preset in backend.alba_backend.presets:
if preset['name'] == preset_name:
assert preset['compression'] == compression,\
"Alba compression {0} does not match configured {1} type".format(preset['compression'],
compression)
assert preset['fragment_encryption'][0] == encryption, \
"Alba encryption {0} does not match configured {1} type".format(preset['fragment_encryption'],
encryption)
for policy in policies:
valid = False
for alba_policy in preset['policies']:
print 'Matching: {0} with {1}'.format(tuple(policy), alba_policy)
if tuple(policy) == alba_policy:
valid = True
continue
return valid
def ovs_2263_verify_alba_namespace_cleanup_test():
"""
Verify ALBA namespace cleanup
Create an amount of namespaces in ALBA
Create a vPool and create some volumes
Verify the amount of namespaces before and after vPool creation
Remove the vPool and the manually created namespaces
Verify the amount of namespaces before and after vPool deletion
"""
# Create some namespaces in alba
no_namespaces = 3
backend_name = General.get_config().get('backend', 'name')
backend = GeneralBackend.get_by_name(name=backend_name)
namespace_name = 'autotest-ns_'
namespace_name_regex = re.compile('^autotest-ns_\d$')
for nmspc_index in range(no_namespaces):
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'create-namespace', ['{0}{1}'.format(namespace_name, nmspc_index), 'default'], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces, "Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
# Create a vPool and create volumes on it
vpool, _ = GeneralVPool.add_vpool()
root_client = SSHClient(GeneralStorageRouter.get_local_storagerouter(), username='******')
if vpool.storagedrivers[0].storagerouter.pmachine.hvtype == 'VMWARE':
GeneralVPool.mount_vpool(vpool=vpool,
root_client=root_client)
vdisks = []
for disk_index in range(no_namespaces):
vdisks.append(GeneralVDisk.create_volume(size=10,
vpool=vpool,
root_client=root_client))
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend)
assert len(result) == 2 * no_namespaces + 1, "Expected {0} namespaces present on the {1} backend, found {2}".format(2 * no_namespaces + 1, backend_name, len(result))
# Remove files and vPool
for vdisk in vdisks:
GeneralVDisk.delete_volume(vdisk=vdisk,
vpool=vpool,
root_client=root_client)
if vpool.storagedrivers[0].storagerouter.pmachine.hvtype == 'VMWARE':
GeneralVPool.unmount_vpool(vpool=vpool,
root_client=root_client)
GeneralVPool.remove_vpool(vpool)
# Verify amount of namespaces
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces, "Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
for namespace in result:
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'delete-namespace', [namespace['name']], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == 0, "Expected no namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
def teardown():
"""
Teardown for vPool package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
be = GeneralBackend.get_by_name(autotest_config.get('backend', 'name'))
if be:
GeneralAlba.unclaim_disks_and_remove_alba_backend(alba_backend=be.alba_backend)
def add_alba_backend(name, wait=True):
"""
Put an ALBA backend in the model
:param name: Name of the backend
:param wait: Wait for backend to enter RUNNING state
:return: Newly created ALBA backend
"""
backend = GeneralBackend.get_by_name(name)
if backend is None:
backend = GeneralBackend.add_backend(name, 'alba')
alba_backend = AlbaBackend(GeneralAlba.api.add('alba/backends', {'backend_guid': backend.guid})['guid'])
if wait is True:
GeneralAlba.wait_for_alba_backend_status(alba_backend)
out, err, _ = General.execute_command('etcdctl ls /ovs/alba/asdnodes')
if err == '' and len(out):
AlbaNodeController.model_local_albanode()
return GeneralBackend.get_by_name(name).alba_backend
def is_preset_present(name):
"""
Verify if a preset is present
:param name: Name of the preset
:return: True if present
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
for preset in backend.alba_backend.presets:
if name == preset['name']:
return True
return False
def be_0007_add_update_remove_preset_test():
"""
Add, update and remove a preset
Validation for OVS-3187 - edit policy of preset
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
alba_backend = GeneralAlba.add_alba_backend(TestALBA.backend_name)
else:
alba_backend = backend.alba_backend
GeneralAlba.claim_asds(alba_backend=alba_backend, nr_of_asds=3, disk_type='SATA')
timeout = 300
preset_name = 'be_preset_0007'
namespace_name = 'be_0007_ns'
compression = 'none'
encryption = 'aes-cbc-256'
org_policy = [[1, 1, 1, 2]]
new_policy = [[2, 2, 3, 3]]
TestALBA.add_validate_remove_preset(preset_name, compression, encryption, org_policy, remove_when_finished=False)
result = GeneralAlba.list_alba_namespaces(alba_backend=alba_backend,
name=namespace_name)
for namespace in result:
GeneralAlba.execute_alba_cli_action(alba_backend, 'delete-namespace', [namespace['name']], False)
GeneralAlba.execute_alba_cli_action(alba_backend, 'create-namespace', [namespace_name, preset_name], False)
GeneralAlba.upload_file(alba_backend=alba_backend, namespace_name=namespace_name, file_size=1024 * 1024)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert len(result) == 1, "Only one policy should be present, found: {0}".format(result)
# update and verify policies for preset
GeneralAlba.update_preset(alba_backend, preset_name, new_policy)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert len(result) == 1, "Expected 1 policy, but got: {0}".format(result)
object_has_new_policy = False
for _ in xrange(timeout):
if GeneralAlba.is_bucket_count_valid_with_policy(result, new_policy):
object_has_new_policy = True
break
time.sleep(1)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert object_has_new_policy is True, "Object was not rewritten within {0} seconds: {1}".format(timeout, result)
# cleanup
GeneralAlba.execute_alba_cli_action(alba_backend, 'delete-namespace', [namespace_name], False)
GeneralAlba.remove_preset(alba_backend, preset_name)
def ovs_3977_maintenance_agent_test():
"""
Test maintenance agent processes
"""
def _get_agent_distribution(agent_name):
result = {}
total = 0
for ip in alba_node_ips:
count = General.execute_command_on_node(ip, 'ls /etc/init/alba-maintenance_{0}-* | wc -l'.format(agent_name))
if count:
count = int(count)
else:
count = 0
total += count
result[ip] = count
result['total'] = total
print 'Maintenance agent distribution: {0}'.format(result)
for ip in alba_node_ips:
assert (result[ip] == total / len(alba_node_ips) or result[ip] == (total / len(alba_node_ips)) + 1),\
"Agents not equally distributed!"
return result
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
name = backend.alba_backend.name
alba_node_ips = [node.ip for node in GeneralAlba.get_alba_nodes()]
etcd_key = '/ovs/alba/backends/{0}/maintenance/nr_of_agents'.format(backend.alba_backend.guid)
nr_of_agents = EtcdConfiguration.get(etcd_key)
print '1. - nr of agents: {0}'.format(nr_of_agents)
actual_nr_of_agents = _get_agent_distribution(name)['total']
assert nr_of_agents == actual_nr_of_agents, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
# set nr to zero
EtcdConfiguration.set(etcd_key, 0)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 0, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '2. - nr of agents: {0}'.format(nr_of_agents)
# set nr to 10
EtcdConfiguration.set(etcd_key, 10)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 10, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '3. - nr of agents: {0}'.format(nr_of_agents)
def be_0005_add_remove_preset_no_compression_encryption_test():
"""
Add and remove a preset without compression and with encryption
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
name = 'be_preset_05'
compression = 'none'
encryption = 'aes-cbc-256'
policies = [[1, 1, 1, 2]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def validate_alba_backend_removal(alba_backend_info):
"""
Validate whether the backend has been deleted properly
alba_backend_info should be a dictionary containing:
- guid
- name
- maintenance_service_names
:param alba_backend_info: Information about the backend
:return: None
"""
Toolbox.verify_required_params(actual_params=alba_backend_info,
required_params={'name': (str, None),
'guid': (str, Toolbox.regex_guid),
'maintenance_service_names': (list, None)},
exact_match=True)
alba_backend_guid = alba_backend_info['guid']
alba_backend_name = alba_backend_info['name']
backend = GeneralBackend.get_by_name(alba_backend_name)
assert backend is None,\
'Still found a backend in the model with name {0}'.format(alba_backend_name)
# Validate services removed from model
for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.ALBA_MGR):
assert service.name != '{0}-abm'.format(alba_backend_name),\
'An AlbaManager service has been found with name {0}'.format(alba_backend_name)
for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.NS_MGR):
assert service.name.startswith('{0}-nsm_'.format(alba_backend_name)) is False,\
'An NamespaceManager service has been found with name {0}'.format(alba_backend_name)
# Validate ALBA backend configuration structure
alba_backend_key = '/ovs/alba/backends'
actual_configuration_keys = [key for key in Configuration.list(alba_backend_key)]
assert alba_backend_guid not in actual_configuration_keys,\
'Configuration still contains an entry in {0} with guid {1}'.format(alba_backend_key, alba_backend_guid)
# Validate Arakoon configuration structure
arakoon_keys = [key for key in Configuration.list('/ovs/arakoon') if key.startswith(alba_backend_name)]
assert len(arakoon_keys) == 0,\
'Configuration still contains configurations for clusters: {0}'.format(', '.join(arakoon_keys))
# Validate services
for storagerouter in GeneralStorageRouter.get_storage_routers():
root_client = SSHClient(endpoint=storagerouter, username='******')
maintenance_services = alba_backend_info['maintenance_service_names']
abm_arakoon_service_name = 'ovs-arakoon-{0}-abm'.format(alba_backend_name)
nsm_arakoon_service_name = 'ovs-arakoon-{0}-nsm_0'.format(alba_backend_name)
for service_name in [abm_arakoon_service_name, nsm_arakoon_service_name] + maintenance_services:
assert GeneralService.has_service(name=service_name, client=root_client) is False,\
'Service {0} still deployed on Storage Router {1}'.format(service_name, storagerouter.name)
def be_0004_validate_preset_with_replication_copies_test():
"""
Validate a preset
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
compression = 'none'
encryption = 'none'
name_prefix = 'be_preset_04'
for nr in xrange(6):
name = name_prefix + str(nr)
policies = [[1, nr, 1, 1 + nr]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def be_0006_add_remove_preset_compression_encryption_test():
"""
Add and remove a preset with compression and encryption
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
name = 'be_preset_06a'
compression = 'bz2'
encryption = 'aes-cbc-256'
policies = [[1, 1, 1, 2]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
name = 'be_preset_06b'
compression = 'snappy'
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def ovs_3490_add_remove_preset_test():
"""
Adds and removes a preset with encryption to an existing alba backend
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
name = 'ovs-3490'
policies = [[1, 1, 1, 2]]
compression = 'none'
encryption = 'aes-cbc-256'
status, message = GeneralAlba.add_preset(backend.alba_backend, name, policies, compression, encryption)
assert status, "Add preset failed with: {0}".format(message)
TestALBA.verify_policies_for_preset(name, policies, compression, encryption)
status, message = GeneralAlba.remove_preset(backend.alba_backend, name)
assert status, "Remove preset failed with: {0}".format(message)
assert not TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
def add_validate_remove_preset(name, compression, encryption, policies, remove_when_finished=True):
"""
Add a preset, validate the preset and remove it
:param name: Name of the preset
:param compression: Compression used by the preset
:param encryption: Encryption used by the preset
:param policies: Policies used by the preset
:param remove_when_finished: Remove after validation
:return: None
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
status, message = GeneralAlba.add_preset(backend.alba_backend, name, policies, compression, encryption)
assert status, "Add preset failed with: {0}".format(message)
assert TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
TestALBA.verify_policies_for_preset(name, policies, compression, encryption)
if remove_when_finished:
status, message = GeneralAlba.remove_preset(backend.alba_backend, name, )
assert status, "Remove preset failed with: {0}".format(message)
assert not TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
def teardown():
"""
Teardown for VirtualMachine package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
vpool_name = General.get_config().get('vpool', 'name')
vpool = GeneralVPool.get_vpool_by_name(vpool_name)
assert vpool is not None, "No vpool found where one was expected"
GeneralVMachine.logger.info("Cleaning vpool")
GeneralVPool.remove_vpool(vpool)
autotest_config = General.get_config()
be = GeneralBackend.get_by_name(autotest_config.get('backend', 'name'))
if be:
GeneralAlba.unclaim_disks_and_remove_alba_backend(alba_backend=be.alba_backend)
GeneralVMachine.logger.info("Cleaning management center")
for mgmt_center in GeneralManagementCenter.get_mgmt_centers():
GeneralManagementCenter.remove_mgmt_center(mgmt_center)
def add_alba_backend(name, scaling='LOCAL', wait=True):
"""
Put an ALBA backend in the model
:param name: Name of the backend
:param scaling: Alba backend can be LOCAL or GLOBAL
:param wait: Wait for backend to enter RUNNING state
:return: Newly created ALBA backend
"""
alba_backend = GeneralAlba.get_by_name(name)
if alba_backend is None:
backend = GeneralBackend.add_backend(name, 'alba')
alba_backend = AlbaBackend(GeneralAlba.api.add('alba/backends', {'backend_guid': backend.guid,
'scaling': scaling})['guid'])
if wait is True:
GeneralAlba.wait_for_alba_backend_status(alba_backend)
alba_nodes = [alba_node for alba_node in Configuration.list('/ovs/alba/asdnodes')]
if len(alba_nodes):
AlbaNodeController.model_albanodes()
return GeneralAlba.get_by_name(name)
def ovs_3769_validation_test():
"""
Create an albanode with an asd statistics part set to empty dictionary
Assert code does not raise
"""
an = AlbaNode()
an.password = '******'
an.node_id = 'ovs3769an'
an.port = 1234
an.ip = '127.0.0.1'
an.username = '******'
an.save()
bet = GeneralBackend.get_backendtype_by_code('alba')
be = Backend()
be.backend_type = bet
be.name = 'ovs3769be'
be.save()
abe = AlbaBackend()
abe.backend = be
abe.save()
ad = AlbaDisk()
ad.name = 'ovs3769ad'
ad.alba_node = an
ad.save()
asd = AlbaASD()
asd.alba_backend = abe
asd.asd_id = 'ovs3769asd'
asd.alba_disk = ad
asd.save()
try:
abe.statistics
except KeyError, ex:
logger.error('Regression OVS-3769 - asd statistics raises a KeyError: {0}'.format(str(ex)))
def check_vpool_cleanup(vpool_info, storagerouters=None):
"""
Check if everything related to a vPool has been cleaned up on the storagerouters provided
vpool_info should be a dictionary containing:
- type
- guid
- files
- directories
- name (optional)
- vpool (optional)
If vpool is provided:
- storagerouters need to be provided, because on these Storage Routers, we check whether the vPool has been cleaned up
If name is provided:
- If storagerouters is NOT provided, all Storage Routers will be checked for a correct vPool removal
- If storagerouters is provided, only these Storage Routers will be checked for a correct vPool removal
:param vpool_info: Information about the vPool
:param storagerouters: Storage Routers to check if vPool has been cleaned up
:return: None
"""
for required_param in ['type', 'guid', 'files', 'directories']:
if required_param not in vpool_info:
raise ValueError('Incorrect vpool_info provided')
if 'vpool' in vpool_info and 'name' in vpool_info:
raise ValueError('vpool and name are mutually exclusive')
if 'vpool' not in vpool_info and 'name' not in vpool_info:
raise ValueError('Either vpool or vpool_name needs to be provided')
vpool = vpool_info.get('vpool')
vpool_name = vpool_info.get('name')
vpool_guid = vpool_info['guid']
vpool_type = vpool_info['type']
files = vpool_info['files']
directories = vpool_info['directories']
supported_backend_types = GeneralBackend.get_valid_backendtypes()
if vpool_type not in supported_backend_types:
raise ValueError('Unsupported Backend Type provided. Please choose from: {0}'.format(', '.join(supported_backend_types)))
if storagerouters is None:
storagerouters = GeneralStorageRouter.get_storage_routers()
if vpool_name is not None:
assert GeneralVPool.get_vpool_by_name(vpool_name=vpool_name) is None, 'A vPool with name {0} still exists'.format(vpool_name)
# Prepare some fields to check
vpool_name = vpool.name if vpool else vpool_name
vpool_services = ['ovs-dtl_{0}'.format(vpool_name),
'ovs-volumedriver_{0}'.format(vpool_name)]
if vpool_type == 'alba':
vpool_services.append('ovs-albaproxy_{0}'.format(vpool_name))
# Check etcd
if vpool is None:
assert EtcdConfiguration.exists('/ovs/vpools/{0}'.format(vpool_guid), raw=True) is False, 'vPool config still found in etcd'
else:
remaining_sd_ids = set([storagedriver.storagedriver_id for storagedriver in vpool.storagedrivers])
current_sd_ids = set([item for item in EtcdConfiguration.list('/ovs/vpools/{0}/hosts'.format(vpool_guid))])
assert not remaining_sd_ids.difference(current_sd_ids), 'There are more storagedrivers modelled than present in etcd'
assert not current_sd_ids.difference(remaining_sd_ids), 'There are more storagedrivers in etcd than present in model'
# Perform checks on all storagerouters where vpool was removed
for storagerouter in storagerouters:
# Check management center
mgmt_center = GeneralManagementCenter.get_mgmt_center(pmachine=storagerouter.pmachine)
if mgmt_center is not None:
assert GeneralManagementCenter.is_host_configured(pmachine=storagerouter.pmachine) is False, 'Management Center is still configured on Storage Router {0}'.format(storagerouter.ip)
# Check MDS services
mds_services = GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.MD_SERVER)
assert len([mds_service for mds_service in mds_services if mds_service.storagerouter_guid == storagerouter.guid]) == 0, 'There are still MDS services present for Storage Router {0}'.format(storagerouter.ip)
# Check services
root_client = SSHClient(storagerouter, username='******')
for service in vpool_services:
if ServiceManager.has_service(service, client=root_client):
raise RuntimeError('Service {0} is still configured on Storage Router {1}'.format(service, storagerouter.ip))
# Check KVM vpool
if storagerouter.pmachine.hvtype == 'KVM':
vpool_overview = root_client.run('virsh pool-list --all').splitlines()
vpool_overview.pop(1)
vpool_overview.pop(0)
for vpool_info in vpool_overview:
kvm_vpool_name = vpool_info.split()[0].strip()
if vpool_name == kvm_vpool_name:
raise ValueError('vPool {0} is still defined on Storage Router {1}'.format(vpool_name, storagerouter.ip))
# Check file and directory existence
if storagerouter.guid not in directories:
raise ValueError('Could not find directory information for Storage Router {0}'.format(storagerouter.ip))
if storagerouter.guid not in files:
raise ValueError('Could not find file information for Storage Router {0}'.format(storagerouter.ip))
for directory in directories[storagerouter.guid]:
assert root_client.dir_exists(directory) is False, 'Directory {0} still exists on Storage Router {1}'.format(directory, storagerouter.ip)
for file_name in files[storagerouter.guid]:
assert root_client.file_exists(file_name) is False, 'File {0} still exists on Storage Router {1}'.format(file_name, storagerouter.ip)
# Look for errors in storagedriver log
for error_type in ['error', 'fatal']:
cmd = "cat -vet /var/log/ovs/volumedriver/{0}.log | tail -1000 | grep ' {1} '; echo true > /dev/null".format(vpool_name, error_type)
errors = []
for line in root_client.run(cmd).splitlines():
if "HierarchicalArakoon" in line:
continue
errors.append(line)
if len(errors) > 0:
if error_type == 'error':
print 'Volumedriver log file contains errors on Storage Router {0}\n - {1}'.format(storagerouter.ip, '\n - '.join(errors))
else:
raise RuntimeError('Fatal errors found in volumedriver log file on Storage Router {0}\n - {1}'.format(storagerouter.ip, '\n - '.join(errors)))
def check_vpool_cleanup(vpool_info, storagerouters=None):
"""
Check if everything related to a vPool has been cleaned up on the storagerouters provided
vpool_info should be a dictionary containing:
- type
- guid
- files
- directories
- name (optional)
- vpool (optional)
If vpool is provided:
- storagerouters need to be provided, because on these Storage Routers, we check whether the vPool has been cleaned up
If name is provided:
- If storagerouters is NOT provided, all Storage Routers will be checked for a correct vPool removal
- If storagerouters is provided, only these Storage Routers will be checked for a correct vPool removal
:param vpool_info: Information about the vPool
:param storagerouters: Storage Routers to check if vPool has been cleaned up
:return: None
"""
for required_param in ["type", "guid", "files", "directories"]:
if required_param not in vpool_info:
raise ValueError("Incorrect vpool_info provided")
if "vpool" in vpool_info and "name" in vpool_info:
raise ValueError("vpool and name are mutually exclusive")
if "vpool" not in vpool_info and "name" not in vpool_info:
raise ValueError("Either vpool or vpool_name needs to be provided")
vpool = vpool_info.get("vpool")
vpool_name = vpool_info.get("name")
vpool_guid = vpool_info["guid"]
vpool_type = vpool_info["type"]
files = vpool_info["files"]
directories = vpool_info["directories"]
supported_backend_types = GeneralBackend.get_valid_backendtypes()
if vpool_type not in supported_backend_types:
raise ValueError(
"Unsupported Backend Type provided. Please choose from: {0}".format(", ".join(supported_backend_types))
)
if storagerouters is None:
storagerouters = GeneralStorageRouter.get_storage_routers()
if vpool_name is not None:
assert (
GeneralVPool.get_vpool_by_name(vpool_name=vpool_name) is None
), "A vPool with name {0} still exists".format(vpool_name)
# Prepare some fields to check
vpool_name = vpool.name if vpool else vpool_name
vpool_services = ["ovs-dtl_{0}".format(vpool_name), "ovs-volumedriver_{0}".format(vpool_name)]
if vpool_type == "alba":
vpool_services.append("ovs-albaproxy_{0}".format(vpool_name))
# Check configuration
if vpool is None:
assert (
Configuration.exists("/ovs/vpools/{0}".format(vpool_guid), raw=True) is False
), "vPool config still found in etcd"
else:
remaining_sd_ids = set([storagedriver.storagedriver_id for storagedriver in vpool.storagedrivers])
current_sd_ids = set([item for item in Configuration.list("/ovs/vpools/{0}/hosts".format(vpool_guid))])
assert not remaining_sd_ids.difference(
current_sd_ids
), "There are more storagedrivers modelled than present in etcd"
assert not current_sd_ids.difference(
remaining_sd_ids
), "There are more storagedrivers in etcd than present in model"
# Perform checks on all storagerouters where vpool was removed
for storagerouter in storagerouters:
# Check MDS services
mds_services = GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.MD_SERVER)
assert (
len(
[
mds_service
for mds_service in mds_services
if mds_service.storagerouter_guid == storagerouter.guid
]
)
== 0
), "There are still MDS services present for Storage Router {0}".format(storagerouter.ip)
# Check services
root_client = SSHClient(storagerouter, username="******")
for service in vpool_services:
if ServiceManager.has_service(service, client=root_client):
raise RuntimeError(
"Service {0} is still configured on Storage Router {1}".format(service, storagerouter.ip)
)
# Check KVM vpool
if GeneralHypervisor.get_hypervisor_type() == "KVM":
vpool_overview = root_client.run(["virsh", "pool-list", "--all"]).splitlines()
vpool_overview.pop(1)
vpool_overview.pop(0)
for vpool_info in vpool_overview:
kvm_vpool_name = vpool_info.split()[0].strip()
if vpool_name == kvm_vpool_name:
raise ValueError(
"vPool {0} is still defined on Storage Router {1}".format(vpool_name, storagerouter.ip)
)
# Check file and directory existence
if storagerouter.guid not in directories:
raise ValueError("Could not find directory information for Storage Router {0}".format(storagerouter.ip))
if storagerouter.guid not in files:
raise ValueError("Could not find file information for Storage Router {0}".format(storagerouter.ip))
for directory in directories[storagerouter.guid]:
assert (
root_client.dir_exists(directory) is False
), "Directory {0} still exists on Storage Router {1}".format(directory, storagerouter.ip)
for file_name in files[storagerouter.guid]:
assert (
root_client.file_exists(file_name) is False
), "File {0} still exists on Storage Router {1}".format(file_name, storagerouter.ip)
# Look for errors in storagedriver log
for error_type in ["error", "fatal"]:
cmd = "cat -vet /var/log/ovs/volumedriver/{0}.log | tail -1000 | grep ' {1} '; echo true > /dev/null".format(
vpool_name, error_type
)
errors = []
for line in root_client.run(cmd, allow_insecure=True).splitlines():
if "HierarchicalArakoon" in line:
continue
errors.append(line)
if len(errors) > 0:
if error_type == "error":
print "Volumedriver log file contains errors on Storage Router {0}\n - {1}".format(
storagerouter.ip, "\n - ".join(errors)
)
else:
raise RuntimeError(
"Fatal errors found in volumedriver log file on Storage Router {0}\n - {1}".format(
storagerouter.ip, "\n - ".join(errors)
)
)