def prepare_alba_backend(name=None):
"""
Create an ALBA backend and claim disks
:param name: Name for the backend
:return: None
"""
# @TODO: Fix this, because backend_type should not be configurable if you always create an ALBA backend
# @TODO 2: Get rid of these asserts, any test (or testsuite) should verify the required params first before starting execution
autotest_config = General.get_config()
if name is None:
name = autotest_config.get('backend', 'name')
nr_of_disks_to_claim = autotest_config.getint('backend', 'nr_of_disks_to_claim')
type_of_disks_to_claim = autotest_config.get('backend', 'type_of_disks_to_claim')
assert name,\
"Please fill out a valid backend name in autotest.cfg file"
storage_routers = GeneralStorageRouter.get_storage_routers()
for sr in storage_routers:
if GeneralStorageRouter.has_roles(storagerouter=sr, roles='DB') is False:
GeneralDisk.add_db_role(sr)
if GeneralStorageRouter.has_roles(storagerouter=sr, roles=['SCRUB', 'WRITE']) is False:
GeneralDisk.add_write_scrub_roles(sr)
backend = GeneralBackend.get_by_name(name)
if not backend:
alba_backend = GeneralAlba.add_alba_backend(name)
else:
alba_backend = backend.alba_backend
GeneralAlba.claim_asds(alba_backend, nr_of_disks_to_claim, type_of_disks_to_claim)
if GeneralAlba.has_preset(alba_backend=alba_backend,
preset_name=GeneralAlba.ONE_DISK_PRESET) is False:
GeneralAlba.add_preset(alba_backend=alba_backend,
name=GeneralAlba.ONE_DISK_PRESET,
policies=[[1, 1, 1, 2]])
def be_0001_add_and_remove_backend_test():
"""
Create an ALBA backend and verify its status
Validate services, etcd, arakoon without claiming disks
Claim some disks and validate whether backend can be used for storing objects in namespaces
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is not None:
raise ValueError('A backend has already been deployed, cannot execute test')
alba_backend = GeneralAlba.add_alba_backend(TestALBA.backend_name)
GeneralAlba.validate_alba_backend_sanity_without_claimed_disks(alba_backend=alba_backend)
GeneralAlba.claim_asds(alba_backend, 3, 'SATA')
GeneralAlba.validate_alba_backend_sanity_with_claimed_disks(alba_backend=alba_backend)
guid = alba_backend.guid
name = TestALBA.backend_name
service_names = GeneralAlba.get_maintenance_services_for_alba_backend(alba_backend=alba_backend)
GeneralAlba.unclaim_disks(alba_backend)
GeneralAlba.remove_alba_backend(alba_backend)
GeneralAlba.validate_alba_backend_removal(alba_backend_info={'name': name,
'guid': guid,
'maintenance_service_names': service_names})
def setup():
"""
Setup for Arakoon package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
assert backend_name, 'Please fill out a backend name in the autotest.cfg file'
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is True:
GeneralService.stop_service(name='ovs-scheduled-tasks',
client=root_client)
storagerouters = GeneralStorageRouter.get_storage_routers()
for sr in storagerouters:
root_client = SSHClient(sr, username='******')
GeneralDisk.add_db_role(sr)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
GeneralAlba.add_alba_backend(backend_name)
GeneralArakoon.voldrv_arakoon_checkup()
def teardown():
"""
Teardown for Arakoon package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is False:
GeneralService.start_service(name='ovs-scheduled-tasks',
client=root_client)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
for key in KEY_CLEANUP:
if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True):
EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))
def ovs_3188_verify_namespace_test():
"""
Verify namespaces
"""
nr_of_disks_to_create = 5
namespace_prefix = 'ovs_3188-'
compression = 'none'
encryption = 'none'
preset_name = 'be_preset_02'
policies = [[1, 1, 1, 2]]
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
GeneralAlba.add_preset(backend.alba_backend, preset_name, policies, compression, encryption)
for x in range(nr_of_disks_to_create):
namespace_name = namespace_prefix + str(x)
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'create-namespace', [namespace_name, preset_name], False)
GeneralAlba.upload_file(backend.alba_backend, namespace_name, 1024 * 1024 * 1)
AlbaScheduledTaskController.verify_namespaces()
GeneralAlba.remove_alba_namespaces(backend.alba_backend)
GeneralAlba.remove_preset(backend.alba_backend, preset_name)
def verify_policies_for_preset(preset_name, policies, compression, encryption):
"""
Verify the policies of a preset
:param preset_name: Name of preset
:param policies: Policies to verify
:param compression: Compression for preset
:param encryption: Encryption for preset
:return: True is valid
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
valid = False
for preset in backend.alba_backend.presets:
if preset['name'] == preset_name:
assert preset['compression'] == compression,\
"Alba compression {0} does not match configured {1} type".format(preset['compression'],
compression)
assert preset['fragment_encryption'][0] == encryption, \
"Alba encryption {0} does not match configured {1} type".format(preset['fragment_encryption'],
encryption)
for policy in policies:
valid = False
for alba_policy in preset['policies']:
print 'Matching: {0} with {1}'.format(tuple(policy), alba_policy)
if tuple(policy) == alba_policy:
valid = True
continue
return valid
def ovs_2263_verify_alba_namespace_cleanup_test():
"""
Verify ALBA namespace cleanup
Create an amount of namespaces in ALBA
Create a vPool and create some volumes
Verify the amount of namespaces before and after vPool creation
Remove the vPool and the manually created namespaces
Verify the amount of namespaces before and after vPool deletion
"""
# Create some namespaces in alba
no_namespaces = 3
backend_name = General.get_config().get('backend', 'name')
backend = GeneralBackend.get_by_name(name=backend_name)
namespace_name = 'autotest-ns_'
namespace_name_regex = re.compile('^autotest-ns_\d$')
for nmspc_index in range(no_namespaces):
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'create-namespace', ['{0}{1}'.format(namespace_name, nmspc_index), 'default'], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces, "Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
# Create a vPool and create volumes on it
vpool, _ = GeneralVPool.add_vpool()
root_client = SSHClient(GeneralStorageRouter.get_local_storagerouter(), username='******')
if vpool.storagedrivers[0].storagerouter.pmachine.hvtype == 'VMWARE':
GeneralVPool.mount_vpool(vpool=vpool,
root_client=root_client)
vdisks = []
for disk_index in range(no_namespaces):
vdisks.append(GeneralVDisk.create_volume(size=10,
vpool=vpool,
root_client=root_client))
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend)
assert len(result) == 2 * no_namespaces + 1, "Expected {0} namespaces present on the {1} backend, found {2}".format(2 * no_namespaces + 1, backend_name, len(result))
# Remove files and vPool
for vdisk in vdisks:
GeneralVDisk.delete_volume(vdisk=vdisk,
vpool=vpool,
root_client=root_client)
if vpool.storagedrivers[0].storagerouter.pmachine.hvtype == 'VMWARE':
GeneralVPool.unmount_vpool(vpool=vpool,
root_client=root_client)
GeneralVPool.remove_vpool(vpool)
# Verify amount of namespaces
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces, "Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
for namespace in result:
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'delete-namespace', [namespace['name']], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == 0, "Expected no namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
def add_alba_backend(name, wait=True):
"""
Put an ALBA backend in the model
:param name: Name of the backend
:param wait: Wait for backend to enter RUNNING state
:return: Newly created ALBA backend
"""
backend = GeneralBackend.get_by_name(name)
if backend is None:
backend = GeneralBackend.add_backend(name, 'alba')
alba_backend = AlbaBackend(GeneralAlba.api.add('alba/backends', {'backend_guid': backend.guid})['guid'])
if wait is True:
GeneralAlba.wait_for_alba_backend_status(alba_backend)
out, err, _ = General.execute_command('etcdctl ls /ovs/alba/asdnodes')
if err == '' and len(out):
AlbaNodeController.model_local_albanode()
return GeneralBackend.get_by_name(name).alba_backend
def teardown():
"""
Teardown for vPool package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
be = GeneralBackend.get_by_name(autotest_config.get('backend', 'name'))
if be:
GeneralAlba.unclaim_disks_and_remove_alba_backend(alba_backend=be.alba_backend)
def is_preset_present(name):
"""
Verify if a preset is present
:param name: Name of the preset
:return: True if present
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
for preset in backend.alba_backend.presets:
if name == preset['name']:
return True
return False
def be_0007_add_update_remove_preset_test():
"""
Add, update and remove a preset
Validation for OVS-3187 - edit policy of preset
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
alba_backend = GeneralAlba.add_alba_backend(TestALBA.backend_name)
else:
alba_backend = backend.alba_backend
GeneralAlba.claim_asds(alba_backend=alba_backend, nr_of_asds=3, disk_type='SATA')
timeout = 300
preset_name = 'be_preset_0007'
namespace_name = 'be_0007_ns'
compression = 'none'
encryption = 'aes-cbc-256'
org_policy = [[1, 1, 1, 2]]
new_policy = [[2, 2, 3, 3]]
TestALBA.add_validate_remove_preset(preset_name, compression, encryption, org_policy, remove_when_finished=False)
result = GeneralAlba.list_alba_namespaces(alba_backend=alba_backend,
name=namespace_name)
for namespace in result:
GeneralAlba.execute_alba_cli_action(alba_backend, 'delete-namespace', [namespace['name']], False)
GeneralAlba.execute_alba_cli_action(alba_backend, 'create-namespace', [namespace_name, preset_name], False)
GeneralAlba.upload_file(alba_backend=alba_backend, namespace_name=namespace_name, file_size=1024 * 1024)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert len(result) == 1, "Only one policy should be present, found: {0}".format(result)
# update and verify policies for preset
GeneralAlba.update_preset(alba_backend, preset_name, new_policy)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert len(result) == 1, "Expected 1 policy, but got: {0}".format(result)
object_has_new_policy = False
for _ in xrange(timeout):
if GeneralAlba.is_bucket_count_valid_with_policy(result, new_policy):
object_has_new_policy = True
break
time.sleep(1)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert object_has_new_policy is True, "Object was not rewritten within {0} seconds: {1}".format(timeout, result)
# cleanup
GeneralAlba.execute_alba_cli_action(alba_backend, 'delete-namespace', [namespace_name], False)
GeneralAlba.remove_preset(alba_backend, preset_name)
def ovs_3977_maintenance_agent_test():
"""
Test maintenance agent processes
"""
def _get_agent_distribution(agent_name):
result = {}
total = 0
for ip in alba_node_ips:
count = General.execute_command_on_node(ip, 'ls /etc/init/alba-maintenance_{0}-* | wc -l'.format(agent_name))
if count:
count = int(count)
else:
count = 0
total += count
result[ip] = count
result['total'] = total
print 'Maintenance agent distribution: {0}'.format(result)
for ip in alba_node_ips:
assert (result[ip] == total / len(alba_node_ips) or result[ip] == (total / len(alba_node_ips)) + 1),\
"Agents not equally distributed!"
return result
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
name = backend.alba_backend.name
alba_node_ips = [node.ip for node in GeneralAlba.get_alba_nodes()]
etcd_key = '/ovs/alba/backends/{0}/maintenance/nr_of_agents'.format(backend.alba_backend.guid)
nr_of_agents = EtcdConfiguration.get(etcd_key)
print '1. - nr of agents: {0}'.format(nr_of_agents)
actual_nr_of_agents = _get_agent_distribution(name)['total']
assert nr_of_agents == actual_nr_of_agents, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
# set nr to zero
EtcdConfiguration.set(etcd_key, 0)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 0, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '2. - nr of agents: {0}'.format(nr_of_agents)
# set nr to 10
EtcdConfiguration.set(etcd_key, 10)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 10, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '3. - nr of agents: {0}'.format(nr_of_agents)
def be_0005_add_remove_preset_no_compression_encryption_test():
"""
Add and remove a preset without compression and with encryption
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
name = 'be_preset_05'
compression = 'none'
encryption = 'aes-cbc-256'
policies = [[1, 1, 1, 2]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def validate_alba_backend_removal(alba_backend_info):
"""
Validate whether the backend has been deleted properly
alba_backend_info should be a dictionary containing:
- guid
- name
- maintenance_service_names
:param alba_backend_info: Information about the backend
:return: None
"""
Toolbox.verify_required_params(actual_params=alba_backend_info,
required_params={'name': (str, None),
'guid': (str, Toolbox.regex_guid),
'maintenance_service_names': (list, None)},
exact_match=True)
alba_backend_guid = alba_backend_info['guid']
alba_backend_name = alba_backend_info['name']
backend = GeneralBackend.get_by_name(alba_backend_name)
assert backend is None,\
'Still found a backend in the model with name {0}'.format(alba_backend_name)
# Validate services removed from model
for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.ALBA_MGR):
assert service.name != '{0}-abm'.format(alba_backend_name),\
'An AlbaManager service has been found with name {0}'.format(alba_backend_name)
for service in GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.NS_MGR):
assert service.name.startswith('{0}-nsm_'.format(alba_backend_name)) is False,\
'An NamespaceManager service has been found with name {0}'.format(alba_backend_name)
# Validate ALBA backend configuration structure
alba_backend_key = '/ovs/alba/backends'
actual_configuration_keys = [key for key in Configuration.list(alba_backend_key)]
assert alba_backend_guid not in actual_configuration_keys,\
'Configuration still contains an entry in {0} with guid {1}'.format(alba_backend_key, alba_backend_guid)
# Validate Arakoon configuration structure
arakoon_keys = [key for key in Configuration.list('/ovs/arakoon') if key.startswith(alba_backend_name)]
assert len(arakoon_keys) == 0,\
'Configuration still contains configurations for clusters: {0}'.format(', '.join(arakoon_keys))
# Validate services
for storagerouter in GeneralStorageRouter.get_storage_routers():
root_client = SSHClient(endpoint=storagerouter, username='******')
maintenance_services = alba_backend_info['maintenance_service_names']
abm_arakoon_service_name = 'ovs-arakoon-{0}-abm'.format(alba_backend_name)
nsm_arakoon_service_name = 'ovs-arakoon-{0}-nsm_0'.format(alba_backend_name)
for service_name in [abm_arakoon_service_name, nsm_arakoon_service_name] + maintenance_services:
assert GeneralService.has_service(name=service_name, client=root_client) is False,\
'Service {0} still deployed on Storage Router {1}'.format(service_name, storagerouter.name)
def be_0004_validate_preset_with_replication_copies_test():
"""
Validate a preset
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
compression = 'none'
encryption = 'none'
name_prefix = 'be_preset_04'
for nr in xrange(6):
name = name_prefix + str(nr)
policies = [[1, nr, 1, 1 + nr]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def be_0006_add_remove_preset_compression_encryption_test():
"""
Add and remove a preset with compression and encryption
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
name = 'be_preset_06a'
compression = 'bz2'
encryption = 'aes-cbc-256'
policies = [[1, 1, 1, 2]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
name = 'be_preset_06b'
compression = 'snappy'
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def ovs_3490_add_remove_preset_test():
"""
Adds and removes a preset with encryption to an existing alba backend
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
name = 'ovs-3490'
policies = [[1, 1, 1, 2]]
compression = 'none'
encryption = 'aes-cbc-256'
status, message = GeneralAlba.add_preset(backend.alba_backend, name, policies, compression, encryption)
assert status, "Add preset failed with: {0}".format(message)
TestALBA.verify_policies_for_preset(name, policies, compression, encryption)
status, message = GeneralAlba.remove_preset(backend.alba_backend, name)
assert status, "Remove preset failed with: {0}".format(message)
assert not TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
def add_validate_remove_preset(name, compression, encryption, policies, remove_when_finished=True):
"""
Add a preset, validate the preset and remove it
:param name: Name of the preset
:param compression: Compression used by the preset
:param encryption: Encryption used by the preset
:param policies: Policies used by the preset
:param remove_when_finished: Remove after validation
:return: None
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
status, message = GeneralAlba.add_preset(backend.alba_backend, name, policies, compression, encryption)
assert status, "Add preset failed with: {0}".format(message)
assert TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
TestALBA.verify_policies_for_preset(name, policies, compression, encryption)
if remove_when_finished:
status, message = GeneralAlba.remove_preset(backend.alba_backend, name, )
assert status, "Remove preset failed with: {0}".format(message)
assert not TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
def teardown():
"""
Teardown for VirtualMachine package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
vpool_name = General.get_config().get('vpool', 'name')
vpool = GeneralVPool.get_vpool_by_name(vpool_name)
assert vpool is not None, "No vpool found where one was expected"
GeneralVMachine.logger.info("Cleaning vpool")
GeneralVPool.remove_vpool(vpool)
autotest_config = General.get_config()
be = GeneralBackend.get_by_name(autotest_config.get('backend', 'name'))
if be:
GeneralAlba.unclaim_disks_and_remove_alba_backend(alba_backend=be.alba_backend)
GeneralVMachine.logger.info("Cleaning management center")
for mgmt_center in GeneralManagementCenter.get_mgmt_centers():
GeneralManagementCenter.remove_mgmt_center(mgmt_center)
