def teardown():
"""
Teardown for Arakoon package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is False:
GeneralService.start_service(name='ovs-scheduled-tasks',
client=root_client)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
for key in KEY_CLEANUP:
if EtcdConfiguration.exists('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key), raw = True):
EtcdConfiguration.delete('{0}/{1}'.format(GeneralArakoon.ETCD_CONFIG_ROOT, key))
def setup():
"""
Setup for Arakoon package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
assert backend_name, 'Please fill out a backend name in the autotest.cfg file'
backend = GeneralBackend.get_by_name(backend_name)
if backend is not None:
GeneralAlba.remove_alba_backend(backend.alba_backend)
for storagerouter in GeneralStorageRouter.get_masters():
root_client = SSHClient(storagerouter, username='******')
if GeneralService.get_service_status(name='ovs-scheduled-tasks',
client=root_client) is True:
GeneralService.stop_service(name='ovs-scheduled-tasks',
client=root_client)
storagerouters = GeneralStorageRouter.get_storage_routers()
for sr in storagerouters:
root_client = SSHClient(sr, username='******')
GeneralDisk.add_db_role(sr)
for location in TEST_CLEANUP:
root_client.run('rm -rf {0}'.format(location))
GeneralAlba.add_alba_backend(backend_name)
GeneralArakoon.voldrv_arakoon_checkup()
def setup():
"""
Setup for Virtual Machine package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
General.validate_required_config_settings(settings={'vpool': ['name'],
'backend': ['name']})
# Download the template
cmd = '[ -d {0} ] && echo "Dir exists" || echo "Dir does not exists"'.format(GeneralVMachine.template_target_folder)
out, err, _ = General.execute_command(cmd)
if err:
GeneralVMachine.logger.error("Error while executing command {1}: {0}".format(err, cmd))
if 'not' not in out:
General.execute_command('rm -rf {0}'.format(GeneralVMachine.template_target_folder))
General.execute_command('mkdir {0}'.format(GeneralVMachine.template_target_folder))
grid_ip = General.get_config().get('main', 'grid_ip')
if grid_ip.split('.')[0] == '172' and grid_ip.split('.')[1] == '20':
server_location = 'http://172.20.3.8/templates/openvstorage'
else:
server_location = 'http://sso-qpackages-loch.cloudfounders.com/templates/openvstorage'
GeneralVMachine.logger.info("Getting template from {0}".format(server_location))
out, err, _ = General.execute_command('wget -P {0} {1}{2}{3}'.format(GeneralVMachine.template_target_folder, server_location, '/fio_debian/', GeneralVMachine.template_image))
if err:
GeneralVMachine.logger.error("Error while downloading template: {0}".format(err))
out, err, _ = General.execute_command('chown root {0}{1}'.format(GeneralVMachine.template_target_folder, GeneralVMachine.template_image))
if err:
GeneralVMachine.logger.error("Error while changing user owner to root for template: {0}".format(err))
GeneralAlba.prepare_alba_backend()
_, vpool_params = GeneralVPool.add_vpool(vpool_parameters={'preset': GeneralAlba.ONE_DISK_PRESET})
GeneralVPool.validate_vpool_sanity(expected_settings=vpool_params)
def teardown():
"""
Teardown for vPool package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
alba_backend = GeneralAlba.get_by_name(General.get_config().get('backend', 'name'))
if alba_backend:
GeneralAlba.unclaim_disks_and_remove_alba_backend(alba_backend=alba_backend)
def setup():
"""
Setup for vPool package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
General.validate_required_config_settings(settings={'vpool': ['name', 'type', 'readcache_size', 'writecache_size', 'integrate_mgmt',
'storage_ip', 'config_params', 'fragment_cache_on_read', 'fragment_cache_on_write'],
'backend': ['name']})
GeneralAlba.prepare_alba_backend()
def setup():
"""
Setup for VirtualDisk package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
General.validate_required_config_settings(settings={'vpool': ['name'],
'backend': ['name']})
GeneralAlba.prepare_alba_backend()
_, vpool_params = GeneralVPool.add_vpool(vpool_parameters={'preset': GeneralAlba.ONE_DISK_PRESET})
GeneralVPool.validate_vpool_sanity(expected_settings=vpool_params)
def setup():
"""
Setup for vPool package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
assert backend_name, "Please fill out a valid backend name in autotest.cfg file"
GeneralAlba.prepare_alba_backend()
def teardown():
"""
Teardown for Backend package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
backend = GeneralBackend.get_by_name(backend_name)
if backend:
GeneralAlba.unclaim_disks(backend.alba_backend)
GeneralAlba.remove_alba_backend(backend.alba_backend)
def be_0005_add_remove_preset_no_compression_encryption_test():
"""
Add and remove a preset without compression and with encryption
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
name = 'be_preset_05'
compression = 'none'
encryption = 'aes-cbc-256'
policies = [[1, 1, 1, 2]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def be_0004_validate_preset_with_replication_copies_test():
"""
Validate a preset
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
compression = 'none'
encryption = 'none'
name_prefix = 'be_preset_04'
for nr in xrange(6):
name = name_prefix + str(nr)
policies = [[1, nr, 1, 1 + nr]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def setup():
"""
Setup for Backend package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
General.validate_required_config_settings(settings={'backend': ['name']})
my_sr = GeneralStorageRouter.get_local_storagerouter()
if GeneralStorageRouter.has_roles(storagerouter=my_sr, roles='DB') is False:
GeneralDisk.add_db_role(my_sr)
alba_backend_name = General.get_config().get('backend', 'name')
alba_backend = GeneralAlba.get_by_name(alba_backend_name)
if alba_backend is None:
GeneralAlba.add_alba_backend(alba_backend_name)
def teardown():
"""
Teardown for VirtualMachine package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
vpool_name = General.get_config().get('vpool', 'name')
vpool = GeneralVPool.get_vpool_by_name(vpool_name)
assert vpool is not None, "No vpool found where one was expected"
GeneralVMachine.logger.info("Cleaning vpool")
GeneralVPool.remove_vpool(vpool)
alba_backend = GeneralAlba.get_by_name(General.get_config().get('backend', 'name'))
if alba_backend is not None:
GeneralAlba.unclaim_disks_and_remove_alba_backend(alba_backend=alba_backend)
def ovs_2263_verify_alba_namespace_cleanup_test():
"""
Verify ALBA namespace cleanup
Create an amount of namespaces in ALBA
Create a vPool and create some volumes
Verify the amount of namespaces before and after vPool creation
Remove the vPool and the manually created namespaces
Verify the amount of namespaces before and after vPool deletion
"""
# Create some namespaces in alba
no_namespaces = 3
backend_name = General.get_config().get('backend', 'name')
backend = GeneralBackend.get_by_name(name=backend_name)
namespace_name = 'autotest-ns_'
namespace_name_regex = re.compile('^autotest-ns_\d$')
for nmspc_index in range(no_namespaces):
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'create-namespace', ['{0}{1}'.format(namespace_name, nmspc_index), 'default'], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces, "Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
# Create a vPool and create volumes on it
vpool, _ = GeneralVPool.add_vpool()
root_client = SSHClient(GeneralStorageRouter.get_local_storagerouter(), username='******')
if vpool.storagedrivers[0].storagerouter.pmachine.hvtype == 'VMWARE':
GeneralVPool.mount_vpool(vpool=vpool,
root_client=root_client)
vdisks = []
for disk_index in range(no_namespaces):
vdisks.append(GeneralVDisk.create_volume(size=10,
vpool=vpool,
root_client=root_client))
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend)
assert len(result) == 2 * no_namespaces + 1, "Expected {0} namespaces present on the {1} backend, found {2}".format(2 * no_namespaces + 1, backend_name, len(result))
# Remove files and vPool
for vdisk in vdisks:
GeneralVDisk.delete_volume(vdisk=vdisk,
vpool=vpool,
root_client=root_client)
if vpool.storagedrivers[0].storagerouter.pmachine.hvtype == 'VMWARE':
GeneralVPool.unmount_vpool(vpool=vpool,
root_client=root_client)
GeneralVPool.remove_vpool(vpool)
# Verify amount of namespaces
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces, "Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
for namespace in result:
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'delete-namespace', [namespace['name']], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=backend.alba_backend,
name=namespace_name_regex)
assert len(result) == 0, "Expected no namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name, len(result))
def be_0006_add_remove_preset_compression_encryption_test():
"""
Add and remove a preset with compression and encryption
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
GeneralAlba.add_alba_backend(TestALBA.backend_name)
name = 'be_preset_06a'
compression = 'bz2'
encryption = 'aes-cbc-256'
policies = [[1, 1, 1, 2]]
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
name = 'be_preset_06b'
compression = 'snappy'
TestALBA.add_validate_remove_preset(name, compression, encryption, policies)
def ovs_3490_add_remove_preset_test():
"""
Adds and removes a preset with encryption to an existing alba backend
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
name = 'ovs-3490'
policies = [[1, 1, 1, 2]]
compression = 'none'
encryption = 'aes-cbc-256'
status, message = GeneralAlba.add_preset(backend.alba_backend, name, policies, compression, encryption)
assert status, "Add preset failed with: {0}".format(message)
TestALBA.verify_policies_for_preset(name, policies, compression, encryption)
status, message = GeneralAlba.remove_preset(backend.alba_backend, name)
assert status, "Remove preset failed with: {0}".format(message)
assert not TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
def teardown():
"""
Teardown for VirtualDisk package, will be executed when all started tests in this package have ended
Removal actions of possible things left over after the test-run
:return: None
"""
vpool_name = General.get_config().get("vpool", "name")
vpool = GeneralVPool.get_vpool_by_name(vpool_name)
for vd in VDiskList.get_vdisks():
GeneralVDisk.delete_volume(vd, vpool, loop_device='loop0')
if vpool is not None:
GeneralVPool.remove_vpool(vpool)
alba_backend = GeneralAlba.get_by_name(General.get_config().get('backend', 'name'))
if alba_backend is not None:
GeneralAlba.unclaim_disks_and_remove_alba_backend(alba_backend=alba_backend)
def add_validate_remove_preset(name, compression, encryption, policies, remove_when_finished=True):
"""
Add a preset, validate the preset and remove it
:param name: Name of the preset
:param compression: Compression used by the preset
:param encryption: Encryption used by the preset
:param policies: Policies used by the preset
:param remove_when_finished: Remove after validation
:return: None
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
status, message = GeneralAlba.add_preset(backend.alba_backend, name, policies, compression, encryption)
assert status, "Add preset failed with: {0}".format(message)
assert TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
TestALBA.verify_policies_for_preset(name, policies, compression, encryption)
if remove_when_finished:
status, message = GeneralAlba.remove_preset(backend.alba_backend, name, )
assert status, "Remove preset failed with: {0}".format(message)
assert not TestALBA.is_preset_present(name), "Preset with name {0} is not present".format(name)
def setup():
"""
Setup for Virtual Machine package, will be executed when any test in this package is being executed
Make necessary changes before being able to run the tests
:return: None
"""
autotest_config = General.get_config()
backend_name = autotest_config.get('backend', 'name')
assert backend_name, "Please fill out a valid backend name in autotest.cfg file"
# Download the template
cmd = '[ -d {0} ] && echo "Dir exists" || echo "Dir does not exists"'.format(GeneralVMachine.template_target_folder)
out, err, _ = General.execute_command(cmd)
if err:
GeneralVMachine.logger.error("Error while executing command {1}: {0}".format(err, cmd))
if 'not' not in out:
General.execute_command('rm -rf {0}'.format(GeneralVMachine.template_target_folder))
General.execute_command('mkdir {0}'.format(GeneralVMachine.template_target_folder))
grid_ip = General.get_config().get('main', 'grid_ip')
if grid_ip.split('.')[0] == '172' and grid_ip.split('.')[1] == '20':
server_location = 'http://172.20.3.8/templates/openvstorage'
else:
server_location = 'http://sso-qpackages-loch.cloudfounders.com/templates/openvstorage'
GeneralVMachine.logger.info("Getting template from {0}".format(server_location))
out, err, _ = General.execute_command('wget -P {0} {1}{2}{3}'.format(GeneralVMachine.template_target_folder, server_location, '/fio_debian/', GeneralVMachine.template_image))
if err:
GeneralVMachine.logger.error("Error while downloading template: {0}".format(err))
out, err, _ = General.execute_command('chown root {0}{1}'.format(GeneralVMachine.template_target_folder, GeneralVMachine.template_image))
if err:
GeneralVMachine.logger.error("Error while changing user owner to root for template: {0}".format(err))
GeneralAlba.prepare_alba_backend()
GeneralManagementCenter.create_generic_mgmt_center()
GeneralVPool.add_vpool()
def ovs_3977_maintenance_agent_test():
"""
Test maintenance agent processes
"""
def _get_agent_distribution(agent_name):
result = {}
total = 0
for ip in alba_node_ips:
count = General.execute_command_on_node(ip, 'ls /etc/init/alba-maintenance_{0}-* | wc -l'.format(agent_name))
if count:
count = int(count)
else:
count = 0
total += count
result[ip] = count
result['total'] = total
print 'Maintenance agent distribution: {0}'.format(result)
for ip in alba_node_ips:
assert (result[ip] == total / len(alba_node_ips) or result[ip] == (total / len(alba_node_ips)) + 1),\
"Agents not equally distributed!"
return result
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
name = backend.alba_backend.name
alba_node_ips = [node.ip for node in GeneralAlba.get_alba_nodes()]
etcd_key = '/ovs/alba/backends/{0}/maintenance/nr_of_agents'.format(backend.alba_backend.guid)
nr_of_agents = EtcdConfiguration.get(etcd_key)
print '1. - nr of agents: {0}'.format(nr_of_agents)
actual_nr_of_agents = _get_agent_distribution(name)['total']
assert nr_of_agents == actual_nr_of_agents, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
# set nr to zero
EtcdConfiguration.set(etcd_key, 0)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 0, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '2. - nr of agents: {0}'.format(nr_of_agents)
# set nr to 10
EtcdConfiguration.set(etcd_key, 10)
GeneralAlba.checkup_maintenance_agents()
assert _get_agent_distribution(name)['total'] == 10, \
'Actual {0} and requested {1} nr of agents does not match'.format(nr_of_agents, actual_nr_of_agents)
print '3. - nr of agents: {0}'.format(nr_of_agents)
def be_0001_add_and_remove_backend_test():
"""
Create an ALBA backend and verify its status
Validate services, etcd, arakoon without claiming disks
Claim some disks and validate whether backend can be used for storing objects in namespaces
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is not None:
raise ValueError('A backend has already been deployed, cannot execute test')
alba_backend = GeneralAlba.add_alba_backend(TestALBA.backend_name)
GeneralAlba.validate_alba_backend_sanity_without_claimed_disks(alba_backend=alba_backend)
GeneralAlba.claim_asds(alba_backend, 3, 'SATA')
GeneralAlba.validate_alba_backend_sanity_with_claimed_disks(alba_backend=alba_backend)
guid = alba_backend.guid
name = TestALBA.backend_name
service_names = GeneralAlba.get_maintenance_services_for_alba_backend(alba_backend=alba_backend)
GeneralAlba.unclaim_disks(alba_backend)
GeneralAlba.remove_alba_backend(alba_backend)
GeneralAlba.validate_alba_backend_removal(alba_backend_info={'name': name,
'guid': guid,
'maintenance_service_names': service_names})
def ovs_3188_verify_namespace_test():
"""
Verify namespaces
"""
nr_of_disks_to_create = 5
namespace_prefix = 'ovs_3188-'
compression = 'none'
encryption = 'none'
preset_name = 'be_preset_02'
policies = [[1, 1, 1, 2]]
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
backend = GeneralAlba.add_alba_backend(TestALBA.backend_name).backend
GeneralAlba.add_preset(backend.alba_backend, preset_name, policies, compression, encryption)
for x in range(nr_of_disks_to_create):
namespace_name = namespace_prefix + str(x)
GeneralAlba.execute_alba_cli_action(backend.alba_backend, 'create-namespace', [namespace_name, preset_name], False)
GeneralAlba.upload_file(backend.alba_backend, namespace_name, 1024 * 1024 * 1)
AlbaScheduledTaskController.verify_namespaces()
GeneralAlba.remove_alba_namespaces(backend.alba_backend)
GeneralAlba.remove_preset(backend.alba_backend, preset_name)
def be_0007_add_update_remove_preset_test():
"""
Add, update and remove a preset
Validation for OVS-3187 - edit policy of preset
"""
backend = GeneralBackend.get_by_name(TestALBA.backend_name)
if backend is None:
alba_backend = GeneralAlba.add_alba_backend(TestALBA.backend_name)
else:
alba_backend = backend.alba_backend
GeneralAlba.claim_asds(alba_backend=alba_backend, nr_of_asds=3, disk_type='SATA')
timeout = 300
preset_name = 'be_preset_0007'
namespace_name = 'be_0007_ns'
compression = 'none'
encryption = 'aes-cbc-256'
org_policy = [[1, 1, 1, 2]]
new_policy = [[2, 2, 3, 3]]
TestALBA.add_validate_remove_preset(preset_name, compression, encryption, org_policy, remove_when_finished=False)
result = GeneralAlba.list_alba_namespaces(alba_backend=alba_backend,
name=namespace_name)
for namespace in result:
GeneralAlba.execute_alba_cli_action(alba_backend, 'delete-namespace', [namespace['name']], False)
GeneralAlba.execute_alba_cli_action(alba_backend, 'create-namespace', [namespace_name, preset_name], False)
GeneralAlba.upload_file(alba_backend=alba_backend, namespace_name=namespace_name, file_size=1024 * 1024)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert len(result) == 1, "Only one policy should be present, found: {0}".format(result)
# update and verify policies for preset
GeneralAlba.update_preset(alba_backend, preset_name, new_policy)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert len(result) == 1, "Expected 1 policy, but got: {0}".format(result)
object_has_new_policy = False
for _ in xrange(timeout):
if GeneralAlba.is_bucket_count_valid_with_policy(result, new_policy):
object_has_new_policy = True
break
time.sleep(1)
result = GeneralAlba.execute_alba_cli_action(alba_backend, 'show-namespace', [namespace_name])['bucket_count']
assert object_has_new_policy is True, "Object was not rewritten within {0} seconds: {1}".format(timeout, result)
# cleanup
GeneralAlba.execute_alba_cli_action(alba_backend, 'delete-namespace', [namespace_name], False)
GeneralAlba.remove_preset(alba_backend, preset_name)
