def shrink_vpool(storage_driver):
"""
Remove a Storage Driver from a vPool
:param storage_driver: Storage Driver to remove from the vPool
:return: None
"""
vpool = storage_driver.vpool
if GeneralHypervisor.get_hypervisor_type() == "VMWARE":
root_client = SSHClient(storage_driver.storagerouter, username="******")
if storage_driver.mountpoint in General.get_mountpoints(root_client):
root_client.run(["umount", "storage_driver.mountpoint"])
task_result = GeneralVPool.api.execute_post_action(
component="vpools",
guid=vpool.guid,
action="shrink_vpool",
data={"storagerouter_guid": storage_driver.storagerouter.guid},
wait=True,
timeout=GeneralVPool.TIMEOUT_ADD_VPOOL,
)
if task_result[0] is not True:
raise RuntimeError(
"Storage Driver with ID {0} was not successfully removed from vPool {1}".format(
storage_driver.storagedriver_id, vpool.name
),
task_result,
)
return GeneralVPool.get_vpool_by_name(vpool_name=vpool.name)
def get_filesystem_location(vpool, vdisk_name):
"""
Retrieve the absolute path of the disk for the vPool
:param vpool: vPool on which the disk is hosted
:param vdisk_name: Disk to retrieve path for
:return: Absolute path
"""
hv_type = GeneralHypervisor.get_hypervisor_type()
if hv_type == 'VMWARE':
location = '/'.join(['/mnt/{0}'.format(vpool.name), "{0}-flat.vmdk".format(vdisk_name)])
elif hv_type == 'KVM':
location = '/'.join(['/mnt/{0}'.format(vpool.name), "{0}.raw".format(vdisk_name)])
else:
raise RuntimeError('Invalid hypervisor type specified: {0}'.format(hv_type))
return location
def get_related_files(vpool):
"""
Retrieve the files generated during vPool creation
:param vpool: vPool to retrieve the related files for
:return: List of file locations
"""
all_files = {}
for storagedriver in vpool.storagedrivers:
files = set()
if GeneralHypervisor.get_hypervisor_type() == "VMWARE":
volumedriver_mode = Configuration.get(
"/ovs/framework/hosts/{0}/storagedriver|vmware_mode".format(storagedriver.storagerouter.machine_id)
)
if volumedriver_mode == "ganesha":
files.add(
"/opt/OpenvStorage/config/storagedriver/storagedriver/{0}_ganesha.conf".format(vpool.name)
)
all_files[storagedriver.storagerouter.guid] = files
return all_files
def cleanup():
"""
Do some cleanup actions
:return: None
"""
from ci.tests.general.general_pmachine import GeneralPMachine
from ci.tests.general.general_vdisk import GeneralVDisk
from ci.tests.general.general_vmachine import GeneralVMachine
def _get_remote_ssh_connection(ip_address, username, password):
import paramiko
ssh_connection = paramiko.SSHClient()
ssh_connection.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_connection.connect(ip_address, username=username, password=password, timeout=2)
sftp = ssh_connection.open_sftp()
return ssh_connection, sftp
# @TODO: Split this cleanup function up in relevant parts and put them in the correct general files
machine_name = "AT_"
from ci.tests.general import general_hypervisor
from ci.tests.general.general_vpool import GeneralVPool
for vpool in GeneralVPool.get_vpools():
if vpool:
hpv = general_hypervisor.Hypervisor.get(vpool)
vm_names = [vm.name for vm in GeneralVMachine.get_vmachines()]
for name in vm_names:
vm = GeneralVMachine.get_vmachine_by_name(name)
if not vm:
continue
vm = vm[0]
if not vm.name.startswith(machine_name):
continue
if vm.is_vtemplate:
hpv.delete_clones(vm)
logging.log(1, "Deleting {0} on hypervisor".format(vm.name))
hpv.poweroff(vm.name)
hpv.delete(vm.name)
env_macs = General.execute_command("""ip a | awk '/link\/ether/ {gsub(":","",$2);print $2;}'""")[0].splitlines()
if vpool.storagedrivers:
mountpoint = vpool.storagedrivers[0].mountpoint
if os.path.exists(mountpoint):
for d in os.listdir(mountpoint):
if d.startswith(machine_name):
p = '/'.join([mountpoint, d])
if os.path.isdir(p):
logging.log(1, "removing tree: {0}".format(p))
shutil.rmtree(p)
else:
logging.log(1, "removing file: {0}".format(p))
if os.path.isfile(p):
os.remove(p)
for mac in env_macs:
mac_path = '/'.join([mountpoint, mac])
if os.path.exists(mac_path):
for f in os.listdir(mac_path):
logging.log(1, "removing file: {0}".format(f))
os.remove('/'.join([mac_path, f]))
# remove existing disks
vdisks = GeneralVDisk.get_vdisks()
for vdisk in vdisks:
if vdisk:
for junction in vdisk.mds_services:
if junction:
junction.delete()
vdisk.delete()
logging.log(1, 'WARNING: Removed leftover disk: {0}'.format(vdisk.name))
GeneralVPool.remove_vpool(vpool)
if GeneralPMachine.get_hypervisor_type() == 'VMWARE':
from ci.tests.general.general_hypervisor import GeneralHypervisor
hypervisor_info = GeneralHypervisor.get_hypervisor_info()
ssh_con = _get_remote_ssh_connection(*hypervisor_info)[0]
cmd = "esxcli storage nfs remove -v {0}".format(vpool.name)
ssh_con.exec_command(cmd)
vmachines = GeneralVMachine.get_vmachines()
for vmachine in vmachines:
logging.log(1, 'WARNING: Removing leftover vmachine: {0}'.format(vmachine.name))
vmachine.delete()
def check_vpool_cleanup(vpool_info, storagerouters=None):
"""
Check if everything related to a vPool has been cleaned up on the storagerouters provided
vpool_info should be a dictionary containing:
- type
- guid
- files
- directories
- name (optional)
- vpool (optional)
If vpool is provided:
- storagerouters need to be provided, because on these Storage Routers, we check whether the vPool has been cleaned up
If name is provided:
- If storagerouters is NOT provided, all Storage Routers will be checked for a correct vPool removal
- If storagerouters is provided, only these Storage Routers will be checked for a correct vPool removal
:param vpool_info: Information about the vPool
:param storagerouters: Storage Routers to check if vPool has been cleaned up
:return: None
"""
for required_param in ["type", "guid", "files", "directories"]:
if required_param not in vpool_info:
raise ValueError("Incorrect vpool_info provided")
if "vpool" in vpool_info and "name" in vpool_info:
raise ValueError("vpool and name are mutually exclusive")
if "vpool" not in vpool_info and "name" not in vpool_info:
raise ValueError("Either vpool or vpool_name needs to be provided")
vpool = vpool_info.get("vpool")
vpool_name = vpool_info.get("name")
vpool_guid = vpool_info["guid"]
vpool_type = vpool_info["type"]
files = vpool_info["files"]
directories = vpool_info["directories"]
supported_backend_types = GeneralBackend.get_valid_backendtypes()
if vpool_type not in supported_backend_types:
raise ValueError(
"Unsupported Backend Type provided. Please choose from: {0}".format(", ".join(supported_backend_types))
)
if storagerouters is None:
storagerouters = GeneralStorageRouter.get_storage_routers()
if vpool_name is not None:
assert (
GeneralVPool.get_vpool_by_name(vpool_name=vpool_name) is None
), "A vPool with name {0} still exists".format(vpool_name)
# Prepare some fields to check
vpool_name = vpool.name if vpool else vpool_name
vpool_services = ["ovs-dtl_{0}".format(vpool_name), "ovs-volumedriver_{0}".format(vpool_name)]
if vpool_type == "alba":
vpool_services.append("ovs-albaproxy_{0}".format(vpool_name))
# Check configuration
if vpool is None:
assert (
Configuration.exists("/ovs/vpools/{0}".format(vpool_guid), raw=True) is False
), "vPool config still found in etcd"
else:
remaining_sd_ids = set([storagedriver.storagedriver_id for storagedriver in vpool.storagedrivers])
current_sd_ids = set([item for item in Configuration.list("/ovs/vpools/{0}/hosts".format(vpool_guid))])
assert not remaining_sd_ids.difference(
current_sd_ids
), "There are more storagedrivers modelled than present in etcd"
assert not current_sd_ids.difference(
remaining_sd_ids
), "There are more storagedrivers in etcd than present in model"
# Perform checks on all storagerouters where vpool was removed
for storagerouter in storagerouters:
# Check MDS services
mds_services = GeneralService.get_services_by_name(ServiceType.SERVICE_TYPES.MD_SERVER)
assert (
len(
[
mds_service
for mds_service in mds_services
if mds_service.storagerouter_guid == storagerouter.guid
]
)
== 0
), "There are still MDS services present for Storage Router {0}".format(storagerouter.ip)
# Check services
root_client = SSHClient(storagerouter, username="******")
for service in vpool_services:
if ServiceManager.has_service(service, client=root_client):
raise RuntimeError(
"Service {0} is still configured on Storage Router {1}".format(service, storagerouter.ip)
)
# Check KVM vpool
if GeneralHypervisor.get_hypervisor_type() == "KVM":
vpool_overview = root_client.run(["virsh", "pool-list", "--all"]).splitlines()
vpool_overview.pop(1)
vpool_overview.pop(0)
for vpool_info in vpool_overview:
kvm_vpool_name = vpool_info.split()[0].strip()
if vpool_name == kvm_vpool_name:
raise ValueError(
"vPool {0} is still defined on Storage Router {1}".format(vpool_name, storagerouter.ip)
)
# Check file and directory existence
if storagerouter.guid not in directories:
raise ValueError("Could not find directory information for Storage Router {0}".format(storagerouter.ip))
if storagerouter.guid not in files:
raise ValueError("Could not find file information for Storage Router {0}".format(storagerouter.ip))
for directory in directories[storagerouter.guid]:
assert (
root_client.dir_exists(directory) is False
), "Directory {0} still exists on Storage Router {1}".format(directory, storagerouter.ip)
for file_name in files[storagerouter.guid]:
assert (
root_client.file_exists(file_name) is False
), "File {0} still exists on Storage Router {1}".format(file_name, storagerouter.ip)
# Look for errors in storagedriver log
for error_type in ["error", "fatal"]:
cmd = "cat -vet /var/log/ovs/volumedriver/{0}.log | tail -1000 | grep ' {1} '; echo true > /dev/null".format(
vpool_name, error_type
)
errors = []
for line in root_client.run(cmd, allow_insecure=True).splitlines():
if "HierarchicalArakoon" in line:
continue
errors.append(line)
if len(errors) > 0:
if error_type == "error":
print "Volumedriver log file contains errors on Storage Router {0}\n - {1}".format(
storagerouter.ip, "\n - ".join(errors)
)
else:
raise RuntimeError(
"Fatal errors found in volumedriver log file on Storage Router {0}\n - {1}".format(
storagerouter.ip, "\n - ".join(errors)
)
)
def validate_vpool_sanity(expected_settings):
"""
Check if all requirements are met for a healthy vPool
:param expected_settings: Parameters used to create a vPool, which will be verified
:type expected_settings: dict
:return: None
"""
if not isinstance(expected_settings, dict) or len(expected_settings) == 0:
raise ValueError("Cannot validate vpool when no settings are passed")
generic_settings = expected_settings.values()[0]
vpool_name = generic_settings["vpool_name"]
mountpoint = "/mnt/{0}".format(vpool_name)
backend_type = generic_settings["type"]
rdma_enabled = (
generic_settings["config_params"]["dtl_transport"] == StorageDriverClient.FRAMEWORK_DTL_TRANSPORT_RSOCKET
)
vpool = GeneralVPool.get_vpool_by_name(vpool_name=vpool_name)
assert vpool is not None, "Could not find vPool with name {0}".format(vpool_name)
vpool_config = GeneralVPool.get_configuration(vpool)
# Verify some basic vPool attributes
assert vpool.name == vpool_name, "Expected name {0} for vPool".format(vpool_name)
assert vpool.status == VPool.STATUSES.RUNNING, "vPool does not have RUNNING status"
assert vpool.rdma_enabled == rdma_enabled, "RDMA enabled setting is incorrect"
assert set(expected_settings.keys()) == set(
[sd.storagerouter for sd in vpool.storagedrivers]
), "vPool storagerouters don't match the expected Storage Routers"
# Verify vPool Storage Driver configuration
expected_vpool_config = copy.deepcopy(generic_settings["config_params"])
for key, value in vpool_config.iteritems():
if key == "dtl_enabled" or key == "tlog_multiplier" or key == "dtl_config_mode":
continue
if key not in expected_vpool_config:
raise ValueError("Expected settings does not contain key {0}".format(key))
if value != expected_vpool_config[key]:
raise ValueError(
"vPool does not have expected configuration {0} for key {1}".format(expected_vpool_config[key], key)
)
expected_vpool_config.pop(key)
if len(expected_vpool_config) > 0:
raise ValueError(
"Actual vPool configuration does not contain keys: {0}".format(", ".join(expected_vpool_config.keys()))
)
# Prepare some fields to check
config = generic_settings["config_params"]
dtl_mode = config["dtl_mode"]
sco_size = config["sco_size"]
cluster_size = config["cluster_size"]
write_buffer = config["write_buffer"]
dtl_transport = config["dtl_transport"]
# @TODO: Add more validations for other expected settings (instead of None)
expected_config = {
"backend_connection_manager": {
"backend_interface_retries_on_error": 5,
"backend_interface_retry_interval_secs": 1,
"backend_interface_retry_backoff_multiplier": 2.0,
},
"content_addressed_cache": {
"clustercache_mount_points": None,
"read_cache_serialization_path": u"/var/rsp/{0}".format(vpool.name),
},
"distributed_lock_store": {
"dls_arakoon_cluster_id": None,
"dls_arakoon_cluster_nodes": None,
"dls_type": u"Arakoon",
},
"distributed_transaction_log": {"dtl_path": None, "dtl_transport": dtl_transport.upper()},
"event_publisher": {"events_amqp_routing_key": u"volumerouter", "events_amqp_uris": None},
"file_driver": {"fd_cache_path": None, "fd_extent_cache_capacity": u"1024", "fd_namespace": None},
"filesystem": {
"fs_dtl_config_mode": u"Automatic",
"fs_dtl_mode": u"{0}".format(StorageDriverClient.VPOOL_DTL_MODE_MAP[dtl_mode]),
"fs_enable_shm_interface": 1,
"fs_file_event_rules": None,
"fs_metadata_backend_arakoon_cluster_nodes": None,
"fs_metadata_backend_mds_nodes": None,
"fs_metadata_backend_type": u"MDS",
"fs_raw_disk_suffix": None,
"fs_virtual_disk_format": None,
},
"metadata_server": {"mds_nodes": None},
"scocache": {"backoff_gap": u"2GB", "scocache_mount_points": None, "trigger_gap": u"1GB"},
"threadpool_component": {"num_threads": 16},
"volume_manager": {
"clean_interval": 1,
"default_cluster_size": 1024 * cluster_size,
"dtl_throttle_usecs": 4000,
"metadata_path": None,
"non_disposable_scos_factor": float(write_buffer)
/ StorageDriverClient.TLOG_MULTIPLIER_MAP[sco_size]
/ sco_size,
"number_of_scos_in_tlog": StorageDriverClient.TLOG_MULTIPLIER_MAP[sco_size],
"tlog_path": None,
},
"volume_registry": {"vregistry_arakoon_cluster_id": u"voldrv", "vregistry_arakoon_cluster_nodes": None},
"volume_router": {
"vrouter_backend_sync_timeout_ms": 5000,
"vrouter_file_read_threshold": 1024,
"vrouter_file_write_threshold": 1024,
"vrouter_id": None,
"vrouter_max_workers": 16,
"vrouter_migrate_timeout_ms": 5000,
"vrouter_min_workers": 4,
"vrouter_redirect_timeout_ms": u"5000",
"vrouter_routing_retries": 10,
"vrouter_sco_multiplier": 1024,
"vrouter_volume_read_threshold": 1024,
"vrouter_volume_write_threshold": 1024,
},
"volume_router_cluster": {"vrouter_cluster_id": None},
}
vpool_services = {
"all": [
"ovs-watcher-volumedriver",
"ovs-dtl_{0}".format(vpool.name),
"ovs-volumedriver_{0}".format(vpool.name),
"ovs-volumerouter-consumer",
],
"extra": [],
"master": ["ovs-arakoon-voldrv"],
}
sd_partitions = {"DB": ["MD", "MDS", "TLOG"], "WRITE": ["FD", "DTL", "SCO"]}
assert Configuration.exists("/ovs/arakoon/voldrv/config", raw=True), "Volumedriver arakoon does not exist"
# Do some verifications for all SDs
storage_ip = None
voldrv_config = GeneralArakoon.get_config("voldrv")
all_files = GeneralVPool.get_related_files(vpool=vpool)
all_directories = GeneralVPool.get_related_directories(vpool=vpool)
for storagedriver in vpool.storagedrivers:
storagerouter = storagedriver.storagerouter
root_client = SSHClient(storagerouter, username="******")
assert Configuration.exists(
"/ovs/vpools/{0}/hosts/{1}/config".format(vpool.guid, storagedriver.storagedriver_id), raw=True
), "vPool config not found in configuration"
# @todo: replace next lines with implementation defined in: http://jira.openvstorage.com/browse/OVS-4577
# current_config_sections = set([item for item in Configuration.list('/ovs/vpools/{0}/hosts/{1}/config'.format(vpool.guid, storagedriver.storagedriver_id))])
# assert not current_config_sections.difference(set(expected_config.keys())), 'New section appeared in the storage driver config in configuration'
# assert not set(expected_config.keys()).difference(current_config_sections), 'Config section expected for storage driver, but not found in configuration'
#
# for key, values in expected_config.iteritems():
# current_config = Configuration.get('/ovs/vpools/{0}/hosts/{1}/config/{2}'.format(vpool.guid, storagedriver.storagedriver_id, key))
# assert set(current_config.keys()).union(set(values.keys())) == set(values.keys()), 'Not all expected keys match for key "{0}" on Storage Driver {1}'.format(key, storagedriver.name)
#
# for sub_key, value in current_config.iteritems():
# expected_value = values[sub_key]
# if expected_value is None:
# continue
# assert value == expected_value, 'Key: {0} - Sub key: {1} - Value: {2} - Expected value: {3}'.format(key, sub_key, value, expected_value)
# Check services
if storagerouter.node_type == "MASTER":
for service_name in vpool_services["all"] + vpool_services["master"]:
if (
service_name == "ovs-arakoon-voldrv"
and GeneralStorageDriver.has_role(storagedriver, "DB") is False
):
continue
exitcode, output = ServiceManager.get_service_status(name=service_name, client=root_client)
if exitcode is not True:
raise ValueError(
"Service {0} is not running on node {1} - {2}".format(
service_name, storagerouter.ip, output
)
)
else:
for service_name in vpool_services["all"] + vpool_services["extra"]:
exitcode, output = ServiceManager.get_service_status(name=service_name, client=root_client)
if exitcode is not True:
raise ValueError(
"Service {0} is not running on node {1} - {2}".format(
service_name, storagerouter.ip, output
)
)
# Check arakoon config
if not voldrv_config.has_section(storagerouter.machine_id):
raise ValueError("Voldrv arakoon cluster does not have section {0}".format(storagerouter.machine_id))
# Basic SD checks
assert (
storagedriver.cluster_ip == storagerouter.ip
), "Incorrect cluster IP. Expected: {0} - Actual: {1}".format(storagerouter.ip, storagedriver.cluster_ip)
assert storagedriver.mountpoint == "/mnt/{0}".format(
vpool.name
), "Incorrect mountpoint. Expected: {0} - Actual: {1}".format(mountpoint, storagedriver.mountpoint)
if storage_ip is not None:
assert (
storagedriver.storage_ip == storage_ip
), "Incorrect storage IP. Expected: {0} - Actual: {1}".format(storage_ip, storagedriver.storage_ip)
storage_ip = storagedriver.storage_ip
# Check required directories and files
if storagerouter.guid not in all_directories:
raise ValueError("Could not find directory information for Storage Router {0}".format(storagerouter.ip))
if storagerouter.guid not in all_files:
raise ValueError("Could not find file information for Storage Router {0}".format(storagerouter.ip))
for directory in all_directories[storagerouter.guid]:
if root_client.dir_exists(directory) is False:
raise ValueError(
"Directory {0} does not exist on Storage Router {1}".format(directory, storagerouter.ip)
)
for file_name in all_files[storagerouter.guid]:
if root_client.file_exists(file_name) is False:
raise ValueError(
"File {0} does not exist on Storage Router {1}".format(file_name, storagerouter.ip)
)
# @TODO: check roles and sub_roles for all storagedrivers and not just once
for partition in storagedriver.partitions:
if partition.role in sd_partitions and partition.sub_role in sd_partitions[partition.role]:
sd_partitions[partition.role].remove(partition.sub_role)
elif (
partition.role in sd_partitions
and partition.sub_role is None
and len(sd_partitions[partition.role])
):
sd_partitions[partition.role].remove("None")
# Verify vPool writeable
if GeneralHypervisor.get_hypervisor_type() == "VMWARE":
GeneralVPool.mount_vpool(vpool=vpool, root_client=root_client)
vdisk = GeneralVDisk.create_volume(size=10, vpool=vpool, root_client=root_client)
GeneralVDisk.write_to_volume(
vdisk=vdisk, vpool=vpool, root_client=root_client, count=10, bs="1M", input_type="random"
)
GeneralVDisk.delete_volume(vdisk=vdisk, vpool=vpool, root_client=root_client)
for role, sub_roles in sd_partitions.iteritems():
for sub_role in sub_roles:
raise ValueError(
"Not a single Storage Driver found with partition role {0} and sub-role {1}".format(role, sub_role)
)
def ovs_2263_verify_alba_namespace_cleanup_test():
"""
Verify ALBA namespace cleanup
Create an amount of namespaces in ALBA
Create a vPool and create some volumes
Verify the amount of namespaces before and after vPool creation
Remove the vPool and the manually created namespaces
Verify the amount of namespaces before and after vPool deletion
"""
# Create some namespaces in alba
no_namespaces = 3
backend_name = General.get_config().get('backend', 'name')
alba_backend = GeneralAlba.get_by_name(name=backend_name)
namespace_name = 'autotest-ns_'
namespace_name_regex = re.compile('^autotest-ns_\d$')
for nmspc_index in range(no_namespaces):
GeneralAlba.execute_alba_cli_action(alba_backend, 'create-namespace', ['{0}{1}'.format(namespace_name, nmspc_index), 'default'], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces,\
"Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name,
len(result))
# Create a vPool and create volumes on it
vpool, vpool_params = GeneralVPool.add_vpool(vpool_parameters={'preset': GeneralAlba.ONE_DISK_PRESET})
GeneralVPool.validate_vpool_sanity(expected_settings=vpool_params)
root_client = SSHClient(GeneralStorageRouter.get_local_storagerouter(), username='******')
if GeneralHypervisor.get_hypervisor_type() == 'VMWARE':
GeneralVPool.mount_vpool(vpool=vpool,
root_client=root_client)
vdisks = []
for disk_index in range(no_namespaces):
vdisks.append(GeneralVDisk.create_volume(size=10,
vpool=vpool,
root_client=root_client))
result = GeneralAlba.list_alba_namespaces(alba_backend=alba_backend)
assert len(result) == 2 * no_namespaces + 1,\
"Expected {0} namespaces present on the {1} backend, found {2}".format(2 * no_namespaces + 1, backend_name, len(result))
# Remove files and vPool
for vdisk in vdisks:
GeneralVDisk.delete_volume(vdisk=vdisk,
vpool=vpool,
root_client=root_client)
if GeneralHypervisor.get_hypervisor_type() == 'VMWARE':
GeneralVPool.unmount_vpool(vpool=vpool,
root_client=root_client)
GeneralVPool.remove_vpool(vpool)
# Verify amount of namespaces
result = GeneralAlba.list_alba_namespaces(alba_backend=alba_backend,
name=namespace_name_regex)
assert len(result) == no_namespaces,\
"Expected {0} namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name,
len(result))
for namespace in result:
GeneralAlba.execute_alba_cli_action(alba_backend, 'delete-namespace', [namespace['name']], False)
result = GeneralAlba.list_alba_namespaces(alba_backend=alba_backend,
name=namespace_name_regex)
assert len(result) == 0,\
"Expected no namespaces present on the {1} backend, found {2}".format(no_namespaces, backend_name,
len(result))
def run(tests='', output_format=TestRunnerOutputFormat.CONSOLE, output_folder='/var/tmp',
project_name='Open vStorage Engineering', always_die=False, qualitylevel='', existing_plan_id="",
interactive=True):
"""
Run single, multiple or all test cases:
- single: - string: 'gui'
- multiple - list: ['gui', 'sanity']
- all - None value or empty list
output format:
- TESTRAIL requires credentials to testrail
- CONSOLE|XML can be used to run tests locally
:param tests: Tests to execute
:param output_format: Format for output CONSOLE, XML, TESTRAIL
:param output_folder: Folder where results file will be stored
:param project_name: Name of testrail project
:param always_die: Die on 1st test that fails
:param qualitylevel: Quality level of setup where tests are executed
:param existing_plan_id: Plan ID for testrail
:param interactive: Run interactively
:return: None
"""
if str(output_format) not in ['CONSOLE', 'XML', 'TESTRAIL']:
if interactive:
output_format = _check_input(predicate=lambda x: getattr(TestRunnerOutputFormat, x, False),
msg='Enter output format - [CONSOLE / XML / TESTRAIL]')
else:
raise RuntimeError('output_format should be: CONSOLE | XML | TESTRAIL')
output_format = getattr(TestRunnerOutputFormat, str(output_format))
if type(always_die) != bool:
if interactive:
always_die = eval(_check_input(predicate=lambda x: type(eval(x)) == bool,
msg="Only boolean values allowed for always_die param\n" +
"Do you want the tests to stop after error/failure?[True/False]"))
else:
raise RuntimeError('always_die parameter should be a boolean: True|False')
if interactive:
if output_format in (TestRunnerOutputFormat.XML, TestRunnerOutputFormat.TESTRAIL) and not output_folder:
output_folder = _check_input(predicate=lambda x: os.path.exists(x) and os.path.isdir(x),
msg='Incorrect output_folder: {0}'.format(output_folder))
elif not output_folder or not(os.path.exists(output_folder) and os.path.isdir(output_folder)):
raise RuntimeError("Output folder incorrect: {0}".format(output_folder))
version = _get_ovs_version()
# Default arguments. First argument is a dummy as it is stripped within nose.
arguments = ['', '--where', General.TESTS_DIR, '--verbosity', '3']
if always_die is True:
arguments.append('-x')
if output_format in (TestRunnerOutputFormat.XML, TestRunnerOutputFormat.TESTRAIL):
testrail_ip = ''
testrail_key = ''
testrail_title = ''
testrail_project = ''
testrail_description = ''
if not output_folder:
raise AttributeError("No output folder for the {0} result files specified".format(output_format))
if not os.path.exists(output_folder):
raise AttributeError("Given output folder doesn't exist. Please create it first!")
if output_format == TestRunnerOutputFormat.TESTRAIL:
if not TESTRAIL_SERVER:
raise AttributeError("No testrail ip specified")
if not TESTRAIL_KEY:
raise AttributeError("No testrail key specified")
if not project_name:
raise AttributeError("No testrail project name specified")
if not qualitylevel:
raise AttributeError("No quality_level specified")
if not version:
raise AttributeError("No version specified")
testrail_ip = TESTRAIL_SERVER
testrail_key = TESTRAIL_KEY
env_info = _get_env_info()
testrail_title = env_info + "__" + version + "__" + qualitylevel + "__" + GeneralHypervisor.get_hypervisor_type()
testrail_project = project_name
testrail_description = _get_description()
arguments.append('--with-xunit_testrail')
arguments.append('--xunit_file2')
arguments.append('/'.join([output_folder, 'test_results{0}.xml'.format(time.time())]))
arguments.append('--testrail-ip')
arguments.append(testrail_ip)
arguments.append('--testrail-key')
arguments.append(testrail_key)
arguments.append('--project-name')
arguments.append(testrail_project)
arguments.append('--push-name')
arguments.append(testrail_title)
arguments.append('--description')
arguments.append(testrail_description)
arguments.append('--plan-id')
arguments.append(existing_plan_id)
if tests:
if type(tests) != list:
tests = [tests]
tests_to_run = []
for test_spec in tests:
test_spec = test_spec.replace(':', '.')
test_spec_parts = test_spec.split('.')
if len(test_spec_parts) < 2 or test_spec_parts[0] != 'ci' or test_spec_parts[1] != 'tests':
raise ValueError('When specifying a testdirectory, testmodule, testclass or testcase, the name needs to start with "ci.tests"')
if len(test_spec_parts) >= 5 and not test_spec_parts[4].startswith('Test'):
raise ValueError('Expected a class name starting with "Test"')
if len(test_spec_parts) >= 5:
test_spec = '{0}:{1}'.format('.'.join(test_spec_parts[:4]), '.'.join(test_spec_parts[4:]))
tests_to_run.append(test_spec)
arguments.append('--tests')
arguments.append(','.join(tests_to_run))
nose.run(argv=arguments, addplugins=[xunit_testrail.XunitTestrail()])
def _get_description(plan_comment="", durations=""):
"""
Generate description for pushing to Testrail
"""
child_process = subprocess.Popen(args="dmidecode | grep -A 12 'Base Board Information'",
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sysinfo, _ = child_process.communicate()
if child_process.returncode != 0:
sysinfo = "NO MOTHERBOARD INFORMATION FOUND"
child_process = subprocess.Popen(args="cat /etc/lsb-release",
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
osinfo, _ = child_process.communicate()
if child_process.returncode != 0:
osinfo = "NO OS INFORMATION FOUND"
sysinfo += osinfo
child_process = subprocess.Popen(args="lshw -short",
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lshwinfo, _ = child_process.communicate()
if child_process.returncode != 0:
lshwinfo = "NO HARDWARE INFORMATION FOUND"
diskinfo = ''
meminfo = []
cpuinfo = []
for line in lshwinfo.splitlines():
if line.find("disk") >= 0:
if line.find('DVD-ROM') >= 0 or line.find('CD-ROM') >= 0:
continue
l = line.split()
l.pop(2)
l.pop(0)
for part in l:
diskinfo += '%13s' % part
diskinfo += '\n'
elif line.find('System Memory') >= 0 or line.find("System memory") >= 0:
l = line.split()
index = l.index('memory') + 1
for item in l[index:]:
meminfo.append(item)
elif line.find('processor') >= 0 and line.find('CPU [empty]') == -1:
l = line.split()
index = l.index('processor') + 1
cinfo = " ".join(l[index:])
cpuinfo.append(cinfo)
hardware_info = "### " + sysinfo + '\n### Disk Information\n' + diskinfo + '\n### Processor Information\n' + '* ' + '\n* '.join(cpuinfo) + '\n### Memory Information\n' + '* ' + ' '.join(meminfo)
description = ""
node_ips = ""
for ip in GeneralStorageRouter.get_all_ips():
node_ips += "* " + ip + "\n"
for item, value in (("ip", "%s" % node_ips),
("testsuite", durations),
("Hypervisor", GeneralHypervisor.get_hypervisor_type()),
("hardware", hardware_info),
("package", _get_package_info()),
("Comment ", ('*' * 40 + "\n" + plan_comment) if plan_comment else '')):
description += "# %s INFO \n%s\n" % (item.upper(), value)
return description
def push_to_testrail(project_name, output_folder, version=None, filename="", milestone="", comment=""):
"""
Push xml file with test results to Testrail
:param project_name: Name of testrail project
:param output_folder: Output folder containing the results in XML format
:param version: Version
:param filename: File name of results
:param milestone: Milestone in testrail
:param comment: Extra comment
:return: Testrail URL
"""
def _get_textual_values(seconds):
if not seconds:
return "%2i %5s, %2i %7s, %2i %7s\n" % (0, 'hours', 0, 'minutes', 0, 'seconds')
hours = seconds / 60 / 60
rest = seconds % (60 * 60)
minutes = rest / 60
rest %= 60
hours_text = 'hour' if hours == 1 else 'hours'
minutes_text = 'minute' if minutes == 1 else 'minutes'
seconds_text = 'second' if rest == 1 else 'seconds'
return "%2i %5s, %2i %7s, %2i %7s\n" % (hours, hours_text, minutes, minutes_text, rest, seconds_text)
if not filename:
if not (os.path.exists(output_folder) and os.path.isdir(output_folder)):
output_folder = _check_input(predicate=(os.path.exists(output_folder) and os.path.isdir(output_folder)),
msg='Incorrect output_folder: {0}'.format(output_folder))
result_files = [f for f in os.listdir(output_folder) if ".xml" in f]
result_files.sort(reverse=True)
if not result_files:
print "\nNo test_results files were found in {0}".format(output_folder)
return
files_to_ask_range = list(range(len(result_files)))
files_to_ask = zip(files_to_ask_range, result_files)
filename_index = eval(_check_input(predicate=lambda x: eval(x) in files_to_ask_range,
msg="Please choose results file \n" + "\n".join(
map(lambda x: str(x[0]) + "->" + str(x[1]), files_to_ask)) + ":\n"))
filename = '/'.join([output_folder, result_files[filename_index]])
if not version:
version = _get_ovs_version()
test_result_file = filename
if not os.path.exists(test_result_file):
raise Exception("Test result file {0} was not found on system".format(test_result_file))
if not os.path.isfile(test_result_file):
raise Exception("{0} is not a valid file".format(test_result_file))
testrail_api = testrailapi.TestrailApi(TESTRAIL_SERVER, key=TESTRAIL_KEY)
project = testrail_api.get_project_by_name(project_name)
project_id = project['id']
milestone_id = None
if milestone:
milestone = testrail_api.get_milestone_by_name(project_id, milestone)
milestone_id = milestone['id']
today = datetime.datetime.today()
date = today.strftime('%a %b %d %H:%M:%S')
env_info = _get_env_info()
name = '_'.join([env_info, version, date])
project_mapping_file = '/'.join([General.CONFIG_DIR, "project_testsuite_mapping.cfg"])
project_map = ConfigParser.ConfigParser()
project_map.read(project_mapping_file)
if not project_map.has_section(project_name):
raise Exception(
"Config file {0} does not contain section for specified project {1}".format(project_mapping_file,
project_name))
error_messages = []
xmlfile = minidom.parse(test_result_file).childNodes[0]
duration_suite_map = {}
for child in xmlfile.childNodes:
suite = child.getAttribute('classname').split('.')[-2]
if suite == '<nose' or suite.startswith('ContextSuite'):
continue
if suite not in duration_suite_map:
duration_suite_map[suite] = 0
duration_suite_map[suite] += int(child.getAttribute('time'))
durations = ''
for key in sorted(duration_suite_map.keys()):
durations += "%30s: " % key + _get_textual_values(duration_suite_map[key])
durations += '\n%30s: ' % 'Total Duration' + _get_textual_values(sum(duration_suite_map.values()))
added_suites = []
plan_id = None
suite_to_run = {}
case_name_to_test_id = {}
# Retrieve test cases from xml file
all_cases = {}
ran_cases = {}
suite_name = ''
suite_id = ''
suite_name_to_id = {}
section_name_to_id = {}
all_sections = {}
for child in xmlfile.childNodes:
classname = child.getAttribute('classname')
if classname.startswith(('<nose', 'nose', '<nose')):
continue
option = classname.split('.')[-3]
if child.childNodes and child.childNodes[0].getAttribute('type') in ['nose.plugins.skip.SkipTest', 'unittest.case.SkipTest'] and \
child.childNodes[0].getAttribute('message') != BLOCKED_MESSAGE:
continue
case_name = child.getAttribute('name')
match = re.search("c\d+_(.+)", case_name)
case_name = match.groups()[0] if match else case_name
previous_suite_name = suite_name
suite_name = project_map.get(project_name, option).split(';')[0]
section_names = project_map.get(project_name, option).split(';')[1].split(',')
section_name = section_names[-1]
if suite_name != previous_suite_name:
suite = testrail_api.get_suite_by_name(project_id, suite_name)
suite_id = suite['id']
suite_name_to_id[suite_name] = suite_id
all_cases[suite_name] = testrail_api.get_cases(project_id, suite['id'])
section = testrail_api.get_section_by_name(project_id, suite['id'], section_name)
if suite['id'] not in all_sections:
all_sections[suite['id']] = testrail_api.get_sections(project_id, suite['id'])
section_id = section['id']
if suite_id in section_name_to_id:
section_name_to_id[suite_id][section_name] = section_id
else:
section_name_to_id[suite_id] = {section_name: section_id}
case_id = [case for case in all_cases[suite_name] if case['section_id'] == section_id and case['title'] == case_name]
if not case_id:
new_case = testrail_api.add_case(section_id=section_id, title=case_name)
all_cases[suite_name].append(new_case)
ran_cases[suite_name] = ran_cases[suite_name].add(case_name) or ran_cases[suite_name] if ran_cases.get(suite_name) else {case_name}
testcase_ids_to_select = []
for child in xmlfile.childNodes:
classname = child.getAttribute('classname')
if classname.startswith(('<nose', 'nose')):
if child.childNodes[0].childNodes and child.childNodes[0].childNodes[0].nodeType == minidom.DocumentType.CDATA_SECTION_NODE:
error_messages.append(child.childNodes[0].childNodes[0].data)
else:
error_messages.append(child.childNodes[0].getAttribute('message'))
continue
suite = classname.split('.')[-3]
is_blocked = False
if child.childNodes and "SkipTest" in child.childNodes[0].getAttribute('type'):
if child.childNodes[0].getAttribute('message') == BLOCKED_MESSAGE:
is_blocked = True
if not project_map.has_option(project_name, suite):
raise Exception("Testsuite '%s' is not configured for project '%s' in '%s'" % (suite, project_name,
project_mapping_file))
full_name = project_map.get(project_name, suite)
suite_name = full_name.split(';')[0]
create_run = False
if suite_name not in added_suites:
create_run = True
if suite_name not in suite_name_to_id:
continue
added_suites.append(suite_name)
testcase_ids_to_select = [case['id'] for case in all_cases[suite_name] if case['title'] in ran_cases[suite_name]]
case_name = child.getAttribute('name')
if plan_id is None:
create_run = False
description = _get_description(plan_comment=comment, durations=durations)
plan_id = testrail_api.add_plan(project_id, name, description, milestone_id or None)['id']
entry = testrail_api.add_plan_entry(plan_id, suite_name_to_id[suite_name], suite_name, include_all=False,
case_ids=testcase_ids_to_select)
run_id = entry['runs'][0]['id']
suite_to_run[suite_name] = run_id
if create_run:
entry = testrail_api.add_plan_entry(plan_id, suite_name_to_id[suite_name], suite_name, include_all=False,
case_ids=testcase_ids_to_select)
run_id = entry['runs'][0]['id']
suite_to_run[suite_name] = run_id
run_id = suite_to_run[suite_name]
if case_name not in case_name_to_test_id:
all_tests_for_run = testrail_api.get_tests(run_id=run_id)
for test in all_tests_for_run:
case_name_to_test_id[test['title'] + str(run_id)] = test['id']
test_id = case_name_to_test_id[case_name + str(run_id)]
comment = ''
if not child.childNodes:
status_id = TESTRAIL_STATUS_ID_PASSED
elif child.childNodes[0].getAttribute('type') in ['nose.plugins.skip.SkipTest', 'unittest.case.SkipTest']:
if is_blocked:
status_id = TESTRAIL_STATUS_ID_BLOCKED
else:
status_id = TESTRAIL_STATUS_ID_SKIPPED
if child.childNodes[0].childNodes and \
child.childNodes[0].childNodes[0].nodeType == minidom.DocumentType.CDATA_SECTION_NODE:
comment = child.childNodes[0].childNodes[0].data
else:
status_id = TESTRAIL_STATUS_ID_FAILED
if child.childNodes[0].childNodes and \
child.childNodes[0].childNodes[0].nodeType == minidom.DocumentType.CDATA_SECTION_NODE:
comment = child.childNodes[0].childNodes[0].data
else:
comment = child.childNodes[0].getAttribute('message')
elapsed = int(child.getAttribute('time'))
if elapsed == 0:
elapsed = 1
testrail_api.add_result(test_id=test_id, status_id=status_id, comment=comment, version=version,
elapsed='%ss' % elapsed, custom_fields={'custom_hypervisor': GeneralHypervisor.get_hypervisor_type()})
xmlfile.unlink()
del xmlfile
if error_messages:
print "\nSome testsuites failed to start because an error occurred during setup:\n{0}".format(error_messages)
url = None
if plan_id:
url = "http://%s/index.php?/plans/view/%s" % (TESTRAIL_SERVER, plan_id)
print "\n" + url
return url
