def test_import_storage_profile(self, setup_local_storage): """ Args: setup_local_storage(str): type of local-storage backing, allowed values: image, remote Setup: Test Steps: 1 check if the storage-profile exists, skip if not 2 get list of profile names current existing in the system 3 import the storage-profile 4 check the storage-profile actually created matching with those in the profile: 1) profilename 2) disk configuration 3) physical volume config 4) local volume group config Teardown: Notes: will cover 1 test cases: 39. Local Storage Profile Import """ local_storage_type, compute_src = setup_local_storage if not system_helper.is_storage_system(): skip("This test requires a storage system") LOG.tc_step('Get the name of the profile and check if it is existing') local_file = self.get_local_storprfoile_file( local_storage_type=local_storage_type) if not local_file: msg = 'Cannot find the profile:{}'.format(local_file) skip(msg) LOG.tc_step('Get list of profile names') pre_import_storprofiles = get_storprofiles() LOG.tc_step( 'Apply the storage-profile via CLI profile-import {}'.format( local_file)) remote_file = self.get_remote_storprofile_file( local_storage_type=local_storage_type) common.scp_from_localhost_to_active_controller(local_file, remote_file) rtn_code, output = self.import_storprofile_profile( profile_file=remote_file) assert 0 == rtn_code, 'Failed to import storage-profile'.format( remote_file) LOG.tc_step( 'Check if the storage profile are correctly imported into the system' ) assert 0 == self.verify_storage_profile_imported( profile=local_file, msg_import=output, pre_import_storprofiles=pre_import_storprofiles)
def test_lock_unlock_host(host_type): """ Verify lock unlock host Test Steps: - Select a host per given type. If type is controller, select standby controller. - Lock selected host and ensure it is successfully locked - Unlock selected host and ensure it is successfully unlocked """ LOG.tc_step("Select a {} node from system if any".format(host_type)) if host_type == 'controller': if system_helper.is_aio_simplex(): host = 'controller-0' else: host = system_helper.get_standby_controller_name() assert host, "No standby controller available" else: if host_type == 'compute' and system_helper.is_aio_system(): skip("No compute host on AIO system") elif host_type == 'storage' and not system_helper.is_storage_system(): skip("System does not have storage nodes") hosts = system_helper.get_hosts(personality=host_type, availability=HostAvailState.AVAILABLE, operational=HostOperState.ENABLED) assert hosts, "No good {} host on system".format(host_type) host = hosts[0] LOG.tc_step("Lock {} host - {} and ensure it is successfully " "locked".format(host_type, host)) HostsToRecover.add(host) host_helper.lock_host(host, swact=False) # wait for services to stabilize before unlocking time.sleep(20) # unlock standby controller node and verify controller node is # successfully unlocked LOG.tc_step("Unlock {} host - {} and ensure it is successfully " "unlocked".format(host_type, host)) host_helper.unlock_host(host)
def cleanup(): if not system_helper.is_storage_system(): skip("This test requires a storage system") profiles_created = self._pop_cleanup_list('profile') old_new_types = self._pop_cleanup_list('local_storage_type') # Add hosts to module level recovery fixture in case of modify or unlock fail in following class level # recovery attempt. for item in old_new_types: HostsToRecover.add(item[0], scope='module') exceptions = [] try: LOG.fixture_step("(class) Delete created storage profiles") while profiles_created: storage_helper.delete_storage_profile( profile=profiles_created.pop()) except Exception as e: LOG.exception(e) exceptions.append(e) try: LOG.fixture_step( "(class) Revert local storage backing for {}".format( old_new_types)) while old_new_types: host_to_revert, old_type, _ = old_new_types.pop() LOG.info("Revert {} local storage to {}".format( host_to_revert, old_type)) host_helper.set_host_storage_backing(host=host_to_revert, inst_backing=old_type, unlock=True) except Exception as e: LOG.exception(e) exceptions.append(e) assert not exceptions, "Failure occurred. Errors: {}".format( exceptions)
def test_patch_orch_strategy(patch_orchestration_setup, storage_apply_type, patch_function_check, compute_apply_type, max_parallel_computes, instance_action, test_patch): """ This test verifies the patch orchestration strategy options Args: patch_orchestration_setup: patch_function_check storage_apply_type: compute_apply_type: max_parallel_computes: instance_action: test_patch: Returns: """ instance_action = instance_action.replace('_', '-') vms = patch_function_check patches, controllers, computes, storages = patch_orchestration_setup if 'STORAGE' in test_patch and not system_helper.is_storage_system(): skip('Skip STORAGE patch test for non-storage system') if "parallel"in storage_apply_type and len(storages) < 4: skip("At least two pairs tier storage nodes required for this test: {}".format(storages)) if "parallel"in compute_apply_type and len(computes) < (max_parallel_computes + 1): skip("At least {} computes are required for this test".format(1+max_parallel_computes)) patch_id = patching_helper.parse_test_patches(patch_ids=patches, search_str=test_patch)[0] patch_file = patches[patch_id] LOG.tc_step("Upload patch file {}".format(patch_file)) uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0] assert patch_id == uploaded_id, "Expected patch {} and uploaded patch {} mismatch"\ .format(patch_id, uploaded_id) LOG.info("Patch {} uploaded".format(uploaded_id)) LOG.tc_step("Apply patch {}".format(patch_id)) applied = patching_helper.apply_patches(patch_ids=[patch_id])[1] assert applied == [patch_id] LOG.info("Patch {} applied".format(patch_id)) LOG.tc_step("Install patches through orchestration for patch {}".format(applied)) patching_helper.wait_for_affecting_alarms_gone() run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type, max_parallel_computes=max_parallel_computes, instance_action=instance_action, alarm_restrictions='relaxed') LOG.info("Install patch through orchestration completed for patch {}".format(applied)) time.sleep(20) LOG.tc_step("Check vms after patch applied: {}".format(applied)) check_vms(vms) LOG.tc_step("Remove test patch {}".format(applied)) patching_helper.remove_patches(patch_ids=applied) partial_remove_ids = get_test_patches(state=PatchState.PARTIAL_REMOVE) assert all(patch in partial_remove_ids for patch in applied), \ "Expected patch {} not in partial-remove state".format(applied) LOG.tc_step("Remove patch through orchestration: {}".format(applied)) run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type, max_parallel_computes=max_parallel_computes, instance_action=instance_action, alarm_restrictions='relaxed') LOG.info("Remove patch through orchestration completed for patch {}".format(applied)) LOG.tc_step("Check vms after patch removed: {}".format(applied)) check_vms(vms)
def _test_storage_profile(personality, from_backing, to_backing): """ This test creates a storage profile and then applies it to a node with identical hardware, assuming one exists. Storage profiles do not apply on controller nodes. Storage profiles can be applied on controller+compute nodes, compute nodes and storage nodes. Arguments: - personality (string) - controller, compute or storage - from_backing (string) - image, remote or None - to_backing (string) - image, remote or None Test Steps: 1. Query system and determine which nodes have compatible hardware. 2. Create a storage profile on one of those nodes 3. Apply the created storage profile on a compatible node* 4. Ensure the storage profiles have been successfully applied. * If the node is a compute node or a controller+compute, we will also change the backend if required for additional coverage. Returns: - Nothing """ global PROFILES_TO_DELETE PROFILES_TO_DELETE = [] # Skip if test is not applicable to hardware under test if personality == 'controller' and not system_helper.is_aio_system(): skip("Test does not apply to controller hosts without subtype compute") hosts = system_helper.get_hosts(personality=personality) if not hosts: skip("No hosts of type {} available".format(personality)) if (from_backing == "remote" or to_backing == "remote") and not system_helper.is_storage_system(): skip("This test doesn't apply to systems without storage hosts") LOG.tc_step("Identify hardware compatible hosts") hash_to_hosts = get_hw_compatible_hosts(hosts) # Pick the hardware group that has the most compatible hosts current_size = 0 candidate_hosts = [] for value in hash_to_hosts: candidate_size = len(hash_to_hosts[value]) if candidate_size > current_size: current_size = candidate_size candidate_hosts = hash_to_hosts[value] LOG.info( "This is the total set of candidate hosts: {}".format(candidate_hosts)) if len(candidate_hosts) < 2: skip("Insufficient hardware compatible hosts to run test") # Rsync lab setup dot files between controllers con_ssh = ControllerClient.get_active_controller() _rsync_files_to_con1(con_ssh=con_ssh, file_to_check="force.txt") # Take the hardware compatible hosts and check if any of them already have # the backend that we want. This will save us test time. new_to_backing = None if personality == "compute": from_hosts = [] to_hosts = [] for host in candidate_hosts: host_backing = host_helper.get_host_instance_backing(host) if host_backing == from_backing: from_hosts.append(host) elif host_backing == to_backing: to_hosts.append(host) else: pass LOG.info( "Candidate hosts that already have the right from backing {}: {}". format(from_backing, from_hosts)) LOG.info( "Candidate hosts that already have the right to backing {}: {}". format(to_backing, to_hosts)) # Determine what hosts to use if not from_hosts and to_hosts: to_host = random.choice(to_hosts) candidate_hosts.remove(to_host) from_host = random.choice(candidate_hosts) elif not to_hosts and from_hosts: from_host = random.choice(from_hosts) candidate_hosts.remove(from_host) to_host = random.choice(candidate_hosts) elif not to_hosts and not from_hosts: to_host = random.choice(candidate_hosts) candidate_hosts.remove(to_host) from_host = random.choice(candidate_hosts) else: to_host = random.choice(to_hosts) from_host = random.choice(from_hosts) LOG.info("From host is: {}".format(from_host)) LOG.info("To host is: {}".format(to_host)) LOG.tc_step( "Check from host backing and convert to {} if necessary".format( from_backing)) host_helper.set_host_storage_backing(from_host, from_backing) system_helper.wait_for_host_values( from_host, availability=HostAvailState.AVAILABLE, timeout=120, fail_ok=False) LOG.tc_step( "Check to host backing and convert to {} if necessary".format( to_backing)) new_to_backing = host_helper.set_host_storage_backing( to_host, to_backing) elif personality == "controller": # For now, we don't want to host reinstall controller-0 since it will default to # pxeboot, but this could be examined as a possible enhancement. from_host = "controller-0" to_host = "controller-1" LOG.info("From host is: {}".format(from_host)) LOG.info("To host is: {}".format(to_host)) LOG.tc_step( "Check from host backing and convert to {} if necessary".format( from_backing)) host_helper.set_host_storage_backing(from_host, from_backing) LOG.tc_step( "Check to host backing and convert to {} if necessary".format( to_backing)) new_to_backing = host_helper.set_host_storage_backing( to_host, to_backing) else: # Backing doesn't apply to storage nodes so just pick from compatible hardware from_host = random.choice(candidate_hosts) candidate_hosts.remove(from_host) to_host = random.choice(candidate_hosts) LOG.tc_step( "Create storage and interface profiles on the from host {}".format( from_host)) prof_name = 'storprof_{}_{}'.format( from_host, time.strftime('%Y%m%d_%H%M%S', time.localtime())) storage_helper.create_storage_profile(from_host, profile_name=prof_name) PROFILES_TO_DELETE.append(prof_name) # Deleting VMs in case the remaining host(s) cannot handle all VMs # migrating on lock, particularly important in the case of AIO-DX systems. LOG.tc_step( "Delete all VMs and lock the host before applying the storage profile") vm_helper.delete_vms() HostsToRecover.add(to_host, scope='function') system_helper.wait_for_host_values(from_host, availability=HostAvailState.AVAILABLE, timeout=120, fail_ok=False) system_helper.wait_for_host_values(to_host, availability=HostAvailState.AVAILABLE, timeout=120, fail_ok=False) # Negative test #1 - attempt to apply profile on unlocked host (should be rejected) LOG.tc_step('Apply the storage-profile {} onto unlocked host:{}'.format( prof_name, to_host)) cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name) rc, msg = cli.system(cmd, fail_ok=True) assert rc != 0, msg host_helper.lock_host(to_host, swact=True) # 3 conditions to watch for: no partitions, ready partitions and in-use # partitions on the compute. If in-use, delete and freshly install host. # If ready, delete all ready partitions to make room for potentially new # partitions. If no partitions, just delete nova-local lvg. if personality == "compute": # Negative test #2 - attempt to apply profile onto host with existing # nova-local (should be rejected) LOG.tc_step( 'Apply the storage-profile {} onto host with existing nova-local:{}' .format(prof_name, to_host)) cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name) rc, msg = cli.system(cmd, fail_ok=True) assert rc != 0, msg # If we were simply switching backing (without applying a storage # profile), the nova-local lvg deletion can be omitted according to design LOG.tc_step("Delete nova-local lvg on to host {}".format(to_host)) cli.system("host-lvg-delete {} nova-local".format(to_host)) in_use = storage_helper.get_host_partitions(to_host, "In-Use") if in_use: # Negative test #3 - attempt to apply profile onto host with existing # in-use partitions (should be rejected) LOG.tc_step('Apply the storage-profile {} onto host with existing \ in-use partitions:{}'.format(prof_name, to_host)) cmd = 'host-apply-storprofile {} {}'.format(to_host, prof_name) rc, msg = cli.system(cmd, fail_ok=True) assert rc != 0, msg LOG.tc_step( "In-use partitions found. Must delete the host and freshly install before proceeding." ) LOG.info("Host {} has in-use partitions {}".format( to_host, in_use)) lab = InstallVars.get_install_var("LAB") lab.update(create_node_dict(lab['compute_nodes'], 'compute')) lab['boot_device_dict'] = create_node_boot_dict(lab['name']) install_helper.open_vlm_console_thread(to_host) LOG.tc_step("Delete the host {}".format(to_host)) cli.system("host-bulk-export") cli.system("host-delete {}".format(to_host)) assert len( system_helper.get_controllers()) > 1, "Host deletion failed" cli.system("host-bulk-add hosts.xml") system_helper.wait_for_host_values( to_host, timeout=6000, availability=HostAvailState.ONLINE) wait_for_disks(to_host) ready = storage_helper.get_host_partitions(to_host, "Ready") if ready: LOG.tc_step( "Ready partitions have been found. Must delete them before profile application" ) LOG.info("Host {} has Ready partitions {}".format(to_host, ready)) for uuid in reversed(ready): storage_helper.delete_host_partition(to_host, uuid) # Don't bother restoring in this case since the system should be # functional after profile is applied. LOG.tc_step('Apply the storage-profile {} onto host:{}'.format( prof_name, to_host)) cli.system('host-apply-storprofile {} {}'.format(to_host, prof_name)) LOG.tc_step("Unlock to host") host_helper.unlock_host(to_host) to_host_backing = host_helper.get_host_instance_backing(to_host) LOG.info("To host backing was {} and is now {}".format( new_to_backing, to_host_backing)) assert to_host_backing == from_backing, "Host backing was not changed on storage profile application" if personality == "storage": if not storage_helper.is_ceph_healthy(): skip("Cannot run test when ceph is not healthy") LOG.tc_step("Delete the host {}".format(to_host)) cli.system("host-bulk-export") cli.system("host-delete {}".format(to_host)) cli.system("host-bulk-add hosts.xml") system_helper.wait_for_host_values(to_host, timeout=6000, availability=HostAvailState.ONLINE) wait_for_disks(to_host) LOG.tc_step('Apply the storage-profile {} onto host:{}'.format( prof_name, to_host)) cli.system('host-apply-storprofile {} {}'.format(to_host, prof_name)) # Re-provision interfaces through lab_setup.sh LOG.tc_step("Reprovision the host as necessary") files = ['interfaces'] con_ssh = ControllerClient.get_active_controller() delete_lab_setup_files(con_ssh, to_host, files) rc, msg = install_helper.run_lab_setup() assert rc == 0, msg LOG.tc_step("Unlock to host") host_helper.unlock_host(to_host) if personality == "controller": # Note, install helper doesn't work on all labs. Some labs don't # display BIOS type which causes install helper to fail lab = InstallVars.get_install_var("LAB") lab.update(create_node_dict(lab['controller_nodes'], 'controller')) lab['boot_device_dict'] = create_node_boot_dict(lab['name']) install_helper.open_vlm_console_thread(to_host) LOG.tc_step("Delete the host {}".format(to_host)) cli.system("host-bulk-export") cli.system("host-delete {}".format(to_host)) assert len(system_helper.get_controllers()) > 1, "Host deletion failed" cli.system("host-bulk-add hosts.xml") system_helper.wait_for_host_values(to_host, timeout=6000, availability=HostAvailState.ONLINE) wait_for_disks(to_host) LOG.tc_step("Apply the storage-profile {} onto host:{}".format( prof_name, to_host)) cli.system("host-apply-storprofile {} {}".format(to_host, prof_name)) # Need to re-provision everything on node through lab_setup (except storage) LOG.tc_step("Reprovision the host as necessary") files = [ 'interfaces', 'cinder_device', 'vswitch_cpus', 'shared_cpus', 'extend_cgts_vg', 'addresses' ] con_ssh = ControllerClient.get_active_controller() delete_lab_setup_files(con_ssh, to_host, files) rc, msg = install_helper.run_lab_setup() assert rc == 0, msg LOG.tc_step("Unlock to host") host_helper.unlock_host(to_host) to_host_backing = host_helper.get_host_instance_backing(to_host) LOG.info("To host backing was {} and is now {}".format( new_to_backing, to_host_backing)) assert to_host_backing == from_backing, "Host backing was not changed on storage profile application"
def test_lock_unlock_host(host_type, collect_kpi): """ Verify lock unlock host Test Steps: - Select a host per given type. If type is controller, select standby controller. - Lock selected host and ensure it is successfully locked - Unlock selected host and ensure it is successfully unlocked """ init_time = None if collect_kpi: init_time = common.get_date_in_format(date_format=KPI_DATE_FORMAT) LOG.tc_step("Select a {} node from system if any".format(host_type)) if host_type == 'controller': if system_helper.is_aio_simplex(): host = 'controller-0' else: host = system_helper.get_standby_controller_name() assert host, "No standby controller available" else: if host_type == 'compute' and (system_helper.is_aio_duplex() or system_helper.is_aio_simplex()): skip("No compute host on AIO system") elif host_type == 'storage' and not system_helper.is_storage_system(): skip("System does not have storage nodes") hosts = system_helper.get_hosts(personality=host_type, availability=HostAvailState.AVAILABLE, operational=HostOperState.ENABLED) assert hosts, "No good {} host on system".format(host_type) host = hosts[0] LOG.tc_step( "Lock {} host - {} and ensure it is successfully locked".format( host_type, host)) HostsToRecover.add(host) host_helper.lock_host(host, swact=False) # wait for services to stabilize before unlocking time.sleep(20) # unlock standby controller node and verify controller node is successfully unlocked LOG.tc_step( "Unlock {} host - {} and ensure it is successfully unlocked".format( host_type, host)) host_helper.unlock_host(host) LOG.tc_step("Check helm list after host unlocked") con_ssh = ControllerClient.get_active_controller() con_ssh.exec_cmd('helm list', fail_ok=False) if collect_kpi: lock_kpi_name = HostLock.NAME.format(host_type) unlock_kpi_name = HostUnlock.NAME.format(host_type) unlock_host_type = host_type if container_helper.is_stx_openstack_deployed(): if system_helper.is_aio_system(): unlock_host_type = 'compute' else: lock_kpi_name += '_platform' unlock_kpi_name += '_platform' if unlock_host_type == 'compute': unlock_host_type = 'compute_platform' LOG.info("Collect kpi for lock/unlock {}".format(host_type)) code_lock, out_lock = kpi_log_parser.record_kpi( local_kpi_file=collect_kpi, kpi_name=lock_kpi_name, host=None, log_path=HostLock.LOG_PATH, end_pattern=HostLock.END.format(host), start_pattern=HostLock.START.format(host), start_path=HostLock.START_PATH, init_time=init_time) time.sleep(30) # delay in sysinv log vs nova hypervisor list code_unlock, out_unlock = kpi_log_parser.record_kpi( local_kpi_file=collect_kpi, kpi_name=unlock_kpi_name, host=None, log_path=HostUnlock.LOG_PATH, end_pattern=HostUnlock.END[unlock_host_type].format(host), init_time=init_time, start_pattern=HostUnlock.START.format(host), start_path=HostUnlock.START_PATH) assert code_lock == 0, 'Failed to collect kpi for host-lock {}. ' \ 'Error: \n'.format(host, out_lock) assert code_unlock == 0, 'Failed to collect kpi for host-unlock {}. ' \ 'Error: \n'.format(host, out_lock)