def unset_flavor(flavor,
properties=None,
project=None,
project_domain=None,
check_first=True,
fail_ok=False,
auth_info=Tenant.get('admin'),
con_ssh=None):
"""
Unset specific extra spec(s) from given flavor.
Args:
flavor (str): id of the flavor
properties (str|list|tuple): extra spec(s) to be removed. At least one should be provided.
project_domain
project
check_first (bool): Whether to check if extra spec exists in flavor before attempt to unset
con_ssh (SSHClient):
auth_info (dict):
fail_ok (bool):
con_ssh
Returns (tuple): (rtn_code (int), message (str))
(-1, 'Extra spec(s) <specs> not exist in flavor. Do nothing.')
(0, 'Flavor extra specs unset successfully.'): required extra spec(s) removed successfully
(1, <stderr>): unset extra spec cli rejected
(2, '<spec_name> is still in the extra specs list'): post action check failed
"""
if isinstance(properties, str):
properties = [properties]
if properties and check_first:
existing_specs = get_flavor_values(flavor,
fields='properties',
con_ssh=con_ssh,
auth_info=auth_info)[0]
properties = list(set(properties) & set(existing_specs.keys()))
args_dict = {
'--property': properties,
'--project': project,
'--project_domain': project_domain,
}
args = common.parse_args(args_dict, repeat_arg=True)
if not args:
msg = "Nothing to unset for flavor {}. Do nothing.".format(flavor)
LOG.info(msg)
return -1, msg
LOG.info("Unsetting flavor {} with args: {}".format(flavor, args))
exit_code, output = cli.openstack('flavor unset',
args,
ssh_client=con_ssh,
fail_ok=fail_ok,
auth_info=auth_info)
if exit_code > 0:
return 1, output
success_msg = "Flavor {} unset successfully".format(flavor)
LOG.info(success_msg)
return 0, success_msg
def test_modify_timezone_log_timestamps():
"""
Test correct log timestamps after timezone change
Prerequisites
- N/A
Test Setups
- Get a random timezone for testing
- Get system time (epoch) before timezone change
Test Steps
- Modify timezone
- While to modification is pending get the last timestamp (epoch) from each log
- Wait for out_of_date alarms to clear
- Get system time (epoch) after timezone change
- Ensure the timezone change effected the date by comparing system time before and after timezone change
- For 3 minutes check each log for a new entry
- Ensure that the new entry in each log is in line with the timezone change
Test Teardown
- N/A
"""
logs = ('auth.log', 'daemon.log', 'fm-event.log', 'fsmond.log', 'kern.log', 'openstack.log',
'pmond.log', 'user.log')
# 'sm-scheduler.log' fails (CGTS-10475)
LOG.tc_step("Collect timezone, system time, and log timestamps before modify timezone")
LOG.info("Get timezones for testing")
prev_timezone = system_helper.get_timezone()
post_timezone = __select_diff_timezone(current_zone=prev_timezone)
con_ssh = ControllerClient.get_active_controller()
# Saving the last entry from the logs; If it is the same later, there is no new entry in the log.
LOG.tc_step("Get last entry from log files and compare with system time")
prev_timestamps = {}
time.sleep(5)
for log in logs:
last_line = con_ssh.exec_cmd('tail -n 1 /var/log/{}'.format(log))[1]
log_epoch = parse_log_time(last_line)
prev_timestamps[log] = log_epoch
prev_system_epoch = get_epoch_date(con_ssh=con_ssh)
prev_check_fail = []
for log, timestamp in prev_timestamps.items():
if timestamp > prev_system_epoch:
prev_check_fail.append(log)
assert not prev_check_fail, "{} timestamp does not match system time".format(prev_check_fail)
start_time = time.time()
LOG.tc_step("Modify timezone from {} to {}".format(prev_timezone, post_timezone))
system_helper.modify_timezone(timezone=post_timezone)
end_time = time.time()
mod_diff = end_time - start_time
LOG.tc_step("Verify system time is updated")
time.sleep(10)
post_system_epoch = get_epoch_date(con_ssh=con_ssh)
time_diff = abs(prev_system_epoch - post_system_epoch)
assert time_diff > 3600, "Timezone change did not affect the date"
LOG.info("system time is updated after timezone modify")
LOG.tc_step("Wait for new logs to generate and verify timestamps for new log entries are updated")
logs_to_test = {}
timeout = time.time() + 300
while time.time() < timeout:
for log_name, value in prev_timestamps.items():
last_line = con_ssh.exec_cmd('tail -n 1 /var/log/{}'.format(log_name))[1]
# If last line does not exist in last_log_entries; New line is in the log; Add log to logs_to_test
new_log_epoch = parse_log_time(last_line)
prev_log_epoch = prev_timestamps[log_name]
epoch_diff_prev_log_to_sys = prev_system_epoch - prev_log_epoch
epoch_diff_prev_log = new_log_epoch - prev_log_epoch
LOG.info('timezone modify time used: {}; pre-modify sys time - last log time: {}'.
format(mod_diff, epoch_diff_prev_log_to_sys))
if abs(epoch_diff_prev_log) > max(mod_diff, 180) + epoch_diff_prev_log_to_sys:
LOG.info("{} has new log entry. Adding to logs_to_test.".format(log_name))
logs_to_test[log_name] = new_log_epoch
del prev_timestamps[log_name]
break
if not prev_timestamps:
break
time.sleep(10)
# Get latest log entries, convert to epoch
failed_logs = {}
for log_name, value in logs_to_test.items():
time_diff = abs(value - post_system_epoch)
LOG.info("log: {} time diff: {}".format(log_name, time_diff))
if time_diff > 330:
failed_logs[log_name] = "time_diff: {}".format(time_diff)
assert not failed_logs, "Timestamp for following new logs are different than system time: {}".format(failed_logs)
def test_set_hosts_storage_backing_equal(instance_backing, number_of_hosts):
"""
Modify hosts storage backing if needed so that system has exact number of hosts in given instance backing
Args:
instance_backing:
number_of_hosts:
Test Steps:
- Calculate the hosts to be configured based on test params
- Configure hosts to meet given criteria
- Check number of hosts in given instance backing is as specified
"""
host_num_mapping = {'zero': 0, 'one': 1, 'two': 2}
number_of_hosts = host_num_mapping[number_of_hosts]
LOG.tc_step("Calculate the hosts to be configured based on test params")
candidate_hosts = get_candidate_hosts(number_of_hosts=number_of_hosts)
hosts_with_backing = host_helper.get_hosts_in_storage_backing(
instance_backing)
if len(hosts_with_backing) == number_of_hosts:
LOG.info("Already have {} hosts in {} backing. Do nothing".format(
number_of_hosts, instance_backing))
return
elif len(hosts_with_backing) < number_of_hosts:
backing_to_config = instance_backing
number_to_config = number_of_hosts - len(hosts_with_backing)
hosts_pool = list(set(candidate_hosts) - set(hosts_with_backing))
else:
backing_to_config = 'remote' if 'image' in instance_backing else 'local_image'
number_to_config = len(hosts_with_backing) - number_of_hosts
hosts_pool = hosts_with_backing
LOG.tc_step(
"Delete vms if any to prepare for system configuration change with best effort"
)
vm_helper.delete_vms(fail_ok=True)
hosts_to_config = hosts_pool[0:number_to_config]
LOG.tc_step("Configure following hosts to {} backing: {}".format(
hosts_to_config, backing_to_config))
for host in hosts_to_config:
host_helper.set_host_storage_backing(host=host,
inst_backing=backing_to_config,
unlock=False,
wait_for_configured=False)
HostsToRecover.add(host)
host_helper.unlock_hosts(hosts_to_config,
check_hypervisor_up=True,
fail_ok=False)
LOG.tc_step("Waiting for hosts in {} aggregate".format(backing_to_config))
for host in hosts_to_config:
host_helper.wait_for_host_in_instance_backing(
host, storage_backing=backing_to_config)
LOG.tc_step("Check number of {} hosts is {}".format(
instance_backing, number_of_hosts))
assert number_of_hosts == len(host_helper.get_hosts_in_storage_backing(instance_backing)), \
"Number of {} hosts is not {} after configuration".format(instance_backing, number_of_hosts)
def delete_strategy(orchestration,
check_first=True,
fail_ok=False,
conn_ssh=None):
"""
Deletes an orchestration strategy
Args:
orchestration (str): indicates the orchestration strategy type. Choices are patch or upgrade
conn_ssh:
check_first (bool): Check if strategy exits, if so, check if strategy is still in-progress, if so, abort it
fail_ok:
Returns (tuple):
(-1, "No strategy available. Do nothing.")
(0, "<orch_type> orchestration strategy deleted successfully.")
(1, <std_err>) # CLI command rejected
"""
if orchestration is None:
raise ValueError(
"The orchestration type (choices are 'patch' or 'upgrade') must be specified"
)
cmd = ''
if orchestration is "patch":
cmd += "patch-strategy "
elif orchestration is "upgrade":
cmd += "upgrade-strategy "
strategy_info = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)
if check_first:
if not strategy_info:
msg = "No strategy available. Do nothing."
LOG.info(msg)
return -1, msg
if strategy_info.get(OrchStrategyKey.INPROGRESS, None) == 'true':
LOG.info("Strategy in progress. Abort.")
strategy_state = strategy_info[OrchStrategyKey.STATE]
if strategy_state in (OrchStrategyState.APPLYING,
OrchStrategyState.BUILDING,
OrchStrategyState.INITIAL):
cli.sw_manager(cmd,
'abort',
ssh_client=conn_ssh,
fail_ok=False)
wait_strategy_phase_completion(orchestration,
OrchestStrategyPhase.ABORT)
elif strategy_state == OrchStrategyState.ABORTING:
wait_strategy_phase_completion(orchestration,
OrchestStrategyPhase.ABORT)
rc, output = cli.sw_manager(cmd,
'delete',
ssh_client=conn_ssh,
fail_ok=fail_ok)
if rc != 0:
return 1, output
post_strategy_info = get_current_strategy_info(orchestration,
conn_ssh=conn_ssh)
if post_strategy_info:
raise exceptions.OrchestrationError(
"{} strategy still exists after deletion: {}".format(
orchestration, post_strategy_info))
msg = "{} orchestration strategy deleted successfully.".format(
orchestration)
LOG.info(msg)
return 0, msg
def test_ldap_create_user(user_name, sudoer, secondary_group, expiry_days,
expiry_warn_days):
"""
Create a LDAP User with the specified name
User Stories: US70961
Steps:
1 create a LDAP User with the specified name
2 verify the LDAP User is successfully created and get its details
"""
sudoer = True if sudoer == 'sudoer' else False
secondary_group = True if secondary_group == 'secondary_group' else False
LOG.tc_step(
'Check if any LDAP User with name:{} existing'.format(user_name))
existing, user_info = theLdapUserManager.find_ldap_user(user_name)
if existing:
LOG.warn(
'LDAP User:{} already existing! Delete it for testing user-creation'
.format(user_name))
code, output = theLdapUserManager.rm_ldap_user(user_name)
if 0 != code:
skip('LDAP User:{} already existing and failed to delete!')
else:
LOG.warn('Existing LDAP User:{} is successfully deleted'.format(
user_name))
else:
LOG.warn(
'OK, LDAP User:{} is not existing, continue to create one'.format(
user_name))
LOG.tc_step('Creating LDAP User:{}'.format(user_name))
code, user_settings = theLdapUserManager.create_ldap_user(
user_name,
sudoer=sudoer,
secondary_group=secondary_group,
password_expiry_days=expiry_days,
password_expiry_warn_days=expiry_warn_days,
check_if_existing=True,
delete_if_existing=True)
if 0 == code:
LOG.info('OK, created LDAP for User:{}, user-details:\n{}'.format(
user_name, user_settings))
else:
if 1 == code:
msg = 'Already exists the LDAP User:{}.'.format(user_name)
elif 2 == code:
msg = 'Failed to find the created LDAP User:{} although creating succeeded.'.format(
user_name)
elif 3 == code:
msg = 'Failed to create the LDAP User:{}.'.format(user_name)
else:
msg = 'Failed to create the LDAP User:{} for unknown reason.'.format(
user_name)
LOG.error(msg)
assert False, msg
LOG.info('OK, successfully created the LDAP User {}'.format(user_name))
def change_linux_user_password(password, new_password, user=None, host=None):
if not user:
user = HostLinuxUser.get_user()
LOG.info(
'Attempt to change password, from password:{}, to new-password:{}, '
'on host:{}'.format(password, new_password, host))
input_outputs = (
(
'passwd',
(r'\(current\) UNIX password: '******'New password: '******': Authentication token manipulation error',
EOF,
),
),
(
new_password,
('Retype new password:'******'BAD PASSWORD: The password is too similar to the old one',
'BAD PASSWORD: No password supplied',
'passwd: Have exhausted maximum number of retries for service',
EOF,
),
),
(
new_password,
(
': all authentication tokens updated successfully.',
Prompt.CONTROLLER_PROMPT,
),
(),
),
)
conn_to_ac = ControllerClient.get_active_controller()
initial_prompt = r'.*{}\:~\$ '.format(host)
LOG.info('Will login as user:"******", password:"******", to host:"{}"'.format(
user, password, host))
conn = SSHFromSSH(conn_to_ac,
host,
user,
password,
force_password=True,
initial_prompt=initial_prompt)
passed = True
try:
conn.connect(retry=False, use_password=True)
for cmd, expected, errors in input_outputs:
# conn.flush()
LOG.info("Send '{}'\n".format(cmd))
conn.send(cmd)
blob_list = list(expected) + list(errors)
LOG.info("Expect: {}\n".format(blob_list))
index = conn.expect(blob_list=blob_list)
LOG.info('returned index:{}\n'.format(index))
if len(expected) <= index:
passed = False
break
except Exception as e:
LOG.warn('Caught exception when connecting to host:{} as user:{} with '
'pasword:{}\n{}\n'.format(host, user, password, e))
raise
finally:
if user != HostLinuxUser.get_user():
conn.close()
# flush the output to the cli so the next cli is correctly registered
conn.flush()
LOG.info('Successfully changed password from:\n{}\nto:{} for user:{} on '
'host:{}'.format(password, new_password, user, host))
return passed, new_password
def apply_strategy(orchestration,
timeout=OrchestrationPhaseTimeout.APPLY,
conn_ssh=None,
fail_ok=False):
"""
applies an orchestration strategy
Args:
orchestration (str): indicates the orchestration strategy type. Choices are patch or upgrade
timeout (int):
conn_ssh:
fail_ok:
Returns (tuple):
(0, <actual_states>(dict)) - success strategy applied successfully
(1, <std_err>) - CLI command rejected
(2, <actual_states>(dict)) - Strategy apply failed
"""
if orchestration not in ('patch', 'upgrade'):
raise ValueError(
"The orchestration type (choices are 'patch' or 'upgrade') must be specified"
)
# strategy_values = get_current_strategy_values(orchestration)
if orchestration is "patch":
cmd = "patch-strategy apply"
else:
cmd = "upgrade-strategy apply"
rc, output = cli.sw_manager(cmd, ssh_client=conn_ssh, fail_ok=fail_ok)
if rc != 0:
return 1, output
res, results = wait_strategy_phase_completion(orchestration,
OrchestStrategyPhase.APPLY,
timeout=timeout,
conn_ssh=conn_ssh,
fail_ok=True)
if not res:
c_phase = results[OrchStrategyKey.CURRENT_PHASE]
c_compl = results['current-phase-completion']
if c_phase == OrchestStrategyPhase.APPLY and int(
c_compl.strip()[:-1]) > 50:
LOG.warning(
'current-phase-completion > 50%, extend wait time for {} strategy to apply'
.format(orchestration))
res, results = wait_strategy_phase_completion(
orchestration,
OrchestStrategyPhase.APPLY,
timeout=timeout,
conn_ssh=conn_ssh,
fail_ok=True)
if not res:
msg = "{} strategy failed to apply. Current state: {}".format(
orchestration, results[OrchStrategyKey.STATE])
LOG.warn(msg)
if fail_ok:
return 2, results
else:
raise exceptions.OrchestrationError(msg)
if results[OrchStrategyKey.STATE] != OrchStrategyState.APPLIED or \
results[OrchStrategyKey.APPLY_RESULT] != 'success':
raise exceptions.OrchestrationError(
'{} strategy not in applied state after completion'.format(
orchestration))
LOG.info("apply {} strategy completed successfully: {}".format(
orchestration, results))
return 0, results
def test_ima_no_event(operation, file_path):
"""
This test validates following scenarios will not generate IMA event:
- create symlink of a monitored file
- copy a root file with the proper IMA signature, the nexcute it
- make file attribute changes, include: chgrp, chown, chmod
- create and execute a files as sysadmin
Test Steps:
- Perform specified operation on given file
- Confirm IMA violation event is not triggered
Teardown:
- Delete created test file
Maps to TC_17684/TC_17644/TC_17640/TC_17902 from US105523
This test also covers TC_17665/T_16397 from US105523 (FM Event Log Updates)
"""
global files_to_delete
start_time = common.get_date_in_format()
source_file = file_path
con_ssh = ControllerClient.get_active_controller()
LOG.tc_step("{} for {}".format(operation, source_file))
if operation == 'create_symlink':
dest_file = "my_symlink"
create_symlink(source_file, dest_file)
files_to_delete.append(dest_file)
checksum_match = checksum_compare(source_file, dest_file)
assert checksum_match, "SHA256 checksum should match source file and " \
"the symlink but didn't"
elif operation == 'copy_and_execute':
dest_file = "/usr/sbin/TEMP"
copy_file(source_file, dest_file)
files_to_delete.append(dest_file)
LOG.info("Execute the copied file")
con_ssh.exec_sudo_cmd("{} -p".format(dest_file))
elif operation == 'change_file_attributes':
if HostLinuxUser.get_home() != 'sysadmin':
skip('sysadmin user is required to run this test')
dest_file = "/usr/sbin/TEMP"
copy_file(source_file, dest_file)
files_to_delete.append(dest_file)
LOG.info("Change permission of copy")
chmod_file(dest_file, "777")
LOG.info("Changing group ownership of file")
chgrp_file(dest_file, "sys_protected")
LOG.info("Changing file ownership")
chown_file(dest_file, "sysadmin:sys_protected")
elif operation == 'create_and_execute':
dest_file = "{}/TEMP".format(HostLinuxUser.get_home())
create_and_execute(file_path=dest_file, sudo=False)
LOG.tc_step("Ensure no IMA events are raised")
events_found = system_helper.wait_for_events(start=start_time,
timeout=60,
num=10,
event_log_id=EventLogID.IMA,
fail_ok=True,
strict=False)
assert not events_found, "Unexpected IMA events found"
def test_ima_event_generation(operation, file_path):
"""
Following IMA violation scenarios are covered:
- append/edit data to/of a monitored file, result in changing of the
hash
- dynamic library changes
- create and execute a files as sysadmin
Test Steps:
- Perform specified file operations
- Check IMA violation event is logged
"""
global files_to_delete
con_ssh = ControllerClient.get_active_controller()
start_time = common.get_date_in_format()
source_file = file_path
backup_file = None
if operation in ('edit_and_execute', 'append_and_execute'):
dest_file = "/usr/sbin/TEMP"
copy_file(source_file, dest_file, cleanup='dest')
if operation == 'edit_and_execute':
LOG.tc_step("Open copy of monitored file and save")
cmd = "vim {} '+:wq!'".format(dest_file)
con_ssh.exec_sudo_cmd(cmd, fail_ok=False)
execute_cmd = "{} -p".format(dest_file)
else:
LOG.tc_step("Append to copy of monitored file")
cmd = 'echo "output" | sudo -S tee -a /usr/sbin/TEMP'.format(
HostLinuxUser.get_password())
con_ssh.exec_cmd(cmd, fail_ok=False)
LOG.tc_step("Execute modified file")
con_ssh.exec_sudo_cmd(dest_file)
execute_cmd = "{}".format(dest_file)
LOG.tc_step("Execute modified file")
con_ssh.exec_sudo_cmd(execute_cmd)
elif operation == 'replace_library':
backup_file = "/root/{}".format(source_file.split('/')[-1])
dest_file_nocsum = "/root/TEMP"
LOG.info("Backup source file {} to {}".format(source_file,
backup_file))
copy_file(source_file, backup_file)
LOG.info("Copy the library without the checksum")
copy_file(source_file, dest_file_nocsum, preserve=False)
LOG.info("Replace the library with the unsigned one")
move_file(dest_file_nocsum, source_file)
elif operation == 'create_and_execute':
dest_file = "{}/TEMP".format(HostLinuxUser.get_home())
create_and_execute(file_path=dest_file, sudo=True)
LOG.tc_step("Check for IMA event")
ima_events = system_helper.wait_for_events(start=start_time,
timeout=60,
num=10,
event_log_id=EventLogID.IMA,
state='log',
severity='major',
fail_ok=True,
strict=False)
if backup_file:
LOG.info("Restore backup file {} to {}".format(backup_file,
source_file))
move_file(backup_file, source_file)
assert ima_events, "IMA event is not generated after {} on " \
"{}".format(operation, file_path)
def _prepare_test(vm1, vm2, get_hosts, with_router):
"""
VMs:
VM1: under test (primary tenant)
VM2: traffic observer
"""
vm1_host = vm_helper.get_vm_host(vm1)
vm2_host = vm_helper.get_vm_host(vm2)
vm1_router = network_helper.get_tenant_router(
auth_info=Tenant.get_primary())
vm2_router = network_helper.get_tenant_router(
auth_info=Tenant.get_secondary())
vm1_router_host = network_helper.get_router_host(router=vm1_router)
vm2_router_host = network_helper.get_router_host(router=vm2_router)
targets = list(get_hosts)
if vm1_router_host == vm2_router_host:
end_time = time.time() + 360
while time.time() < end_time:
vm1_router_host = network_helper.get_router_host(
router=vm1_router)
vm2_router_host = network_helper.get_router_host(
router=vm2_router)
if vm1_router_host != vm2_router_host:
break
else:
assert vm1_router_host != vm2_router_host, "two routers are located on the same compute host"
if not with_router:
"""
Setup:
VM1 on COMPUTE-A
VM2 not on COMPUTE-A
ROUTER1 on COMPUTE-B
ROUTER2 on COMPUTE-C
"""
if len(get_hosts) < 3:
skip(
"Lab not suitable for without_router, requires at least three hypervisors"
)
LOG.tc_step(
"Ensure VM2, ROUTER2 not on COMPUTE-A, for simplicity, ensure they are on the same compute"
)
if vm2_host != vm2_router_host:
vm_helper.live_migrate_vm(vm_id=vm2,
destination_host=vm2_router_host)
vm2_host = vm_helper.get_vm_host(vm2)
assert vm2_host == vm2_router_host, "live-migration failed"
host_observer = vm2_host
LOG.tc_step(
"Ensure VM1 and (ROUTER1, VM2, ROUTER2) are on different hosts"
)
if vm1_router_host in targets:
# ensure vm1_router_host is not selected for vm1
# vm1_router_host can be backed by any type of storage
targets.remove(vm1_router_host)
if vm2_host in targets:
targets.remove(vm2_host)
if vm1_host in targets:
host_src_evacuation = vm1_host
else:
assert targets, "no suitable compute for vm1, after excluding ROUTER1, VM2, ROUTER2 's hosts"
host_src_evacuation = targets[0]
vm_helper.live_migrate_vm(vm_id=vm1,
destination_host=host_src_evacuation)
vm1_host = vm_helper.get_vm_host(vm1)
assert vm1_host == host_src_evacuation, "live-migration failed"
# verify setup
vm1_host = vm_helper.get_vm_host(vm1)
vm2_host = vm_helper.get_vm_host(vm2)
vm1_router_host = network_helper.get_router_host(router=vm1_router)
vm2_router_host = network_helper.get_router_host(router=vm2_router)
assert vm1_router_host != vm1_host and vm2_host != vm1_host and vm2_router_host != vm1_host, \
"setup is incorrect"
else:
"""
Setup:
VM1, ROUTER1 on COMPUTE-A
VM2 not on COMPUTE-A
ROUTER2 on COMPUTE-B
"""
LOG.tc_step("Ensure VM1, ROUTER1 on COMPUTE-A")
# VM1 must be sitting on ROUTER1's host, thus vm1_router_host must be backed by local_image
assert vm1_router_host in targets, "vm1_router_host is not backed by local_image"
if vm1_host != vm1_router_host:
vm_helper.live_migrate_vm(vm_id=vm1,
destination_host=vm1_router_host)
vm1_host = vm_helper.get_vm_host(vm1)
assert vm1_host == vm1_router_host, "live-migration failed"
host_src_evacuation = vm1_host
LOG.tc_step(
"Ensure VM2, ROUTER2 not on COMPUTE-A, for simplicity, ensure they are on the same compute"
)
targets.remove(host_src_evacuation)
if vm2_host in targets:
host_observer = vm2_host
else:
assert targets, "no suitable compute for vm2, after excluding COMPUTE-A"
host_observer = targets[0]
vm_helper.live_migrate_vm(vm_id=vm2,
destination_host=host_observer)
vm2_host = vm_helper.get_vm_host(vm2)
assert vm2_host == host_observer, "live-migration failed"
# verify setup
vm1_host = vm_helper.get_vm_host(vm1)
vm2_host = vm_helper.get_vm_host(vm2)
vm1_router_host = network_helper.get_router_host(router=vm1_router)
vm2_router_host = network_helper.get_router_host(router=vm2_router)
assert vm1_host == vm1_router_host and vm2_host != vm1_host and vm2_router_host != vm1_host, \
"setup is incorrect"
assert vm1_host == host_src_evacuation and vm2_host == host_observer, "setup is incorrect"
LOG.info("Evacuate: VM {} on {}, ROUTER on {}".format(
vm1, vm1_host, vm1_router_host))
LOG.info("Observer: VM {} on {}, ROUTER on {}".format(
vm2, vm2_host, vm2_router_host))
return host_src_evacuation, host_observer
def test_ipv6_subnet(vif_model, check_avs_pattern):
"""
Ipv6 Subnet feature test cases
Test Steps:
- Create networks
- Create Ipv6 enabled subnet
- Boot the first vm with the ipv6 subnet
- Boot the second vm with ipv6 subnet
- Configure interfaces to get ipv6 addr
- Verify connectivity ipv6 interfaces
- Ping default router
Test Teardown:
- Delete vms, subnets, and networks created
"""
network_names = ['network11']
net_ids = []
sub_nets = ["fd00:0:0:21::/64"]
gateway_ipv6 = "fd00:0:0:21::1"
subnet_ids = []
dns_server = "2001:4860:4860::8888"
LOG.tc_step("Create Networks to setup IPV6 subnet")
for net in network_names:
net_ids.append(
network_helper.create_network(name=net, cleanup='function')[1])
LOG.tc_step("Create IPV6 Subnet on the Network Created")
for sub, network in zip(sub_nets, net_ids):
subnet_ids.append(
network_helper.create_subnet(network=network,
ip_version=6,
dns_servers=dns_server,
subnet_range=sub,
gateway='none',
cleanup='function')[1])
LOG.tc_step("Boot a VM with mgmt net and Network with IPV6 subnet")
mgmt_net_id = network_helper.get_mgmt_net_id()
nics = [{
'net-id': mgmt_net_id
}, {
'net-id': net_ids[0],
'vif-model': vif_model
}]
image = None
if vif_model == 'e1000':
image = glance_helper.create_image(name=vif_model,
hw_vif_model=vif_model,
cleanup='function')[1]
LOG.tc_step("Boot a vm with created nets")
vm_id = vm_helper.boot_vm(name='vm-with-ipv6-nic',
nics=nics,
image_id=image,
cleanup='function')[1]
LOG.tc_step("Setup interface script inside guest and restart network")
_bring_up_interface(vm_id)
LOG.tc_step("Boot a second vm with created nets")
vm_id2 = vm_helper.boot_vm(name='vm2-with-ipv6-nic',
nics=nics,
cleanup='function')[1]
LOG.tc_step("Setup interface script inside guest and restart network")
_bring_up_interface(vm_id2)
with vm_helper.ssh_to_vm_from_natbox(vm_id) as vm_ssh:
ip_addr = _get_ipv6_for_eth(eth_name='eth1', ssh_client=vm_ssh)
if ip_addr is '':
LOG.info('Ip addr is not assigned')
assert ip_addr != '', "Failed to assign ip"
else:
LOG.info("Got Ipv6 address:{}".format(ip_addr))
with vm_helper.ssh_to_vm_from_natbox(vm_id2) as vm_ssh:
LOG.tc_step("ping b/w vms on the ipv6 net")
ping = _ping6_vms(ssh_client=vm_ssh, ipv6_addr=ip_addr)
assert ping == 0, "Ping between VMs failed"
LOG.tc_step("ping Default Gateway from vms on the ipv6 net")
ping = _ping6_vms(ssh_client=vm_ssh, ipv6_addr=gateway_ipv6)
assert ping == 0, "Ping to default router failed"
def test_horizon_storage_overview_display(storage_overview_pg):
"""
Tests the storage overview display:
Setups:
- Login as Admin
- Go to Admin > Platform > Storage Overview
Teardown:
- Logout
Test Steps:
- Test Storage cluster UUID, Health Status and Details display
- Test host and rank table display
- Test osd.# table and status display
"""
con_ssh = ControllerClient.get_active_controller()
standby = system_helper.get_standby_controller_name(con_ssh=con_ssh)
if standby == 'controller-1':
LOG.info('Workaround for CGTS-16739')
host_helper.swact_host(con_ssh=con_ssh)
LOG.tc_step('Check storage cluster UUID, ceph health and storage usage display')
cli_storage_service_info = []
uuid = system_helper.get_clusters(field='cluster_uuid')[0]
cli_storage_service_info.append(uuid)
# 'ceph health' cmd output sample:
# HEALTH_ERR 1728 pgs are stuck inactive for more than 300 seconds; 1728 pgs stuck inactive; 1728 pgs stuck unclean;\
# 1 mons down, quorum 0,1 controller-0,controller-1
health_details = con_ssh.exec_cmd('ceph health')[1]
health_status = health_details.split(' ')[0]
cli_storage_service_info.append(health_status)
if health_status == 'HEALTH_ERR':
health_details = health_details.split('HEALTH_ERR ')[1]
elif health_status == 'HEALTH_WARN':
health_details = health_details.split('HEALTH_WARN ')[1]
cli_storage_service_info.append(health_details)
horizon_ceph_info = storage_overview_pg.storage_service_info.get_content()
for info in cli_storage_service_info:
assert info in horizon_ceph_info.values(), 'Horizon storage cluster info does not match to cli info'
LOG.tc_step('Storage service details display correct')
LOG.info('Test host and rank table display')
ceph_mon_status = eval(con_ssh.exec_cmd('ceph mon_status')[1])
mon_map = ceph_mon_status.get('monmap')
cli_ceph_monitor = {}
# mon_map.get('mons') returns a dict list
for mon_info_dict in mon_map.get('mons'):
host_name = mon_info_dict.get('name')
host_rank = mon_info_dict.get('rank')
cli_ceph_monitor[host_name] = str(host_rank)
for host_name in cli_ceph_monitor.keys():
cli_rank_val = cli_ceph_monitor[host_name]
horizon_rank_val = storage_overview_pg.get_storage_overview_monitor_info(host_name, 'Rank')
assert horizon_rank_val == cli_rank_val, '{} rank display incorrectly'.format(host_name)
LOG.info('Host and rank table display correct')
LOG.tc_step('Test osd table and status display')
osd_list = storage_helper.get_osds()
for osd_id in osd_list:
if osd_id is not None:
expt_horizon = {}
for header in storage_overview_pg.osds_table.column_names:
host_name = storage_helper.get_osd_host(osd_id)
osd_name = 'osd.{}'.format(osd_id)
expt_horizon['Host'] = host_name
expt_horizon['Name'] = osd_name
expt_horizon['Status'] = 'up'
if not storage_helper.is_osd_up(osd_id, con_ssh):
expt_horizon['Status'] = 'down'
horizon_val = storage_overview_pg.get_storage_overview_osd_info(osd_name, header)
assert expt_horizon[header] == horizon_val, '{}{} display incorrect'.format(osd_name, header)
LOG.info('Osd table display correct')
horizon.test_result = True
def create_aggregate(field='name',
name=None,
avail_zone=None,
properties=None,
check_first=True,
fail_ok=False,
con_ssh=None,
auth_info=Tenant.get('admin')):
"""
Add a aggregate with given name and availability zone.
Args:
field (str): name or id
name (str): name for aggregate to create
avail_zone (str|None):
properties (dict|None)
check_first (bool)
fail_ok (bool):
con_ssh (SSHClient):
auth_info (dict):
Returns (tuple):
(0, <rtn_val>) -- aggregate successfully created
(1, <stderr>) -- cli rejected
(2, "Created aggregate is not as specified") -- name and/or availability zone mismatch
"""
if not name:
existing_names = get_aggregates(field='name')
name = common.get_unique_name(name_str='cgcsauto',
existing_names=existing_names)
args_dict = {
'--zone': avail_zone,
'--property': properties,
}
args = '{} {}'.format(common.parse_args(args_dict, repeat_arg=True), name)
if check_first:
aggregates_ = get_aggregates(field=field,
name=name,
avail_zone=avail_zone)
if aggregates_:
LOG.warning(
"Aggregate {} already exists. Do nothing.".format(name))
return -1, aggregates_[0]
LOG.info("Adding aggregate {}".format(name))
res, out = cli.openstack('aggregate create',
args,
ssh_client=con_ssh,
fail_ok=fail_ok,
auth_info=auth_info)
if res == 1:
return res, out
out_tab = table_parser.table(out)
succ_msg = "Aggregate {} is successfully created".format(name)
LOG.info(succ_msg)
return 0, table_parser.get_value_two_col_table(out_tab, field)
def delete_server_groups(srv_grp_ids=None,
check_first=True,
fail_ok=False,
auth_info=Tenant.get('admin'),
con_ssh=None):
"""
Delete server group(s)
Args:
srv_grp_ids (list|str): id(s) for server group(s) to delete.
check_first (bool): whether to check existence of given server groups before attempt to
delete. Default: True.
fail_ok (bool):
auth_info (dict|None):
con_ssh (SSHClient):
Returns (tuple): (rtn_code(int), msg(str)) # rtn_code 1,2 only returns when fail_ok=True
(-1, 'No server group(s) to delete.')
(-1, 'None of the given server group(s) exists on system.')
(0, "Server group(s) deleted successfully.")
(1, <stderr>) # Deletion rejected for all of the server groups. Return CLI stderr.
(2, "Some deleted server group(s) still exist on system:: <srv_grp_ids>")
"""
existing_sgs = None
if not srv_grp_ids:
existing_sgs = srv_grp_ids = get_server_groups(con_ssh=con_ssh,
auth_info=auth_info)
elif isinstance(srv_grp_ids, str):
srv_grp_ids = [srv_grp_ids]
srv_grp_ids = [sg for sg in srv_grp_ids if sg]
if not srv_grp_ids:
LOG.info("No server group(s) to delete. Do Nothing")
return -1, 'No server group(s) to delete.'
if check_first:
if existing_sgs is None:
existing_sgs = get_server_groups(con_ssh=con_ssh,
auth_info=auth_info)
srv_grp_ids = list(set(srv_grp_ids) & set(existing_sgs))
if not srv_grp_ids:
msg = "None of the given server group(s) exists on system. Do nothing"
LOG.info(msg)
return -1, msg
LOG.info("Deleting server group(s): {}".format(srv_grp_ids))
code, output = cli.openstack('server group delete',
' '.join(srv_grp_ids),
ssh_client=con_ssh,
fail_ok=True,
auth_info=auth_info,
timeout=60)
if code == 1:
return 1, output
existing_sgs = get_server_groups(con_ssh=con_ssh, auth_info=auth_info)
grps_undeleted = list(set(srv_grp_ids) & set(existing_sgs))
if grps_undeleted:
msg = "Some server group(s) still exist on system after deletion: {}".format(
grps_undeleted)
LOG.warning(msg)
if fail_ok:
return 2, msg
raise exceptions.NovaError(msg)
msg = "Server group(s) deleted successfully."
LOG.info(msg)
return 0, "Server group(s) deleted successfully."
def test_multi_node_failure_avoidance(reserve_unreserve_all_hosts_module,
mnfa_timeout, mnfa_threshold):
"""
Test multi node failure avoidance
Args:
mnfa_timeout
mnfa_threshold
reserve_unreserve_all_hosts_module: test fixture to reserve unreserve all vlm nodes for lab under test
Setups:
- Reserve all nodes in vlm
Test Steps:
- Power off compute/storage nodes in vlm using multi-processing to simulate a power outage on computes
- Power on all nodes compute nodes
- Wait for nodes to become degraded state during the mnfa mode
- Wait for nodes to become active state
- Check new event is are created for multi node failure
- Verify the time differences between multi node failure enter and exit in the event log equal to configured
mnfa thereshold value.
"""
hosts_to_check = system_helper.get_hosts(
availability=(HostAvailState.AVAILABLE, HostAvailState.ONLINE))
hosts_to_test = [
host for host in hosts_to_check if 'controller' not in host
]
if len(hosts_to_test) < mnfa_threshold:
skip(
"Compute and storage host count smaller than mnfa threshhold value"
)
elif len(hosts_to_test) > mnfa_threshold + 1:
hosts_to_test = hosts_to_test[:mnfa_threshold + 1]
LOG.info("Online or Available hosts before power-off: {}".format(
hosts_to_check))
start_time = common.get_date_in_format(date_format='%Y-%m-%d %T')
LOG.tc_step('Modify mnfa_timeout parameter to {}'.format(mnfa_timeout))
system_helper.modify_service_parameter(service='platform',
section='maintenance',
name='mnfa_timeout',
apply=True,
value=str(mnfa_timeout))
system_helper.modify_service_parameter(service='platform',
section='maintenance',
name='mnfa_threshold',
apply=True,
value=str(mnfa_threshold))
try:
LOG.tc_step("Power off hosts and check for degraded state: {}".format(
hosts_to_test))
vlm_helper.power_off_hosts_simultaneously(hosts=hosts_to_test)
time.sleep(20)
degraded_hosts = system_helper.get_hosts(
availability=HostAvailState.DEGRADED, hostname=hosts_to_check)
finally:
LOG.tc_step("Power on hosts and ensure they are recovered: {}".format(
hosts_to_test))
vlm_helper.power_on_hosts(hosts=hosts_to_test,
reserve=False,
hosts_to_check=hosts_to_check,
check_interval=20)
assert sorted(degraded_hosts) == sorted(
hosts_to_test), 'Degraded hosts mismatch with powered-off hosts'
LOG.tc_step(
"Check MNFA duration is the same as MNFA timeout value via system event log"
)
active_con = system_helper.get_active_controller_name()
entity_instance_id = 'host={}.event=mnfa_enter'.format(active_con)
first_event = system_helper.wait_for_events(
num=1,
timeout=70,
start=start_time,
fail_ok=True,
strict=False,
event_log_id=EventLogID.MNFA_MODE,
field='Time Stamp',
entity_instance_id=entity_instance_id)
entity_instance_id = 'host={}.event=mnfa_exit'.format(active_con)
second_event = system_helper.wait_for_events(
num=1,
timeout=70,
start=start_time,
fail_ok=False,
strict=False,
event_log_id=EventLogID.MNFA_MODE,
field='Time Stamp',
entity_instance_id=entity_instance_id)
pattern = '%Y-%m-%dT%H:%M:%S'
event_duration = datetime.strptime(second_event[0][:-7],
pattern) - datetime.strptime(
first_event[0][:-7], pattern)
event_duration = event_duration.total_seconds()
assert abs(event_duration - mnfa_timeout) <= 1, 'MNFA event duration {} is different than MNFA timeout value {}'.\
format(event_duration, mnfa_timeout)
def test_ima_keyring_protection():
"""
This test validates that the IMA keyring is safe from user space attacks.
Test Steps:
- Attempt to add new keys to the keyring
- Extract key ID and save
- Attempt to change the key timeout
- Attempt to change the group and ownership of the key
- Attempt to delete the key
This test maps to TC_17667/T_16387 from US105523 (IMA keyring is safe from
user space attacks)
"""
con_ssh = ControllerClient.get_active_controller()
LOG.info("Extract ima key ID")
exitcode, msg = con_ssh.exec_sudo_cmd("cat /proc/keys | grep _ima")
raw_key_id = msg.split(" ", maxsplit=1)[0]
key_id = "0x{}".format(raw_key_id)
LOG.info("Extracted key is: {}".format(key_id))
LOG.info("Attempting to add new keys to keyring")
exitcode, msg = con_ssh.exec_sudo_cmd("keyctl add keyring TEST stuff "
"{}".format(key_id))
assert exitcode != 0, \
"Key addition should have failed but instead succeeded"
LOG.info("Attempt to change the timeout on a key")
exitcode, msg = con_ssh.exec_sudo_cmd("keyctl timeout {} "
"3600".format(key_id))
assert exitcode != 0, \
"Key timeout modification should be rejected but instead succeeded"
LOG.info("Attempt to change the group of a key")
exitcode, msg = con_ssh.exec_sudo_cmd("keyctl chgrp {} 0".format(key_id))
assert exitcode != 0, \
"Key group modification should be rejected but instead succeeded"
LOG.info("Attempt to change the ownership of a key")
exitcode, msg = con_ssh.exec_sudo_cmd(
"keyctl chown {} 1875".format(key_id))
assert exitcode != 0, \
"Key ownership modification should be rejected but instead succeeded"
LOG.info("Attempt to delete a key")
exitcode, msg = con_ssh.exec_sudo_cmd("keyctl clear {}".format(key_id))
assert exitcode != 0, \
"Key ownership deletion should be rejected but instead succeeded"
def login_as_ldap_user(self,
user_name,
password,
host=None,
pre_store=False,
disconnect_after=False):
"""
Login as the specified user name and password onto the specified host
Args:
user_name (str): user name
password (str): password
host (str): host to login to
pre_store (bool):
True - pre-store keystone user credentials for
session
False - chose 'N' (by default) meaning do not
pre-store keystone user credentials
disconnect_after (bool):
True - disconnect the logged in session
False - keep the logged in session
Returns (tuple):
logged_in (bool) - True if successfully logged into the
specified host
using the specified user/password
password (str) - the password used to login
ssh_con (object) - the ssh session logged in
"""
if not host:
host = 'controller-1'
if system_helper.is_aio_simplex():
host = 'controller-0'
prompt_keystone_user_name = r'Enter Keystone username \[{}\]: '.format(
user_name)
cmd_expected = (
(
'ssh -l {} -o UserKnownHostsFile=/dev/null {}'.format(
user_name, host),
(r'Are you sure you want to continue connecting \(yes/no\)\?',
),
('ssh: Could not resolve hostname {}: Name or service not '
'known'.format(host), ),
),
(
'yes',
(r'{}@{}\'s password: '******'{}'.format(password),
(
prompt_keystone_user_name,
Prompt.CONTROLLER_PROMPT,
),
(r'Permission denied, please try again\.', ),
),
)
logged_in = False
self.ssh_con.flush()
for i in range(len(cmd_expected)):
cmd, expected, errors = cmd_expected[i]
LOG.info('cmd={}\nexpected={}\nerrors={}\n'.format(
cmd, expected, errors))
self.ssh_con.send(cmd)
index = self.ssh_con.expect(blob_list=list(expected + errors))
if len(expected) <= index:
break
elif 3 == i:
if expected[index] == prompt_keystone_user_name:
assert pre_store, \
'pre_store is False, while selecting "y" to ' \
'"Pre-store Keystone user credentials ' \
'for this session!"'
else:
logged_in = True
break
else:
logged_in = True
if logged_in:
if disconnect_after:
self.ssh_con.send('exit')
return logged_in, password, self.ssh_con
def wait_for_apps_status(apps,
status,
timeout=360,
check_interval=5,
fail_ok=False,
con_ssh=None,
auth_info=Tenant.get('admin_platform')):
"""
Wait for applications to reach expected status via system application-list
Args:
apps:
status:
timeout:
check_interval:
fail_ok:
con_ssh:
auth_info:
Returns (tuple):
"""
status = '' if not status else status
if isinstance(apps, str):
apps = [apps]
apps_to_check = list(apps)
check_failed = []
end_time = time.time() + timeout
LOG.info("Wait for {} application(s) to reach status: {}".format(
apps, status))
while time.time() < end_time:
apps_status = get_apps(application=apps_to_check,
field=('application', 'status'),
con_ssh=con_ssh,
auth_info=auth_info)
apps_status = {item[0]: item[1] for item in apps_status if item}
checked = []
for app in apps_to_check:
current_app_status = apps_status.get(app, '')
if current_app_status == status:
checked.append(app)
elif current_app_status.endswith('ed'):
check_failed.append(app)
checked.append(app)
apps_to_check = list(set(apps_to_check) - set(checked))
if not apps_to_check:
if check_failed:
msg = '{} failed to reach status - {}'.format(
check_failed, status)
LOG.warning(msg)
if fail_ok:
return False, check_failed
else:
raise exceptions.ContainerError(msg)
LOG.info("{} reached expected status {}".format(apps, status))
return True, None
time.sleep(check_interval)
check_failed += apps_to_check
msg = '{} did not reach status {} within {}s'.format(
check_failed, status, timeout)
LOG.warning(msg)
if fail_ok:
return False, check_failed
raise exceptions.ContainerError(msg)
def modify_https(enable_https=True,
check_first=True,
con_ssh=None,
auth_info=Tenant.get('admin_platform'),
fail_ok=False):
"""
Modify platform https via 'system modify https_enable=<bool>'
Args:
enable_https (bool): True/False to enable https or not
check_first (bool): if user want to check if the lab is already in
the state that user try to enable
con_ssh (SSHClient):
auth_info (dict):
fail_ok (bool):
Returns (tuple):
(-1, msg)
(0, msg)
(1, <std_err>)
"""
if check_first:
is_https = keystone_helper.is_https_enabled(source_openrc=False,
auth_info=auth_info,
con_ssh=con_ssh)
if (is_https and enable_https) or (not is_https and not enable_https):
msg = "Https is already {}. Do nothing.".format(
'enabled' if enable_https else 'disabled')
LOG.info(msg)
return -1, msg
LOG.info("Modify system to {} https".format(
'enable' if enable_https else 'disable'))
res, output = system_helper.modify_system(fail_ok=fail_ok,
con_ssh=con_ssh,
auth_info=auth_info,
https_enabled='{}'.format(
str(enable_https).lower()))
if res == 1:
return 1, output
LOG.info("Wait up to 60s for config out-of-date alarm with best effort.")
system_helper.wait_for_alarm(alarm_id=EventLogID.CONFIG_OUT_OF_DATE,
entity_id='controller-',
strict=False,
con_ssh=con_ssh,
timeout=60,
fail_ok=True,
auth_info=auth_info)
LOG.info("Wait up to 600s for config out-of-date alarm to clear.")
system_helper.wait_for_alarm_gone(EventLogID.CONFIG_OUT_OF_DATE,
con_ssh=con_ssh,
timeout=600,
check_interval=20,
fail_ok=False,
auth_info=auth_info)
LOG.info("Wait up to 300s for public endpoints to be updated")
expt_status = 'enabled' if enable_https else 'disabled'
end_time = time.time() + 300
while time.time() < end_time:
if keystone_helper.is_https_enabled(con_ssh=con_ssh,
source_openrc=False,
auth_info=auth_info) == \
enable_https:
break
time.sleep(10)
else:
raise exceptions.KeystoneError(
"Https is not {} in 'openstack endpoint list'".format(expt_status))
msg = 'Https is {} successfully'.format(expt_status)
LOG.info(msg)
# TODO: install certificate for https. There will be a warning msg if
# self-signed certificate is used
if not ProjVar.get_var('IS_DC') or \
(auth_info and auth_info.get('region', None) in (
'RegionOne', 'SystemController')):
# If DC, use the central region https as system https, since that is
# the one used for external access
CliAuth.set_vars(HTTPS=enable_https)
return 0, msg
def apply_app(app_name,
check_first=False,
fail_ok=False,
applied_timeout=300,
check_interval=10,
wait_for_alarm_gone=True,
con_ssh=None,
auth_info=Tenant.get('admin_platform')):
"""
Apply/Re-apply application via system application-apply. Check for status
reaches 'applied'.
Args:
app_name (str):
check_first:
fail_ok:
applied_timeout:
check_interval:
con_ssh:
wait_for_alarm_gone (bool):
auth_info:
Returns (tuple):
(-1, "<app_name> is already applied. Do nothing.") # only returns
if check_first=True.
(0, "<app_name> (re)applied successfully")
(1, <std_err>) # cli rejected
(2, "<app_name> failed to apply") # did not reach applied status
after apply.
"""
if check_first:
app_status = get_apps(application=app_name,
field='status',
con_ssh=con_ssh,
auth_info=auth_info)
if app_status and app_status[0] == AppStatus.APPLIED:
msg = '{} is already applied. Do nothing.'.format(app_name)
LOG.info(msg)
return -1, msg
LOG.info("Apply application: {}".format(app_name))
code, output = cli.system('application-apply',
app_name,
ssh_client=con_ssh,
fail_ok=fail_ok,
auth_info=auth_info)
if code > 0:
return 1, output
res = wait_for_apps_status(apps=app_name,
status=AppStatus.APPLIED,
timeout=applied_timeout,
check_interval=check_interval,
con_ssh=con_ssh,
auth_info=auth_info,
fail_ok=fail_ok)[0]
if not res:
return 2, "{} failed to apply".format(app_name)
if wait_for_alarm_gone:
alarm_id = EventLogID.CONFIG_OUT_OF_DATE
if system_helper.wait_for_alarm(alarm_id=alarm_id,
entity_id='controller',
timeout=15,
fail_ok=True,
auth_info=auth_info,
con_ssh=con_ssh)[0]:
system_helper.wait_for_alarm_gone(alarm_id=alarm_id,
entity_id='controller',
timeout=120,
check_interval=10,
con_ssh=con_ssh,
auth_info=auth_info)
msg = '{} (re)applied successfully'.format(app_name)
LOG.info(msg)
return 0, msg
def create_strategy(orchestration,
controller_apply_type=None,
storage_apply_type=None,
compute_apply_type=None,
max_parallel_computes=0,
instance_action=None,
alarm_restrictions=None,
wait_for_completion=True,
timeout=None,
conn_ssh=None,
fail_ok=False):
"""
Creates a orchestration strategy
Args:
orchestration (str): indicates the orchestration strategy type. Choices are patch or upgrade
controller_apply_type (str): Valid only for patch orchestration. indicates how the strategy is applied on
controllers: serial/parallel/ignore. Default is serial
storage_apply_type (str): indicates how the strategy is applied on storage hosts. Choices are: serial/parallel/
ignore. Default is serial
compute_apply_type (str): indicates how the strategy is applied on computes hosts. Choices are: serial/parallel/
ignore. 'parallel' is valid only for patch orchestration. Default is serial.
max_parallel_computes(int): indicates the maximum compute hosts to apply strategy in parallel
instance_action (str): valid only for patch orchestration. Indicates what action to perform on instances.
Choices are migrate or stop-start. The default is to stop-start
alarm_restrictions(str): indicates how to handle alarm restrictions based on the management affecting statuses.
Choices are strict or relaxed. Default is strict.
wait_for_completion:
timeout:
conn_ssh:
fail_ok:
Returns: tuple (rc, dict/msg)
(0, dict) - success
(1, output) - CLI reject
(2, err_msg) - strategy build completion timeout
(3, err_msg) - strategy build completed but with failed state
"""
if orchestration is None:
raise ValueError(
"The orchestration type must be specified: choices are 'patch' or 'upgrade'"
)
args_dict = {
'--storage-apply-type':
storage_apply_type,
'--compute-apply-type':
compute_apply_type,
'--max-parallel-compute-hosts':
max_parallel_computes if max_parallel_computes >= 2 else None,
'--alarm-restrictions':
alarm_restrictions,
}
cmd = ''
if orchestration is "patch":
cmd += "patch-strategy create"
args_dict['--controller-apply-type'] = controller_apply_type
args_dict['--instance-action'] = instance_action
elif orchestration is "upgrade":
cmd += "upgrade-strategy create"
else:
raise exceptions.OrchestrationError(
"Invalid orchestration type (choices are 'patch' or 'upgrade') specified"
)
args = ''
for key, val in args_dict.items():
if val is not None and val != '':
args += ' {} {}'.format(key, val)
LOG.info("Creating {} orchestration strategy with arguments: {} {}".format(
orchestration, cmd, args))
rc, output = cli.sw_manager(cmd,
args,
ssh_client=conn_ssh,
fail_ok=fail_ok)
LOG.info("Verifying if the {} orchestration strategy is created".format(
orchestration))
if rc != 0:
msg = "Create {} strategy failed: {}".format(orchestration, output)
LOG.warn(msg)
if fail_ok:
return 1, msg
else:
raise exceptions.OrchestrationError(msg)
if 'strategy-uuid' not in [tr.strip() for tr in output.split(':')]:
msg = "The {} strategy not created: {}".format(orchestration, output)
LOG.warn(msg)
if fail_ok:
return 1, msg
else:
raise exceptions.OrchestrationError(msg)
if wait_for_completion:
if timeout is None:
timeout = OrchestrationPhaseTimeout.BUILD
if not wait_strategy_phase_completion(orchestration,
OrchestStrategyPhase.BUILD,
timeout,
conn_ssh=conn_ssh)[0]:
msg = "The {} strategy created failed build: {}".format(
orchestration, output)
if fail_ok:
LOG.warning(msg)
return 2, msg
else:
raise exceptions.OrchestrationError(msg)
# get values of the created strategy
results = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)
if OrchestStrategyPhase.BUILD != results['current-phase']:
msg = "Unexpected {} strategy phase= {} encountered. A 'build' phase was expected. "\
.format(orchestration, results["current-phase"])
if fail_ok:
LOG.warning(msg)
return 3, msg
else:
raise exceptions.OrchestrationError(msg)
if "failed" in results["state"]:
msg = "The {} strategy 'failed' in build phase; reason = {}".format(
orchestration, results["build-reason"])
if fail_ok:
LOG.warning(msg)
return 3, msg
else:
raise exceptions.OrchestrationError(msg)
LOG.info("Create {} strategy completed successfully: {}".format(
orchestration, results))
return 0, results
def test_lock_unlock_host(host_type, collect_kpi):
"""
Verify lock unlock host
Test Steps:
- Select a host per given type. If type is controller, select standby controller.
- Lock selected host and ensure it is successfully locked
- Unlock selected host and ensure it is successfully unlocked
"""
init_time = None
if collect_kpi:
init_time = common.get_date_in_format(date_format=KPI_DATE_FORMAT)
LOG.tc_step("Select a {} node from system if any".format(host_type))
if host_type == 'controller':
if system_helper.is_aio_simplex():
host = 'controller-0'
else:
host = system_helper.get_standby_controller_name()
assert host, "No standby controller available"
else:
if host_type == 'compute' and (system_helper.is_aio_duplex()
or system_helper.is_aio_simplex()):
skip("No compute host on AIO system")
elif host_type == 'storage' and not system_helper.is_storage_system():
skip("System does not have storage nodes")
hosts = system_helper.get_hosts(personality=host_type,
availability=HostAvailState.AVAILABLE,
operational=HostOperState.ENABLED)
assert hosts, "No good {} host on system".format(host_type)
host = hosts[0]
LOG.tc_step(
"Lock {} host - {} and ensure it is successfully locked".format(
host_type, host))
HostsToRecover.add(host)
host_helper.lock_host(host, swact=False)
# wait for services to stabilize before unlocking
time.sleep(20)
# unlock standby controller node and verify controller node is successfully unlocked
LOG.tc_step(
"Unlock {} host - {} and ensure it is successfully unlocked".format(
host_type, host))
host_helper.unlock_host(host)
LOG.tc_step("Check helm list after host unlocked")
con_ssh = ControllerClient.get_active_controller()
con_ssh.exec_cmd('helm list', fail_ok=False)
if collect_kpi:
lock_kpi_name = HostLock.NAME.format(host_type)
unlock_kpi_name = HostUnlock.NAME.format(host_type)
unlock_host_type = host_type
if container_helper.is_stx_openstack_deployed():
if system_helper.is_aio_system():
unlock_host_type = 'compute'
else:
lock_kpi_name += '_platform'
unlock_kpi_name += '_platform'
if unlock_host_type == 'compute':
unlock_host_type = 'compute_platform'
LOG.info("Collect kpi for lock/unlock {}".format(host_type))
code_lock, out_lock = kpi_log_parser.record_kpi(
local_kpi_file=collect_kpi,
kpi_name=lock_kpi_name,
host=None,
log_path=HostLock.LOG_PATH,
end_pattern=HostLock.END.format(host),
start_pattern=HostLock.START.format(host),
start_path=HostLock.START_PATH,
init_time=init_time)
time.sleep(30) # delay in sysinv log vs nova hypervisor list
code_unlock, out_unlock = kpi_log_parser.record_kpi(
local_kpi_file=collect_kpi,
kpi_name=unlock_kpi_name,
host=None,
log_path=HostUnlock.LOG_PATH,
end_pattern=HostUnlock.END[unlock_host_type].format(host),
init_time=init_time,
start_pattern=HostUnlock.START.format(host),
start_path=HostUnlock.START_PATH)
assert code_lock == 0, 'Failed to collect kpi for host-lock {}. ' \
'Error: \n'.format(host, out_lock)
assert code_unlock == 0, 'Failed to collect kpi for host-unlock {}. ' \
'Error: \n'.format(host, out_lock)
def wait_strategy_phase_completion(orchestration,
current_phase,
timeout=None,
conn_ssh=None,
fail_ok=False):
"""
Waits until the orchestration strategy phase is completed
Args:
orchestration (str): - indicates the orchestration type. possible values: upgrade or patch
current_phase (str): - indicates the current phase of the orchestration. Possible values: build, apply or abort
timeout (int): - indicates the timeout value to wait for the current phase to complete
conn_ssh:
fail_ok:
Returns (tuple):
(True, <strategy info>(dict))
(False, <strategy info>(dict))
"""
if orchestration is None:
raise ValueError(
"The orchestration type (choices are 'patch' or 'upgrade') must be specified"
)
elif orchestration is not "patch" and orchestration is not "upgrade":
raise ValueError(
"Invalid orchestration type (choices are 'patch' or 'upgrade') specified"
)
if not validate_current_strategy_phase(orchestration, current_phase):
raise exceptions.OrchestrationError(
"Current {} strategy phase does not match the specified phase={}".
format(orchestration, current_phase))
check_interval = PHASE_COMPLETION_CHECK_INTERVAL
if timeout is not None and timeout < PHASE_COMPLETION_CHECK_INTERVAL:
timeout = PHASE_COMPLETION_CHECK_INTERVAL
else:
if timeout is None:
if current_phase == OrchestStrategyPhase.BUILD:
timeout = OrchestrationPhaseTimeout.BUILD
elif current_phase == OrchestStrategyPhase.APPLY:
timeout = OrchestrationPhaseTimeout.APPLY
check_interval = 40
else:
timeout = OrchestrationPhaseTimeout.ABORT
end_time = time.time() + timeout
output = None
prev_phase_completion = "0%"
if conn_ssh is None:
conn_ssh = ControllerClient.get_active_controller()
while time.time() < end_time:
if not conn_ssh.is_connected():
# ssh connection is lost. Controllers may swact in path application.
time.sleep(30)
conn_ssh.connect(retry=True, retry_timeout=HostTimeout.SWACT - 30)
time.sleep(60)
end_time = end_time + HostTimeout.SWACT
output = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)
if output:
if current_phase == OrchestStrategyPhase.ABORT:
if output[OrchStrategyKey.
STATE] == OrchStrategyState.ABORT_TIMEOUT:
msg = '{} strategy abort timed out. Stop waiting for completion.'.format(
orchestration)
if fail_ok:
LOG.warning(msg)
return False, output
else:
raise exceptions.OrchestrationError(msg)
elif output[OrchStrategyKey.
CURRENT_PHASE] == OrchestStrategyPhase.ABORT:
msg = "{} strategy {} phase is aborted. Stop waiting for completion."\
.format(orchestration, current_phase)
if fail_ok:
LOG.warn(msg)
return False, output
else:
raise exceptions.OrchestrationError(msg)
phase_completion = output['current-phase-completion']
if phase_completion != prev_phase_completion:
LOG.info(
"Orchestration current phase completion is at {}".format(
phase_completion))
prev_phase_completion = phase_completion
if phase_completion == '100%':
return True, output
else:
time.sleep(check_interval)
else:
msg = "{} strategy {} phase did not complete within automation timeout {}seconds. Current status: {}"\
.format(orchestration, current_phase, timeout, output)
if fail_ok:
LOG.warn(msg)
return False, output
else:
raise exceptions.OrchestrationError(msg)
def test_sriov_robustness(self, sriov_prep, add_admin_role_func):
"""
Exhaust all CPUs on one compute by spawning VMs with 2 SR-IOV interface
Args:
sriov_prep: test fixture to set up test environment and get proper pci nets/hosts
Setups:
- select two hosts configured with same pci-sriov providernet
- add the two hosts to cgcsauto aggregate to limit the vms host to the selected hosts
- Select one network under above providernet
Test Steps:
- Boot 2+ pci-sriov vms with pci-sriov vif over selected network onto same host
- Verify resource usage for providernet is increased as expected
- Lock vms host and ensure vms are all migrated to other host
- Verify vms' pci-sriov interfaces reachable and resource usage for pnet unchanged
- 'sudo reboot -f' new vms host, and ensure vms are evacuated to initial host
- Verify vms' pci-sriov interfaces reachable and resource usage for pnet unchanged
Teardown:
- Delete vms, volumes, flavor created
- Remove admin role to tenant
- Recover hosts if applicable
- Remove cgcsauto aggregate - class
"""
net_type, pci_net, pci_hosts, pnet_id, nics, initial_host, other_host, vfs_use_init, vm_num, vm_vcpus = \
sriov_prep
vif_model = 'pci-sriov'
# proc0_vm, proc1_vm = host_helper.get_logcores_counts(initial_host, functions='VMs')
# if system_helper.is_hyperthreading_enabled(initial_host):
# proc0_vm *= 2
# proc1_vm *= 2
# vm_vcpus = int(min(proc1_vm, proc0_vm) / (vm_num/2))
# Create flavor with calculated vcpu number
LOG.tc_step(
"Create a flavor with dedicated cpu policy and {} vcpus".format(
vm_vcpus))
flavor_id = nova_helper.create_flavor(
name='dedicated_{}vcpu'.format(vm_vcpus), ram=1024,
vcpus=vm_vcpus)[1]
ResourceCleanup.add('flavor', flavor_id, scope='module')
extra_specs = {
FlavorSpec.CPU_POLICY: 'dedicated',
}
# FlavorSpec.PCI_NUMA_AFFINITY: 'preferred'} # LP1854516
nova_helper.set_flavor(flavor=flavor_id, **extra_specs)
# Boot vms with 2 {} vifs each, and wait for pingable
LOG.tc_step("Boot {} vms with 2 {} vifs each".format(
vm_num, vif_model))
vms = []
for i in range(vm_num):
sriov_nics = nics.copy()
sriov_nic2 = sriov_nics[-1].copy()
sriov_nic2['port-id'] = network_helper.create_port(
net_id=sriov_nic2.pop('net-id'),
vnic_type='direct',
name='sriov_port')[1]
sriov_nics.append(sriov_nic2)
LOG.info("Booting vm{}...".format(i + 1))
vm_id = vm_helper.boot_vm(flavor=flavor_id,
nics=sriov_nics,
cleanup='function',
vm_host=initial_host,
avail_zone='cgcsauto')[1]
vms.append(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
check_vm_pci_interface(vms=vms, net_type=net_type)
# TODO: feature unavailable atm. Update required
# vfs_use_post_boot = nova_helper.get_provider_net_info(pnet_id, field='pci_vfs_used')
# assert vfs_use_post_boot - vfs_use_init == vm_num * 2, "Number of PCI vfs used is not as expected"
HostsToRecover.add(pci_hosts)
LOG.tc_step("Lock host of {} vms: {}".format(vif_model, initial_host))
host_helper.lock_host(host=initial_host, check_first=False, swact=True)
LOG.tc_step(
"Check vms are migrated to other host: {}".format(other_host))
for vm in vms:
vm_host = vm_helper.get_vm_host(vm_id=vm)
assert other_host == vm_host, "VM did not move to {} after locking {}".format(
other_host, initial_host)
check_vm_pci_interface(vms,
net_type=net_type,
ping_timeout=VMTimeout.DHCP_RETRY)
# TODO: feature unavailable atm. Update required
# vfs_use_post_lock = nova_helper.get_provider_net_info(pnet_id, field='pci_vfs_used')
# assert vfs_use_post_boot == vfs_use_post_lock, "Number of PCI vfs used after locking host is not as expected"
LOG.tc_step("Unlock {}".format(initial_host))
host_helper.unlock_host(initial_host)
LOG.tc_step("Reboot {} and ensure vms are evacuated to {}".format(
other_host, initial_host))
vm_helper.evacuate_vms(other_host,
vms,
post_host=initial_host,
wait_for_host_up=True)
check_vm_pci_interface(vms, net_type=net_type)
def delete_test_pod():
LOG.info("Delete {} pod if exists".format(POD_NAME))
kube_helper.delete_resources(resource_names=POD_NAME, fail_ok=True)
def test_pcipt_robustness(self, pcipt_prep):
"""
TC3_robustness: PCI-passthrough by locking and rebooting pci_vm host
Args:
pcipt_prep: test fixture to set up test environment and get proper pci nets/hosts/seg_id
Setups:
- select a providernet with pcipt interfaces configured
- get pci hosts configured with same above providernet
- get one network under above providernet (or two for CX4 nic)
Test Steps:
- Boot 2 pcipt vms with pci-passthrough vif over selected network
- Verify resource usage for providernet is increased as expected
- Lock pci_vm host and ensure vm migrated to other host (or fail to lock if no other pcipt host available)
- (Delete above tested pcipt vm if only two pcipt hosts available)
- Lock host for another pcipt vm, and lock is successful
- Verify vms' pci-pt interfaces reachable and resource usage for pnet as expected
- 'sudo reboot -f' pci_vm host, and ensure vm evacuated or up on same host if no other pcipt host available
- Repeat above step for another pcipt vm
- Verify vms' pci-pt interfaces reachable and resource usage for pnet unchanged
Teardown:
- Delete vms, volumes, flavor created
- Recover hosts if applicable
"""
net_type, pci_net_name, pci_hosts, pnet_id, nics, min_vcpu_host, seg_id, vm_num, vm_vcpus, pfs_use_init = \
pcipt_prep
vif_model = 'pci-passthrough'
# Create flavor with calculated vcpu number
LOG.fixture_step(
"Create a flavor with dedicated cpu policy and {} vcpus".format(
vm_vcpus))
flavor_id = nova_helper.create_flavor(
name='dedicated_{}vcpu'.format(vm_vcpus), ram=1024,
vcpus=vm_vcpus)[1]
ResourceCleanup.add('flavor', flavor_id, scope='module')
extra_specs = {
FlavorSpec.CPU_POLICY: 'dedicated',
}
# FlavorSpec.PCI_NUMA_AFFINITY: 'preferred'} # LP1854516
nova_helper.set_flavor(flavor=flavor_id, **extra_specs)
# Boot vms with 2 {} vifs each, and wait for pingable
LOG.tc_step("Boot {} vms with 2 {} vifs each".format(
vm_num, vif_model))
vms = []
for i in range(vm_num):
LOG.info("Booting pci-passthrough vm{}".format(i + 1))
vm_id = vm_helper.boot_vm(flavor=flavor_id,
nics=nics,
cleanup='function')[1]
vms.append(vm_id)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id)
vm_helper.add_vlan_for_vm_pcipt_interfaces(vm_id,
seg_id,
init_conf=True)
# TODO: feature unavailable atm. Update required
# pfs_use_post_boot = nova_helper.get_provider_net_info(pnet_id, field='pci_pfs_used')
# resource_change = 2 if isinstance(seg_id, dict) else 1
# assert pfs_use_post_boot - pfs_use_init == vm_num * resource_change, "Number of PCI pfs used is not as expected"
check_vm_pci_interface(vms=vms, net_type=net_type)
HostsToRecover.add(pci_hosts)
# pfs_use_pre_action = pfs_use_post_boot
iter_count = 2 if len(pci_hosts) < 3 else 1
for i in range(iter_count):
if i == 1:
LOG.tc_step(
"Delete a pcipt vm and test lock and reboot pcipt host again for success pass"
)
vm_helper.delete_vms(vms=vms[1])
vms.pop()
# TODO: feature unavailable atm. Update required
# pfs_use_pre_action -= resource_change
# common.wait_for_val_from_func(expt_val=pfs_use_pre_action, timeout=30, check_interval=3,
# func=nova_helper.get_provider_net_info,
# providernet_id=pnet_id, field='pci_pfs_used')
LOG.tc_step("Test lock {} vms hosts started - iter{}".format(
vif_model, i + 1))
for vm in vms:
pre_lock_host = vm_helper.get_vm_host(vm)
assert pre_lock_host in pci_hosts, "VM is not booted on pci_host"
LOG.tc_step("Lock host of {} vms: {}".format(
vif_model, pre_lock_host))
code, output = host_helper.lock_host(host=pre_lock_host,
check_first=False,
swact=True,
fail_ok=True)
post_lock_host = vm_helper.get_vm_host(vm)
assert post_lock_host in pci_hosts, "VM is not on pci host after migrating"
if len(pci_hosts) < 3 and i == 0:
assert 5 == code, "Expect host-lock fail due to migration of vm failure. Actual: {}".format(
output)
assert pre_lock_host == post_lock_host, "VM host should not change when no other host to migrate to"
else:
assert 0 == code, "Expect host-lock successful. Actual: {}".format(
output)
assert pre_lock_host != post_lock_host, "VM host did not change"
LOG.tc_step("Unlock {}".format(pre_lock_host))
check_vm_pci_interface(vms, net_type=net_type)
host_helper.unlock_host(pre_lock_host, available_only=True)
# TODO: feature unavailable atm. Update required
# pfs_use_post_lock = nova_helper.get_provider_net_info(pnet_id, field='pci_pfs_used')
# assert pfs_use_pre_action == pfs_use_post_lock, "Number of PCI pfs used after host-lock is not as expected"
LOG.tc_step("Test evacuate {} vms started - iter{}".format(
vif_model, i + 1))
for vm in vms:
pre_evac_host = vm_helper.get_vm_host(vm)
LOG.tc_step(
"Reboot {} and ensure {} vm are evacuated when applicable".
format(pre_evac_host, vif_model))
code, output = vm_helper.evacuate_vms(pre_evac_host,
vm,
fail_ok=True,
wait_for_host_up=True)
if len(pci_hosts) < 3 and i == 0:
assert 1 == code, "Expect vm stay on same host due to migration fail. Actual:{}".format(
output)
vm_helper.wait_for_vm_status(vm_id=vm)
else:
assert 0 == code, "Expect vm evacuated to other host. Actual: {}".format(
output)
post_evac_host = vm_helper.get_vm_host(vm)
assert post_evac_host in pci_hosts, "VM is not on pci host after evacuation"
check_vm_pci_interface(vms, net_type=net_type)
def test_boot_vm(name, flavor, source, source_name):
vm_id = vm_helper.boot_vm(name=name,
flavor=flavor,
source=source,
source_id=source_name)[1]
LOG.info("VM ID: {}".format(vm_id))
def test_system_upgrade_simplex(upgrade_setup,
check_system_health_query_upgrade):
"""
This script starts the upgrade with creating a backup file which is wipes the disk at the end of the execution .
to complete the upgrade test_upgrade_simplex_restore.py need to be executed with the backup file path.
Args:
upgrade_setup: This will check parameters ftp upload load and patches
check_system_health_query_upgrade: Check the health of system for upgrade
Example
To Execute
check_system_health_query_upgrade: Checks the upgrade health .
steps:
1. FTP load and patches and loads to system.
2. Checks the health of the upgrade
3. Start upgrade
4. Checks the backup files.
5. Backup the volume and images
6. Execute host-upgrade
7. Ftp backup files
teardown:
flush ssh.
"""
lab = upgrade_setup['lab']
current_version = upgrade_setup['current_version']
upgrade_version = upgrade_setup['upgrade_version']
if not system_helper.is_aio_simplex():
assert False, "This lab is not simplex to start upgrade"
force = False
controller0 = lab['controller-0']
backup_dest_path = BackupVars.get_backup_var('BACKUP_DEST_PATH')
backup_dest_full_path = '{}/{}/'.format(backup_dest_path,
lab['short_name'])
date = time.strftime(BACKUP_FILE_DATE_STR)
build_id = system_helper.get_build_info()['BUILD_ID']
lab_system_name = lab['name']
backup_file_name = "{}{}_{}_{}".format(PREFIX_BACKUP_FILE, date, build_id,
lab_system_name)
print('Backup_File_Name', backup_file_name)
# ssh to test server
test_server_attr = dict()
test_server_attr['name'] = TestFileServer.get_hostname().split('.')[0]
test_server_attr['server_ip'] = TestFileServer.get_server()
test_server_attr['prompt'] = r'\[{}@{} {}\]\$ ' \
.format(TestFileServer.get_user(), test_server_attr['name'], TestFileServer.get_user())
test_server_conn = install_helper.establish_ssh_connection(
test_server_attr['name'],
user=TestFileServer.get_user(),
password=TestFileServer.get_password(),
initial_prompt=test_server_attr['prompt'])
test_server_conn.set_prompt(test_server_attr['prompt'])
test_server_conn.deploy_ssh_key(install_helper.get_ssh_public_key())
test_server_attr['ssh_conn'] = test_server_conn
test_server_obj = Server(**test_server_attr)
dest_server = test_server_obj
# test if backup path for the lab exist in Test server
if test_server_conn.exec_cmd(
"test -e {}".format(backup_dest_full_path))[0]:
test_server_conn.exec_cmd("mkdir -p {}".format(backup_dest_full_path))
LOG.tc_step("Checking system health for upgrade .....")
if check_system_health_query_upgrade[0] == 0:
LOG.info("System health OK for upgrade......")
if check_system_health_query_upgrade[0] == 1:
assert False, "System health query upgrade failed: {}".format(
check_system_health_query_upgrade[1])
if check_system_health_query_upgrade[
0] == 3 or check_system_health_query_upgrade[0] == 2:
LOG.info(
"System health indicate minor alarms; using --force option to start upgrade......"
)
force = True
vol_ids = cinder_helper.get_volumes(auth_info=Tenant.get('admin'))
if len(vol_ids) > 0:
LOG.info("Exporting cinder volumes: {}".format(vol_ids))
exported = install_helper.export_cinder_volumes(
backup_dest='local',
backup_dest_path=backup_dest_full_path,
dest_server=dest_server)
assert len(exported) > 0, "Fail to export all volumes"
assert len(exported) == len(
vol_ids), "Some volumes failed export: {}".format(
set(vol_ids) - set(exported))
else:
LOG.info(
"No cinder volumes are avaialbe in the system; skipping cinder volume export..."
)
LOG.tc_step("Starting upgrade from release {} to target release {}".format(
current_version, upgrade_version))
upgrade_helper.system_upgrade_start(force=force)
upgrade_helper.wait_for_upgrade_states('started',
timeout=1360,
check_interval=30,
fail_ok=True)
LOG.info("upgrade started successfully......")
# scp backup files to test server
LOG.tc_step("SCP system and image tgz file into test server {} ",
backup_dest_full_path)
source_file = '/opt/backups/upgrade_data_*system.tgz '
backup_dest_full_path_image = backup_dest_full_path
backup_dest_full_path = backup_dest_full_path + "/" + backup_file_name + "_system.tgz"
common.scp_from_active_controller_to_test_server(source_file,
backup_dest_full_path,
is_dir=False)
backup_dest_full_path_image = backup_dest_full_path_image + "/" + backup_file_name + "_images.tgz"
source_file = '/opt/backups/upgrade_data_*images.tgz '
common.scp_from_active_controller_to_test_server(
source_file, backup_dest_full_path_image, is_dir=False)
LOG.info("Starting {} upgrade.....".format(controller0.name))
# Below line will wipe disk
# upgrade_helper.upgrade_host(controller0.name, lock=True)
LOG.tc_step(
"Host Upgrade executed .This will wipe the disk reboot controller-0 .")
time.sleep(3)
# open vlm console for controller-0 for boot through mgmt interface
LOG.info(
"Upgrade simpelx backup is complete . Resotore script should be run on this backup to compelte upgrade "
)
def barr_func(func_num, rep, barrier):
barrier.wait(10)
for i in range(0, rep):
LOG.info("function #{}".format(func_num))
sleep(1)
def __remove_or_add_hosts_in_aggregate(aggregate,
hosts=None,
remove=False,
check_first=True,
fail_ok=False,
con_ssh=None,
auth_info=Tenant.get('admin')):
"""
Remove/Add hosts from/to given aggregate
Args:
aggregate (str): name of the aggregate to add/remove hosts. cgcsauto aggregate can be added
via add_cgcsauto_zone session fixture
hosts (list|str):
remove (bool): True if remove hosts from given aggregate, otherwise add hosts to aggregate
check_first (bool):
fail_ok (bool):
con_ssh (SSHClient):
auth_info (dict):
Returns (tuple):
(0, "Hosts successfully removed from aggregate")
(1, <stderr>) cli rejected on at least one host
(2, "Host(s) still exist in aggregate <aggr> after aggregate-remove-host: <unremoved_hosts>)
"""
hosts_in_aggregate = get_hosts_in_aggregate(aggregate, con_ssh=con_ssh)
if hosts is None:
if remove:
hosts = hosts_in_aggregate
else:
from keywords import host_helper
hosts = host_helper.get_hypervisors()
if isinstance(hosts, str):
hosts = [hosts]
msg_str = 'Remov' if remove else 'Add'
LOG.info("{}ing hosts {} in aggregate {}".format(msg_str, hosts,
aggregate))
if check_first:
if remove:
hosts_to_rm_or_add = list(set(hosts) & set(hosts_in_aggregate))
else:
hosts_to_rm_or_add = list(set(hosts) - set(hosts_in_aggregate))
else:
hosts_to_rm_or_add = list(hosts)
if not hosts_to_rm_or_add:
warn_str = 'No' if remove else 'All'
msg = "{} given host(s) in aggregate {}. Do nothing. Given hosts: {}; hosts in " \
"aggregate: {}".format(warn_str, aggregate, hosts, hosts_in_aggregate)
LOG.warning(msg)
return -1, msg
failed_res = {}
cmd = 'aggregate remove host' if remove else 'aggregate add host'
for host in hosts_to_rm_or_add:
args = '{} {}'.format(aggregate, host)
code, output = cli.openstack(cmd,
args,
ssh_client=con_ssh,
fail_ok=True,
auth_info=auth_info)
if code > 0:
failed_res[host] = output
if failed_res:
err_msg = "'{}' is rejected for following host(s) in aggregate {}: {}".format(
cmd, aggregate, failed_res)
if fail_ok:
LOG.warning(err_msg)
return 1, err_msg
else:
raise exceptions.NovaError(err_msg)
post_hosts_in_aggregate = get_hosts_in_aggregate(aggregate,
con_ssh=con_ssh)
if remove:
failed_hosts = list(set(hosts) & set(post_hosts_in_aggregate))
else:
failed_hosts = list(set(hosts) - set(post_hosts_in_aggregate))
if failed_hosts:
err_msg = "{} accepted, but some host(s) are not {}ed in aggregate {}: {}".format(
cmd, msg_str, aggregate, failed_hosts)
if fail_ok:
LOG.warning(err_msg)
return 2, err_msg
else:
raise exceptions.NovaError(err_msg)
succ_msg = "Hosts successfully {}ed in aggregate {}: {}".format(
msg_str.lower(), aggregate, hosts)
LOG.info(succ_msg)
return 0, succ_msg
