def test_patch_orch_process(patch_orchestration_setup, patch_function_check, test_patch_type):
"""
Verifies apply/remove rr and in-service test patches through patch orchestration
Args:
patch_orchestration_setup:
patch_function_check
test_patch_type:
Returns:
"""
downloaded_patches, controllers, computes, storages = patch_orchestration_setup
vms = patch_function_check
test_patch_type = test_patch_type + '$' if 'A-C' in test_patch_type else test_patch_type
patches = patching_helper.parse_test_patches(downloaded_patches, search_str=test_patch_type)
if not patches:
skip("No patches with pattern {} available".format(test_patch_type))
patch_files = [downloaded_patches[patch] for patch in patches]
LOG.tc_step("Upload patch file {}".format(patch_files))
uploaded_ids = patching_helper.upload_patches(patch_files)[1]
LOG.info("Patch {} uploaded".format(uploaded_ids))
LOG.tc_step("Apply patch {}".format(uploaded_ids))
apply_all = False if 'A-C' in test_patch_type else True
applied = patching_helper.apply_patches(patch_ids=patches, apply_all=apply_all)[1]
LOG.info("Patch {} applied".format(applied))
compute_count = len(computes)
storage_count = len(storages)
compute_apply_type = 'parallel' if compute_count > 2 else 'serial'
max_parallel_computes = 4 if compute_count > 5 else 2
storage_apply_type = 'parallel' if storage_count / 2 >= 2 else 'serial'
LOG.tc_step("Install patches through orchestration.")
patching_helper.wait_for_affecting_alarms_gone()
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, alarm_restrictions='relaxed')
LOG.info("Install patch through orchestration completed for patches {}".format(applied))
LOG.tc_step("Check vms are in good state after installing patches {}".format(applied))
check_vms(vms)
LOG.tc_step("Remove test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
LOG.tc_step("Remove patches through orchestration: {}".format(applied))
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, alarm_restrictions='relaxed')
LOG.info("Patches successfully removed via orchestration: {}".format(applied))
LOG.tc_step("Check vms are in good state after removing patches {}".format(applied))
check_vms(vms)
def test_patch_orch_reject_with_alarms(patch_orchestration_setup, patch_function_check):
"""
This test verifies the patch orchestration operation can not proceed with presence of alarms that are not normally
ignored by the orchestration. The test generates the alarm ( 700.002 - VM paused) before executing the patch
orchestration.
Args:
patch_orchestration_setup:
patch_function_check
Returns:
"""
vms = patch_function_check
patches, controllers, computes, storages = patch_orchestration_setup
LOG.tc_step("Generate VM paused ( 700.002) critical alarm")
paused_vm, unpaused_vm = vms
vm_helper.pause_vm(paused_vm)
system_helper.wait_for_alarm(alarm_id='700.002')
patch = patching_helper.parse_test_patches(patch_ids=patches, search_str='RR_ALLNODES')[0]
patch_file = patches[patch]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0]
assert patch == uploaded_id, "Expected patch {} and uploaded patch {} mismatch"\
.format(patch, uploaded_id)
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(uploaded_id))
applied = patching_helper.apply_patches(patch_ids=[uploaded_id])[1]
LOG.info("Patch {} applied".format(applied))
LOG.tc_step("Attempt to create patch orchestration strategy; expected to fail")
rc, msg = orchestration_helper.create_strategy('patch', fail_ok=True)
assert rc != 0, "Patch orchestration strategy created with presence of critical alarm; expected to fail: {}"\
.format(msg)
LOG.info("Delete the failed patch orchestration strategy")
orchestration_helper.delete_strategy("patch")
LOG.tc_step("Remove test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
assert 0 == patching_helper.wait_for_patch_states(applied, expected_states=PatchState.AVAILABLE)[0]
LOG.tc_step("Un-pause vm after test patch removal, and check vms are in good state.")
vm_helper.unpause_vm(paused_vm)
vm_helper.wait_for_vm_pingable_from_natbox(paused_vm)
check_vms(vms)
def test_patch_orch_failure(patch_orchestration_setup, failed_patch_setup, patch_type):
"""
This test verifies the patch orchestration operation with invalid or failure test patches. The patches are
expected to fail on applying the patch orchestration.
Args:
patch_orchestration_setup:
failed_patch_setup
patch_type:
Returns:
"""
vms = failed_patch_setup
downloaded_patches, controllers, computes, storages = patch_orchestration_setup
patch_id = patching_helper.parse_test_patches(downloaded_patches, search_str=patch_type, failure_patch=True)[0]
patch_file = downloaded_patches[patch_id]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_file)[1][0]
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(uploaded_id))
applied = patching_helper.apply_patches(patch_ids=uploaded_id, apply_all=True)[1]
LOG.info("Patch {} applied.".format(applied))
LOG.tc_step("Attempt to install the invalid patch {} through orchestration".format(applied))
patching_helper.wait_for_affecting_alarms_gone()
LOG.tc_step("Create patch orchestration strategy")
rc, msg = orchestration_helper.create_strategy('patch', fail_ok=True)
assert rc == 0, "Patch orchestration strategy create failed : {}".format(msg)
LOG.tc_step("Apply patch orchestration strategy with failure patch: {}".format(applied))
rc, msg = orchestration_helper.apply_strategy('patch', fail_ok=True)
assert rc != 0, "Patch orchestration strategy apply succeeded which expected to fail : {}".format(msg)
LOG.tc_step("Check vms are still in good state after apply patch failed.")
check_vms(vms)
def test_patch_orch_with_ignored_alarms(patch_orchestration_setup, patch_function_check, ignored_alarm_texts):
"""
This test verifies the patch orchestration operation with presence of alarms that are normally ignored by the
orchestration. These alarms are '200.001', '700.004,', '900.001', '900.005', '900.101'. This test generates the
alarms host lock (200.001) and VM stopped ( 700.004) before executing the patch orchestration.
Args:
patch_orchestration_setup:
patch_function_check
ignored_alarm_texts:
Returns:
"""
vms = patch_function_check
patches, controllers, computes, storages = patch_orchestration_setup
hosts = controllers + computes + storages
patch_id = patching_helper.parse_test_patches(patch_ids=patches, search_str='INSVC_ALLNODES')[0]
if 'HOST_LOCK' in ignored_alarm_texts and len(hosts) < 2:
skip("Not enough hosts present in the system")
if 'HOST_LOCK' in ignored_alarm_texts:
host = hosts[-1]
HostsToRecover.add(host)
LOG.info("Lock host {} to generate 200.001 alarm".format(host))
host_helper.lock_host(host)
system_helper.wait_for_alarm(alarm_id='200.001', fail_ok=False)
LOG.info("Host {} is locked and 200.001 alarm is generated".format(host))
vm_id_to_stop = None
if 'VM_STOP' in ignored_alarm_texts:
vm_id_to_stop = vms[0]
LOG.info("Stop VM {} to generate 700.004 alarm".format(vm_id_to_stop))
vm_helper.stop_vms(vm_id_to_stop)
system_helper.wait_for_alarm(alarm_id='700.004')
patch_file = patches[patch_id]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0]
assert patch_id == uploaded_id, "Expected patch {} and uploaded patch {} mismatch".format(patch_id, uploaded_id)
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(uploaded_id))
applied = patching_helper.apply_patches(patch_ids=[uploaded_id])[1]
LOG.info("Patch {} applied".format(applied))
LOG.tc_step("Install patch {} through orchestration".format(uploaded_id))
patching_helper.wait_for_affecting_alarms_gone()
run_patch_orchestration_strategy()
LOG.info("Install patch through orchestration completed for patch {}".format(applied))
host_helper.wait_for_hosts_ready(hosts=hosts)
LOG.tc_step("Check vms after patch is installed.")
if vm_id_to_stop:
vm_helper.start_vms(vm_id_to_stop)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_to_stop)
check_vms(vms)
LOG.tc_step("Remove test patch {}".format(applied))
if vm_id_to_stop:
vm_helper.stop_vms(vm_id_to_stop)
patching_helper.remove_patches(patch_ids=applied)
LOG.tc_step("Remove patch through orchestration: {}".format(applied))
run_patch_orchestration_strategy(alarm_restrictions='relaxed')
LOG.info("Apply/Remove through patch orchestration completed for patch {}".format(applied))
LOG.tc_step("Check vms after patch removed: {}.".format(applied))
if vm_id_to_stop:
vm_helper.start_vms(vm_id_to_stop)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_to_stop)
check_vms(vms)
def test_patch_orch_strategy(patch_orchestration_setup, storage_apply_type, patch_function_check,
compute_apply_type, max_parallel_computes, instance_action, test_patch):
"""
This test verifies the patch orchestration strategy options
Args:
patch_orchestration_setup:
patch_function_check
storage_apply_type:
compute_apply_type:
max_parallel_computes:
instance_action:
test_patch:
Returns:
"""
instance_action = instance_action.replace('_', '-')
vms = patch_function_check
patches, controllers, computes, storages = patch_orchestration_setup
if 'STORAGE' in test_patch and not system_helper.is_storage_system():
skip('Skip STORAGE patch test for non-storage system')
if "parallel"in storage_apply_type and len(storages) < 4:
skip("At least two pairs tier storage nodes required for this test: {}".format(storages))
if "parallel"in compute_apply_type and len(computes) < (max_parallel_computes + 1):
skip("At least {} computes are required for this test".format(1+max_parallel_computes))
patch_id = patching_helper.parse_test_patches(patch_ids=patches, search_str=test_patch)[0]
patch_file = patches[patch_id]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0]
assert patch_id == uploaded_id, "Expected patch {} and uploaded patch {} mismatch"\
.format(patch_id, uploaded_id)
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(patch_id))
applied = patching_helper.apply_patches(patch_ids=[patch_id])[1]
assert applied == [patch_id]
LOG.info("Patch {} applied".format(patch_id))
LOG.tc_step("Install patches through orchestration for patch {}".format(applied))
patching_helper.wait_for_affecting_alarms_gone()
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, instance_action=instance_action,
alarm_restrictions='relaxed')
LOG.info("Install patch through orchestration completed for patch {}".format(applied))
time.sleep(20)
LOG.tc_step("Check vms after patch applied: {}".format(applied))
check_vms(vms)
LOG.tc_step("Remove test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
partial_remove_ids = get_test_patches(state=PatchState.PARTIAL_REMOVE)
assert all(patch in partial_remove_ids for patch in applied), \
"Expected patch {} not in partial-remove state".format(applied)
LOG.tc_step("Remove patch through orchestration: {}".format(applied))
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, instance_action=instance_action,
alarm_restrictions='relaxed')
LOG.info("Remove patch through orchestration completed for patch {}".format(applied))
LOG.tc_step("Check vms after patch removed: {}".format(applied))
check_vms(vms)
def test_system_patch_orchestration(patch_orchestration_setup):
"""
This test verifies the patch orchestration operation procedures for release patches. The patch orchestration
automatically patches all hosts on a system in the following order: controllers, storages, and computes.
The test creates a patch orchestration strategy or plan for automated patching operation with the following
options to customize the test:
--controller-apply-type : specifies how controllers are patched serially or in parallel. By default controllers are
patched always in serial regardless of the selection.
--storage-apply-type : specifies how the storages are patched. Possible values are: serial, parallel or ignore. The
default value is serial.
--compute-apply-type : specifies how the computes are patched. Possible values are: serial, parallel or ignore. The
default value is serial.
--max-parallel-compute-hosts: specifies the maximum number of computes to patch in parallel. Possible values
[2 - 100]The default is 2.
--instance-action - For reboot-required patches, specifies how the VM instances are moved from compute hosts being
patched. Possible choices are:
start-stop - VMs are stopped before compute host is patched.
migrate - VMs are either live migrated or cold migrated off the compute before applying the patches.
Args:
patch_orchestration_setup:
Returns:
"""
lab = patch_orchestration_setup['lab']
patching_helper.check_system_health(check_patch_ignored_alarms=False)
LOG.info("Starting patch orchestration for lab {} .....".format(lab))
patches = patch_orchestration_setup['patches']
patch_ids = ' '.join(patches.keys())
LOG.tc_step("Uploading patches {} ... ".format(patch_ids))
patch_dest_dir = HostLinuxUser.get_home() + '/patches'
rc = patching_helper.run_patch_cmd('upload-dir', args=patch_dest_dir)[0]
assert rc in [0,
1], "Fail to upload patches in dir {}".format(patch_dest_dir)
uploaded = patching_helper.get_available_patches()
if rc == 0:
LOG.info("Patches uploaded: {}".format(uploaded))
else:
LOG.info("Patches are already in repo")
if len(uploaded) > 0:
LOG.tc_step("Applying patches ...")
uploaded_patch_ids = ' '.join(uploaded)
applied = patching_helper.apply_patches(
patch_ids=uploaded_patch_ids)[1]
LOG.info("Patches applied: {}".format(applied))
else:
LOG.info("No Patches are applied; Patches may be already applied: {}")
partial_patches_ids = patching_helper.get_patches_in_state(
(PatchState.PARTIAL_APPLY, PatchState.PARTIAL_REMOVE))
if len(partial_patches_ids) > 0:
current_alarms_ids = system_helper.get_alarms(mgmt_affecting=True,
combine_entries=False)
affecting_alarms = [
id_ for id_ in current_alarms_ids
if id_[0] not in orchestration_helper.IGNORED_ALARM_IDS
]
if len(affecting_alarms) > 0:
assert system_helper.wait_for_alarms_gone(alarms=affecting_alarms, timeout=240, fail_ok=True)[0],\
"Alarms present: {}".format(affecting_alarms)
LOG.tc_step("Installing patches through orchestration .....")
patching_helper.orchestration_patch_hosts(
controller_apply_type=patch_orchestration_setup[
'controller_apply_strategy'],
storage_apply_type=patch_orchestration_setup[
'storage_apply_strategy'],
compute_apply_type=patch_orchestration_setup[
'compute_apply_strategy'],
max_parallel_computes=patch_orchestration_setup[
'max_parallel_computes'],
instance_action=patch_orchestration_setup['instance_action'],
alarm_restrictions=patch_orchestration_setup['alarm_restrictions'])
LOG.info(
" Applying Patch orchestration strategy completed for {} ....".
format(partial_patches_ids))
LOG.tc_step("Deleting patches orchestration strategy .....")
delete_patch_strategy()
LOG.info("Deleted patch orchestration strategy .....")
else:
pytest.skip("All patches in patch-dir are already in system.")