def remove_test_patches(failure_patch=False):
LOG.info("Delete patch orchestration strategy")
orchestration_helper.delete_strategy("patch")
applied = get_test_patches(state=(PatchState.PARTIAL_APPLY, PatchState.APPLIED))
if applied:
LOG.info("Remove applied test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
partial_removed = get_test_patches(state=PatchState.PARTIAL_REMOVE)
if partial_removed and not patching_helper.get_affecting_alarms():
LOG.info("Partial-Removed patches to be installed via patch orch: {}".format(partial_removed))
run_patch_orchestration_strategy(alarm_restrictions='relaxed')
# Install if needed
install_code, installed, failed = patching_helper.install_patches(remove=True, fail_ok=True)
available_patches = get_test_patches(state=PatchState.AVAILABLE)
if available_patches:
LOG.info("Delete test patches: {}".format(available_patches))
patching_helper.delete_patches(available_patches)
patching_helper.wait_for_affecting_alarms_gone()
if not failure_patch:
assert install_code <= 0, "Patches failed to install on removal: {}".format(failed)
def test_patch_orch_process(patch_orchestration_setup, patch_function_check, test_patch_type):
"""
Verifies apply/remove rr and in-service test patches through patch orchestration
Args:
patch_orchestration_setup:
patch_function_check
test_patch_type:
Returns:
"""
downloaded_patches, controllers, computes, storages = patch_orchestration_setup
vms = patch_function_check
test_patch_type = test_patch_type + '$' if 'A-C' in test_patch_type else test_patch_type
patches = patching_helper.parse_test_patches(downloaded_patches, search_str=test_patch_type)
if not patches:
skip("No patches with pattern {} available".format(test_patch_type))
patch_files = [downloaded_patches[patch] for patch in patches]
LOG.tc_step("Upload patch file {}".format(patch_files))
uploaded_ids = patching_helper.upload_patches(patch_files)[1]
LOG.info("Patch {} uploaded".format(uploaded_ids))
LOG.tc_step("Apply patch {}".format(uploaded_ids))
apply_all = False if 'A-C' in test_patch_type else True
applied = patching_helper.apply_patches(patch_ids=patches, apply_all=apply_all)[1]
LOG.info("Patch {} applied".format(applied))
compute_count = len(computes)
storage_count = len(storages)
compute_apply_type = 'parallel' if compute_count > 2 else 'serial'
max_parallel_computes = 4 if compute_count > 5 else 2
storage_apply_type = 'parallel' if storage_count / 2 >= 2 else 'serial'
LOG.tc_step("Install patches through orchestration.")
patching_helper.wait_for_affecting_alarms_gone()
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, alarm_restrictions='relaxed')
LOG.info("Install patch through orchestration completed for patches {}".format(applied))
LOG.tc_step("Check vms are in good state after installing patches {}".format(applied))
check_vms(vms)
LOG.tc_step("Remove test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
LOG.tc_step("Remove patches through orchestration: {}".format(applied))
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, alarm_restrictions='relaxed')
LOG.info("Patches successfully removed via orchestration: {}".format(applied))
LOG.tc_step("Check vms are in good state after removing patches {}".format(applied))
check_vms(vms)
def test_patch_orch_reject_with_alarms(patch_orchestration_setup, patch_function_check):
"""
This test verifies the patch orchestration operation can not proceed with presence of alarms that are not normally
ignored by the orchestration. The test generates the alarm ( 700.002 - VM paused) before executing the patch
orchestration.
Args:
patch_orchestration_setup:
patch_function_check
Returns:
"""
vms = patch_function_check
patches, controllers, computes, storages = patch_orchestration_setup
LOG.tc_step("Generate VM paused ( 700.002) critical alarm")
paused_vm, unpaused_vm = vms
vm_helper.pause_vm(paused_vm)
system_helper.wait_for_alarm(alarm_id='700.002')
patch = patching_helper.parse_test_patches(patch_ids=patches, search_str='RR_ALLNODES')[0]
patch_file = patches[patch]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0]
assert patch == uploaded_id, "Expected patch {} and uploaded patch {} mismatch"\
.format(patch, uploaded_id)
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(uploaded_id))
applied = patching_helper.apply_patches(patch_ids=[uploaded_id])[1]
LOG.info("Patch {} applied".format(applied))
LOG.tc_step("Attempt to create patch orchestration strategy; expected to fail")
rc, msg = orchestration_helper.create_strategy('patch', fail_ok=True)
assert rc != 0, "Patch orchestration strategy created with presence of critical alarm; expected to fail: {}"\
.format(msg)
LOG.info("Delete the failed patch orchestration strategy")
orchestration_helper.delete_strategy("patch")
LOG.tc_step("Remove test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
assert 0 == patching_helper.wait_for_patch_states(applied, expected_states=PatchState.AVAILABLE)[0]
LOG.tc_step("Un-pause vm after test patch removal, and check vms are in good state.")
vm_helper.unpause_vm(paused_vm)
vm_helper.wait_for_vm_pingable_from_natbox(paused_vm)
check_vms(vms)
def test_patch_orch_with_ignored_alarms(patch_orchestration_setup, patch_function_check, ignored_alarm_texts):
"""
This test verifies the patch orchestration operation with presence of alarms that are normally ignored by the
orchestration. These alarms are '200.001', '700.004,', '900.001', '900.005', '900.101'. This test generates the
alarms host lock (200.001) and VM stopped ( 700.004) before executing the patch orchestration.
Args:
patch_orchestration_setup:
patch_function_check
ignored_alarm_texts:
Returns:
"""
vms = patch_function_check
patches, controllers, computes, storages = patch_orchestration_setup
hosts = controllers + computes + storages
patch_id = patching_helper.parse_test_patches(patch_ids=patches, search_str='INSVC_ALLNODES')[0]
if 'HOST_LOCK' in ignored_alarm_texts and len(hosts) < 2:
skip("Not enough hosts present in the system")
if 'HOST_LOCK' in ignored_alarm_texts:
host = hosts[-1]
HostsToRecover.add(host)
LOG.info("Lock host {} to generate 200.001 alarm".format(host))
host_helper.lock_host(host)
system_helper.wait_for_alarm(alarm_id='200.001', fail_ok=False)
LOG.info("Host {} is locked and 200.001 alarm is generated".format(host))
vm_id_to_stop = None
if 'VM_STOP' in ignored_alarm_texts:
vm_id_to_stop = vms[0]
LOG.info("Stop VM {} to generate 700.004 alarm".format(vm_id_to_stop))
vm_helper.stop_vms(vm_id_to_stop)
system_helper.wait_for_alarm(alarm_id='700.004')
patch_file = patches[patch_id]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0]
assert patch_id == uploaded_id, "Expected patch {} and uploaded patch {} mismatch".format(patch_id, uploaded_id)
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(uploaded_id))
applied = patching_helper.apply_patches(patch_ids=[uploaded_id])[1]
LOG.info("Patch {} applied".format(applied))
LOG.tc_step("Install patch {} through orchestration".format(uploaded_id))
patching_helper.wait_for_affecting_alarms_gone()
run_patch_orchestration_strategy()
LOG.info("Install patch through orchestration completed for patch {}".format(applied))
host_helper.wait_for_hosts_ready(hosts=hosts)
LOG.tc_step("Check vms after patch is installed.")
if vm_id_to_stop:
vm_helper.start_vms(vm_id_to_stop)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_to_stop)
check_vms(vms)
LOG.tc_step("Remove test patch {}".format(applied))
if vm_id_to_stop:
vm_helper.stop_vms(vm_id_to_stop)
patching_helper.remove_patches(patch_ids=applied)
LOG.tc_step("Remove patch through orchestration: {}".format(applied))
run_patch_orchestration_strategy(alarm_restrictions='relaxed')
LOG.info("Apply/Remove through patch orchestration completed for patch {}".format(applied))
LOG.tc_step("Check vms after patch removed: {}.".format(applied))
if vm_id_to_stop:
vm_helper.start_vms(vm_id_to_stop)
vm_helper.wait_for_vm_pingable_from_natbox(vm_id_to_stop)
check_vms(vms)
def test_patch_orch_strategy(patch_orchestration_setup, storage_apply_type, patch_function_check,
compute_apply_type, max_parallel_computes, instance_action, test_patch):
"""
This test verifies the patch orchestration strategy options
Args:
patch_orchestration_setup:
patch_function_check
storage_apply_type:
compute_apply_type:
max_parallel_computes:
instance_action:
test_patch:
Returns:
"""
instance_action = instance_action.replace('_', '-')
vms = patch_function_check
patches, controllers, computes, storages = patch_orchestration_setup
if 'STORAGE' in test_patch and not system_helper.is_storage_system():
skip('Skip STORAGE patch test for non-storage system')
if "parallel"in storage_apply_type and len(storages) < 4:
skip("At least two pairs tier storage nodes required for this test: {}".format(storages))
if "parallel"in compute_apply_type and len(computes) < (max_parallel_computes + 1):
skip("At least {} computes are required for this test".format(1+max_parallel_computes))
patch_id = patching_helper.parse_test_patches(patch_ids=patches, search_str=test_patch)[0]
patch_file = patches[patch_id]
LOG.tc_step("Upload patch file {}".format(patch_file))
uploaded_id = patching_helper.upload_patches(patch_files=patch_file)[1][0]
assert patch_id == uploaded_id, "Expected patch {} and uploaded patch {} mismatch"\
.format(patch_id, uploaded_id)
LOG.info("Patch {} uploaded".format(uploaded_id))
LOG.tc_step("Apply patch {}".format(patch_id))
applied = patching_helper.apply_patches(patch_ids=[patch_id])[1]
assert applied == [patch_id]
LOG.info("Patch {} applied".format(patch_id))
LOG.tc_step("Install patches through orchestration for patch {}".format(applied))
patching_helper.wait_for_affecting_alarms_gone()
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, instance_action=instance_action,
alarm_restrictions='relaxed')
LOG.info("Install patch through orchestration completed for patch {}".format(applied))
time.sleep(20)
LOG.tc_step("Check vms after patch applied: {}".format(applied))
check_vms(vms)
LOG.tc_step("Remove test patch {}".format(applied))
patching_helper.remove_patches(patch_ids=applied)
partial_remove_ids = get_test_patches(state=PatchState.PARTIAL_REMOVE)
assert all(patch in partial_remove_ids for patch in applied), \
"Expected patch {} not in partial-remove state".format(applied)
LOG.tc_step("Remove patch through orchestration: {}".format(applied))
run_patch_orchestration_strategy(storage_apply_type=storage_apply_type, compute_apply_type=compute_apply_type,
max_parallel_computes=max_parallel_computes, instance_action=instance_action,
alarm_restrictions='relaxed')
LOG.info("Remove patch through orchestration completed for patch {}".format(applied))
LOG.tc_step("Check vms after patch removed: {}".format(applied))
check_vms(vms)