def test_set_global_maintenance(ansible_host0):
logging.info('Waiting For System Stability...')
he_utils.wait_until_engine_vm_is_not_migrating(ansible_host0)
he_utils.set_and_test_global_maintenance_mode(ansible_host0, True)
assert assert_utils.true_within_short(
lambda: he_utils.all_hosts_state_global_maintenance(ansible_host0))
logging.info('Global maintenance state set on all hosts')
def test_deactivate_storage_domain(engine_api):
# TODO: this also seems to leave running tasks behind which break the deactivation.
# TODO: it should be tested in multiple runs or properly waited for.
VnicSetup.vnic_setup().init(engine_api.system_service(), VM2_NAME, DC_NAME,
CLUSTER_NAME)
engine = engine_api.system_service()
dc = test_utils.data_center_service(engine_api.system_service(), DC_NAME)
correlation_id = 'deactivate_storage_domain'
def _deactivate_with_running_ovf_update_task():
try:
test_utils.get_attached_storage_domain(
dc, SD_SECOND_NFS_NAME, service=True).deactivate(
query={'correlation_id': correlation_id})
return True
except ovirtsdk4.Error as err:
# The storage domain's deactivation may fail if it has running tasks.
# In case of updating ovf_store disks task (UploadStream),
# ignore. Otherwise, raise the exception.
if not (('UploadStream' in err.args[0]) or ('OVF' in err.args[0])):
raise
return False
def _is_deactivation_job_finished():
deactivation_job_statuses = engine_utils.get_jobs_statuses(
engine, correlation_id)
return (ovirtsdk4.types.JobStatus.FINISHED in deactivation_job_statuses
) and not (ovirtsdk4.types.JobStatus.STARTED
in deactivation_job_statuses)
assert assert_utils.true_within_short(
_deactivate_with_running_ovf_update_task)
# Wait for the storage deactivation to be finished.
assert assert_utils.true_within_short(_is_deactivation_job_finished)
assert assert_utils.equals_within_short(
lambda: test_utils.get_attached_storage_domain(dc, SD_SECOND_NFS_NAME).
status,
ovirtsdk4.types.StorageDomainStatus.MAINTENANCE,
)
def restart():
ansible_engine.systemd(name='ovirt-engine', state='stopped')
ansible_engine.systemd(name='ovirt-engine', state='started')
health_url = 'http://{}/ovirt-engine/services/health'.format(
engine_fqdn)
def engine_is_alive():
engine_download(health_url)
return True
assert assert_utils.true_within_short(engine_is_alive,
allowed_exceptions=[ShellError])
def test_metrics_and_log_collector(setup_log_collector, suite_dir,
ansible_engine, ansible_hosts):
def configure():
try:
vt = utils.VectorThread(
[
functools.partial(configure_metrics, suite_dir,
ansible_engine, ansible_hosts),
functools.partial(run_log_collector, ansible_engine),
],
daemon=True,
)
vt.start_all()
vt.join_all(timeout=120)
except utils.TimeoutException:
LOGGER.debug("Metrics configuration timed out")
return False
return True
assert assert_utils.true_within_short(configure)
def he_status(ansible_host):
ret = {}
def get_value():
nonlocal ret
ansible_res = ansible_host.shell(
'hosted-engine --vm-status --json')['stdout']
try:
status = json.loads(ansible_res)
except ValueError:
raise RuntimeError(f'could not parse JSON: {ansible_res}')
# This outputs a dict whose keys are either numbers, and then the
# values are data about the corresponding host, or
# 'global_maintenance', and then the value is true or false.
# Put all hosts' data in a new item 'hosts' so that callers do not
# have to check isdigit themselves.
# Also: one of the items per host is 'extra', which is a string
# containing a newline-separated key=value list. Convert this
# to a dict as well.
result = {}
result['global_maintenance'] = status['global_maintenance']
result['hosts'] = {}
for i, data in status.items():
if i.isdigit():
# We use short hostnames, while HE may return FQDN.
# That happens in IPv6-only networks. So let's trim.
hostname = data['hostname'].split('.')[0]
result['hosts'][hostname] = data
result['hosts'][hostname]['extra'] = dict(
item.split('=') for item in data['extra'].split('\n')
if item)
ret = result
logging.debug(f'he_status: {ret}')
return ret
assert assert_utils.true_within_short(
lambda: bool(get_value()),
allowed_exceptions=[RuntimeError, AnsibleExecutionError],
)
return ret
def set_and_test_global_maintenance_mode(ansible_host, mode):
"""
Updates the global maintenance mode and tests if the value was stored.
Sometimes there is a race condition where the command that modifies the
global maintenance flag is ignored. That is why the maintenance mode is
updated repeatedly in a loop until it succeeds.
'mode' must be a bool value:
True - set maintenance mode to global
False - set maintenance mode to none
"""
def _set_and_test_global_maintenance_mode():
logging.debug('_set_and_test_global_maintenance_mode: Start')
ansible_host.shell('hosted-engine '
'--set-maintenance '
'--mode={}'.format('global' if mode else 'none'))
logging.debug('_set_and_test_global_maintenance_mode: After setting')
return is_global_maintenance_mode(ansible_host) == mode
logging.info(f'set_and_test_global_maintenance_mode: Start, mode={mode}')
assert assert_utils.true_within_short(
_set_and_test_global_maintenance_mode)
def add_domain(system_service, sd_name, url):
target_server = sdk4.types.OpenStackImageProvider(
name=sd_name,
description=sd_name,
url=url,
requires_authentication=False,
)
try:
providers_service = system_service.openstack_image_providers_service()
providers_service.add(target_server)
glance = []
def get():
providers = [
provider for provider in providers_service.list()
if provider.name == sd_name
]
if not providers:
return False
instance = providers_service.provider_service(providers.pop().id)
if instance:
glance.append(instance)
return True
else:
return False
assert assert_utils.true_within_short(
func=get, allowed_exceptions=[sdk4.NotFoundError])
except (AssertionError, sdk4.NotFoundError):
# RequestError if add method was failed.
# AssertionError if add method succeed but we couldn't verify that
# glance was actually added
return None
return glance.pop()
def _start_he_vm(ansible_host):
logging.info('Starting the engine VM...')
ansible_host.shell('hosted-engine --vm-start')
logging.info('Waiting for the engine VM to be UP...')
assert assert_utils.true_within_short(lambda: he_utils.engine_vm_is_up(ansible_host))
logging.info('Engine VM is UP.')
def test_local_maintenance(hosts_service, get_vm_service_for_vm,
ansible_host0):
logging.info('Waiting For System Stability...')
he_utils.wait_until_engine_vm_is_not_migrating(ansible_host0)
vm_service = get_vm_service_for_vm(VM_HE_NAME)
he_host_id = vm_service.get().host.id
host_service = hosts_service.host_service(id=he_host_id)
host_name = host_service.get().name
logging.info(f'Performing Deactivation on {host_name}...')
def _do_deactivate():
logging.debug(f'Trying to deactivate host {host_name}')
try:
host_service.deactivate()
except ovirtsdk4.Error:
return False
return True
assert assert_utils.true_within_short(_do_deactivate)
def _is_in_maintenance():
logging.debug(f'Checking if host {host_name} is in maintenance')
status = host_service.get().status
hosted_engine = host_service.get(all_content=True).hosted_engine
logging.debug(f'status={status}')
logging.debug(f'hosted_engine={_hosted_engine_info(hosted_engine)}')
# Original test was:
# (
# status == types.HostStatus.MAINTENANCE or
# hosted_engine.local_maintenance
# )
# But this does not test local_maintenance (presumably the "local
# maintenance" status as reported by the HA daemons?).
# So I tried to change the "or" to "and" (require both), and it
# never happened - local_maintenance always remained False.
# Giving up on this for now and checking only status.
# TODO: Find out why, fix what's needed, change the code to require
# both. Also for do_verified_activation below.
return status == types.HostStatus.MAINTENANCE
assert assert_utils.true_within_long(_is_in_maintenance)
logging.info('Performing Activation...')
def _do_activate():
logging.debug(f'Trying to activate host {host_name}')
try:
host_service.activate()
except ovirtsdk4.Error:
return False
return True
assert assert_utils.true_within_short(_do_activate)
def _is_active():
logging.info(f'Checking if host {host_name} is active')
status = host_service.get().status
hosted_engine = host_service.get(all_content=True).hosted_engine
logging.debug(f'status={status}')
logging.debug(f'hosted_engine={_hosted_engine_info(hosted_engine)}')
# TODO See comment above
return status == types.HostStatus.UP
assert assert_utils.true_within_long(_is_active)
logging.info('Verifying that all hosts have score higher than 0...')
assert assert_utils.true_within_long(
lambda: host_service.get(all_content=True).hosted_engine.score > 0)
logging.info('Validating Migration...')
prev_host_id = he_host_id
he_host_id = vm_service.get().host.id
assert prev_host_id != he_host_id