def _late_setup(self):
self.logger.info('Checking maintenance mode')
vmstatus = vm_status.VmStatus()
status = vmstatus.get_status()
self.logger.debug('hosted-engine-status: {s}'.format(s=status))
if not status['global_maintenance']:
self.logger.error(
_('Please enable global maintenance mode before '
'upgrading or restoring'))
raise RuntimeError(_('Not in global maintenance mode'))
if self.environment[ohostedcons.CoreEnv.ROLLBACK_UPGRADE]:
if status['engine_vm_up']:
self.logger.error(
_('The engine VM seams up, please stop it before '
'restoring.'))
raise RuntimeError(_('Engine VM is up'))
else:
self.logger.info(_('The engine VM is down.'))
if self.environment[ohostedcons.CoreEnv.UPGRADING_APPLIANCE]:
cli = self.environment[ohostedcons.VDSMEnv.VDS_CLI]
response = cli.list()
self.logger.debug(response)
if response['status']['code'] == 0:
if 'items' in response:
vms = set(response['items'])
else:
vms = set([])
if self.environment[ohostedcons.VMEnv.VM_UUID] not in vms:
raise RuntimeError(
_('The engine VM is not running on this host'))
else:
self.logger.info('The engine VM is running on this host')
else:
raise RuntimeError(_('Unable to get VM list from VDSM'))
def _late_setup(self):
self.logger.info('Checking maintenance mode')
vmstatus = vm_status.VmStatus()
status = vmstatus.get_status()
self.logger.debug('hosted-engine-status: {s}'.format(s=status))
if not status['global_maintenance']:
self.logger.error(
_('Please enable global maintenance mode before '
'upgrading or restoring'))
raise RuntimeError(_('Not in global maintenance mode'))
if self.environment[ohostedcons.CoreEnv.ROLLBACK_UPGRADE]:
if status['engine_vm_up']:
self.logger.error(
_('The engine VM seams up, please stop it before '
'restoring.'))
raise RuntimeError(_('Engine VM is up'))
else:
self.logger.info(_('The engine VM is down.'))
if self.environment[ohostedcons.CoreEnv.UPGRADING_APPLIANCE]:
cli = self.environment[ohostedcons.VDSMEnv.VDS_CLI]
try:
vmList = cli.Host.getVMList()
self.logger.debug(vmList)
except ServerError as e:
raise RuntimeError(
_('Unable to get VM list from VDSM. Error: {m}').format(
m=str(e)))
vms = set(vmList)
if self.environment[ohostedcons.VMEnv.VM_UUID] not in vms:
raise RuntimeError(
_('The engine VM is not running on this host'))
self.logger.info('The engine VM is running on this host')
def engine_vm_up_check(self):
timeout = ohostedcons.Const.VM_LIVELINESS_CHECK_TIMEOUT
vmstatus = vm_status.VmStatus()
self.logger.info('Waiting for engine to start...')
health_good = False
while timeout >= 0:
try:
status = vmstatus.get_status()
for host in status['all_host_stats'].values():
if 'engine-status' in host:
if '"health": "good"' in host['engine-status']:
health_good = True
break
except (BrokerConnectionError, RuntimeError) as e:
self.logger.debug(str(e))
if health_good:
self.logger.info('Engine is up')
break
time.sleep(5)
timeout -= 5
if timeout % 60 == 0:
self.logger.info('Still waiting for engine to start...')
if timeout < 0:
self.logger.info(
'Engine vm is still powering up, please check the vm\'s '
'status using: \'hosted-engine --vm-status\''
)
def _validate_lm_volumes(self):
"""
This method, if the relevant uuids aren't in the initial answerfile,
will look for lockspace and metadata volumes on the shared
storage identifying them by their description.
We need to re-scan each time we run the upgrade flow since they
could have been created in a previous upgrade attempt.
If the volumes are not on disk, it triggers volume creation as for
fresh deployments; volume creation code will also remove the previous
file and create a new symlink to the volume using the same file name.
"""
self.logger.info(_('Scanning for lockspace and metadata volumes'))
cli = self.environment[ohostedcons.VDSMEnv.VDS_CLI]
img = image.Image(
self.environment[ohostedcons.StorageEnv.DOMAIN_TYPE],
self.environment[ohostedcons.StorageEnv.SD_UUID],
)
img_list = img.get_images_list(cli)
self.logger.debug('img list: {il}'.format(il=img_list))
sdUUID = self.environment[ohostedcons.StorageEnv.SD_UUID]
spUUID = ohostedcons.Const.BLANK_UUID
for img in img_list:
try:
volumeslist = cli.StorageDomain.getVolumes(
imageID=img,
storagepoolID=spUUID,
storagedomainID=sdUUID,
)
self.logger.debug('volumeslist: {vl}'.format(vl=volumeslist))
except ServerError as e:
# avoid raising here, simply skip the unknown image
self.logger.debug(
'Error fetching volumes for {image}: {message}'.format(
image=img,
message=str(e),
))
continue
for vol_uuid in volumeslist['items']:
try:
volumeinfo = cli.Volume.getInfo(
volumeID=vol_uuid,
imageID=img,
storagepoolID=spUUID,
storagedomainID=sdUUID,
)
self.logger.debug(volumeinfo)
except ServerError as e:
# avoid raising here, simply skip the unknown volume
self.logger.debug(('Error fetching volume info '
'for {volume}: {message}').format(
volume=vol_uuid,
message=str(e),
))
continue
disk_description = volumeinfo['description']
if disk_description == self.environment[
ohostedcons.SanlockEnv.LOCKSPACE_NAME] + '.lockspace':
self.environment[ohostedcons.StorageEnv.
LOCKSPACE_VOLUME_UUID] = vol_uuid
self.environment[
ohostedcons.StorageEnv.LOCKSPACE_IMAGE_UUID] = img
elif disk_description == self.environment[
ohostedcons.SanlockEnv.LOCKSPACE_NAME] + '.metadata':
self.environment[
ohostedcons.StorageEnv.METADATA_VOLUME_UUID] = vol_uuid
self.environment[
ohostedcons.StorageEnv.METADATA_IMAGE_UUID] = img
if (self.environment[ohostedcons.StorageEnv.LOCKSPACE_VOLUME_UUID] and
self.environment[ohostedcons.StorageEnv.METADATA_VOLUME_UUID]):
self.logger.info(
_('Lockspace and metadata volumes are already on the '
'HE storage domain'))
return
interactive = self.environment[
ohostedcons.Upgrade.LM_VOLUMES_UPGRADE_PROCEED] is None
if interactive:
self.environment[
ohostedcons.Upgrade.
LM_VOLUMES_UPGRADE_PROCEED] = self.dialog.queryString(
name=ohostedcons.Confirms.LM_VOLUMES_UPGRADE_PROCEED,
note=_(
'This system was initially deployed with oVirt 3.4 '
'using file based metadata and lockspace area.\n'
'Now you have to upgrade to up to date structure '
'using this tool.\n'
'In order to do that please manually stop ovirt-ha-agent '
'and ovirt-ha-broker on all the other HE hosts '
'(but not this one). '
'At the end you of this procedure you can simply '
'manually upgrade ovirt-hosted-engine-ha and '
'restart ovirt-ha-agent and ovirt-ha-broker on all '
'the hosted-engine hosts.\n'
'Are you sure you want to continue? '
'(@VALUES@)[@DEFAULT@]: '),
prompt=True,
validValues=(_('Yes'), _('No')),
caseSensitive=False,
default=_('Yes')) == _('Yes').lower()
if not self.environment[
ohostedcons.Upgrade.LM_VOLUMES_UPGRADE_PROCEED]:
raise otopicontext.Abort('Aborted by user')
self.logger.info(
_('Waiting for HA agents on other hosts to be stopped'))
vmstatus = vm_status.VmStatus()
ready = False
while not ready:
ready = True
status = vmstatus.get_status()
self.logger.debug('hosted-engine-status: {s}'.format(s=status))
for h in status['all_host_stats']:
host_id = status['all_host_stats'][h]['host-id']
hostname = status['all_host_stats'][h]['hostname']
if 'stopped' in status['all_host_stats'][h]:
stopped = status['all_host_stats'][h]['stopped']
if host_id == self.environment[
ohostedcons.StorageEnv.HOST_ID]:
if stopped:
self.logger.warning(
_('Please keep ovirt-ha-agent running '
'on this host'))
ready = False
else:
if not stopped:
self.logger.warning(
_('ovirt-ha-agent is still active on host {h}, '
'please stop it '
'(it can require a few seconds).').format(
h=hostname))
ready = False
else:
self.logger.warning(
_("Ignoring inconsistent info for host {h}".format(
h=hostname, )))
if not ready:
time.sleep(2)
self.environment[ohostedcons.Upgrade.UPGRADE_CREATE_LM_VOLUMES] = True
def _check_upgrade_requirements(self):
self.logger.info('Checking version requirements')
upg = upgrade.Upgrade()
if not upg.is_conf_file_uptodate():
self.logger.error(
_('Hosted-engine configuration didn\'t correctly reach 3.6 '
'level. Please successfully complete the upgrade to '
'3.6 before proceeding with this upgrade. '))
raise RuntimeError(
_('Unsupported hosted-engine configuration level'))
self.logger.info(_('Checking metadata area'))
vmstatus = vm_status.VmStatus()
status = vmstatus.get_status()
self.logger.debug('hosted-engine-status: {s}'.format(s=status))
old_metadata = False
for h in status['all_host_stats']:
if 'stopped' not in status['all_host_stats'][h]:
self.logger.error(
_('Metadata for host {h} is incompatible with this tool.\n'
'Before proceeding with this upgrade, '
'please correctly upgrade it to 3.6 '
'or clean its metadata area with\n'
' \'hosted-engine --clean-metadata --host-id={id}\'\n'
'if decommissioned or not anymore involved in HE.').
format(
h=status['all_host_stats'][h]['hostname'],
id=status['all_host_stats'][h]['host-id'],
))
old_metadata = True
if old_metadata:
raise RuntimeError(_('Host with unsupported metadata area'))
self.logger.info(
_('Hosted-engine configuration is at a compatible level'))
engine_api = engineapi.get_engine_api(self)
self.logger.debug('Successfully connected to the engine')
elements = engine_api.clusters.list() + engine_api.datacenters.list()
for e in elements:
if isinstance(e, brokers.DataCenter):
element_t = 'datacenter'
else:
element_t = 'cluster'
version = e.get_version()
release = '{ma}.{mi}'.format(
ma=version.major,
mi=version.minor,
)
if release not in ohostedcons.Const.UPGRADE_REQUIRED_CLUSTER_V:
self.logger.error(
_('{t} {name} is at version {release} which is not '
'supported by this upgrade flow. '
'Please fix it before upgrading.').format(
t=element_t.title(),
name=e.get_name(),
release=release,
))
raise RuntimeError(
_('Unsupported {t} level'.format(t=element_t)))
self.logger.info(
_('All the datacenters and clusters are at a compatible level'))
e_major = engine_api.get_product_info().version.major
e_minor = engine_api.get_product_info().version.minor
if not e_major:
# just for compatibility
# see: bz#1405386
e_major = engine_api.get_product_info().get_version().major
e_minor = engine_api.get_product_info().get_version().minor
if e_major is not None and e_minor is not None:
self._e_version = '{ma}.{mi}'.format(
ma=e_major,
mi=e_minor,
)