def update_controller_state():
mydbapi = dbapi.get_instance()
LOG.info("Updating upgrades data in sysinv database")
hostname = constants.CONTROLLER_1_HOSTNAME
if system_mode == constants.SYSTEM_MODE_SIMPLEX:
hostname = constants.CONTROLLER_0_HOSTNAME
host = mydbapi.ihost_get_by_hostname(hostname)
# Update the states for controller-1
update_values = {
'administrative': constants.ADMIN_UNLOCKED,
'operational': constants.OPERATIONAL_ENABLED,
'availability': constants.AVAILABILITY_AVAILABLE
}
mydbapi.ihost_update(host.uuid, update_values)
# Update the from and to load for controller-1
loads = mydbapi.load_get_list()
target_load = utils.get_imported_load(loads)
host_upgrade = mydbapi.host_upgrade_get_by_host(host.id)
update_values = {
'software_load': target_load.id,
'target_load': target_load.id
}
mydbapi.host_upgrade_update(host_upgrade.id, update_values)
# Update the upgrade state
upgrade = mydbapi.software_upgrade_get_one()
upgrade_update = {'state': constants.UPGRADE_UPGRADING_CONTROLLERS}
mydbapi.software_upgrade_update(upgrade.uuid, upgrade_update)
def post(self, body):
"""Create a new Software Upgrade instance and start upgrade."""
# Only start the upgrade from controller-0
if socket.gethostname() != constants.CONTROLLER_0_HOSTNAME:
raise wsme.exc.ClientSideError(
_("upgrade-start rejected: An upgrade can only be started "
"when %s is active." % constants.CONTROLLER_0_HOSTNAME))
# There must not already be an upgrade in progress
try:
pecan.request.dbapi.software_upgrade_get_one()
except exception.NotFound:
pass
else:
raise wsme.exc.ClientSideError(
_("upgrade-start rejected: An upgrade is already in progress.")
)
# Determine the from_load and to_load
loads = pecan.request.dbapi.load_get_list()
from_load = cutils.get_active_load(loads)
from_version = from_load.software_version
to_load = cutils.get_imported_load(loads)
to_version = to_load.software_version
controller_0 = pecan.request.dbapi.ihost_get_by_hostname(
constants.CONTROLLER_0_HOSTNAME)
force = body.get('force', False) is True
try:
# Set the upgrade flag in VIM
# This prevents VM changes during the upgrade and health checks
if utils.get_system_mode() != constants.SYSTEM_MODE_SIMPLEX:
vim_api.set_vim_upgrade_state(controller_0, True)
except Exception as e:
LOG.exception(e)
raise wsme.exc.ClientSideError(
_("upgrade-start rejected: Unable to set VIM upgrade state"))
success, output = pecan.request.rpcapi.get_system_health(
pecan.request.context, force=force, upgrade=True)
if not success:
LOG.info("Health audit failure during upgrade start. Health "
"query results: %s" % output)
if os.path.exists(constants.SYSINV_RUNNING_IN_LAB) and force:
LOG.info("Running in lab, ignoring health errors.")
else:
vim_api.set_vim_upgrade_state(controller_0, False)
raise wsme.exc.ClientSideError(
_("upgrade-start rejected: System is not in a valid state "
"for upgrades. Run system health-query-upgrade for more "
"details."))
# Create upgrade record. Must do this before the prepare_upgrade so
# the upgrade record exists when the database is dumped.
create_values = {
'from_load': from_load.id,
'to_load': to_load.id,
'state': constants.UPGRADE_STARTING
}
new_upgrade = None
try:
new_upgrade = pecan.request.dbapi.software_upgrade_create(
create_values)
except Exception as ex:
vim_api.set_vim_upgrade_state(controller_0, False)
LOG.exception(ex)
raise
# Prepare for upgrade
LOG.info("Starting upgrade from release: %s to release: %s" %
(from_version, to_version))
try:
pecan.request.rpcapi.start_upgrade(pecan.request.context,
new_upgrade)
except Exception as ex:
vim_api.set_vim_upgrade_state(controller_0, False)
pecan.request.dbapi.software_upgrade_destroy(new_upgrade.uuid)
LOG.exception(ex)
raise
return Upgrade.convert_with_links(new_upgrade)
def get_system_health_upgrade(self, context, force=False):
"""Ensures the system is in a valid state for an upgrade"""
# Does a general health check then does the following:
# A load is imported
# The load patch requirements are met
# The license is valid for the N+1 load
system_mode = self._dbapi.isystem_get_one().system_mode
simplex = (system_mode == constants.SYSTEM_MODE_SIMPLEX)
health_ok, output = self.get_system_health(context, force)
loads = self._dbapi.load_get_list()
try:
imported_load = utils.get_imported_load(loads)
except Exception as e:
LOG.exception(e)
output += _('No imported load found. Unable to test further\n')
return health_ok, output
upgrade_version = imported_load.software_version
if imported_load.required_patches:
patches = imported_load.required_patches.split('\n')
else:
patches = []
success, missing_patches = self._check_required_patches(patches)
output += _('Required patches are applied: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
if not success:
output += _('Patches not applied: %s\n') \
% ', '.join(missing_patches)
health_ok = health_ok and success
success = self._check_license(upgrade_version)
output += _('License valid for upgrade: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
health_ok = health_ok and success
if not simplex:
controller_1 = self._dbapi.ihost_get_by_hostname(
constants.CONTROLLER_1_HOSTNAME)
# If we are running on CPE we don't want any instances running
# on controller-1 before we start the upgrade, otherwise the
# databases will be out of sync after we lock controller-1
if constants.WORKER in controller_1.subfunctions:
success, running_instances = self._check_running_instances(
controller_1)
output += \
_('No instances running on controller-1: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
if not success:
output += _('Number of instances on controller-1: %s\n') \
% (running_instances)
health_ok = health_ok and success
else:
success = self._check_platform_backup_partition()
output += _('Valid platform-backup partition: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
health_ok = health_ok and success
return health_ok, output
def get_system_health_upgrade(self,
context,
force=False,
alarm_ignore_list=None):
"""
Ensures the system is in a valid state for an upgrade
:param context: request context.
:param force: set to true to ignore minor and warning alarms
:param alarm_ignore_list: list of alarm ids to ignore when performing
a health check
"""
# Does a general health check then does the following:
# A load is imported
# The load patch requirements are met
# The license is valid for the N+1 load
# All kubernetes applications are in a stable state
# Package metadata criteria are met
system_mode = self._dbapi.isystem_get_one().system_mode
simplex = (system_mode == constants.SYSTEM_MODE_SIMPLEX)
health_ok, output = self.get_system_health(
context, force=force, alarm_ignore_list=alarm_ignore_list)
loads = self._dbapi.load_get_list()
try:
imported_load = utils.get_imported_load(loads)
except Exception as e:
LOG.exception(e)
output += _('No imported load found. Unable to test further\n')
return health_ok, output
upgrade_version = imported_load.software_version
if imported_load.required_patches:
patches = imported_load.required_patches.split('\n')
else:
patches = []
success, missing_patches = \
self._check_required_patches_are_applied(patches)
output += _('Required patches are applied: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
if not success:
output += _('Patches not applied: %s\n') \
% ', '.join(missing_patches)
health_ok = health_ok and success
success = self._check_license(upgrade_version)
output += _('License valid for upgrade: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
health_ok = health_ok and success
if not simplex:
controller_1 = self._dbapi.ihost_get_by_hostname(
constants.CONTROLLER_1_HOSTNAME)
# If we are running on CPE we don't want any instances running
# on controller-1 before we start the upgrade, otherwise the
# databases will be out of sync after we lock controller-1
if constants.WORKER in controller_1.subfunctions:
success, running_instances = self._check_running_instances(
controller_1)
output += \
_('No instances running on controller-1: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
if not success:
output += _('Number of instances on controller-1: %s\n') \
% (running_instances)
health_ok = health_ok and success
else:
success = self._check_platform_backup_partition()
output += _('Valid platform-backup partition: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
health_ok = health_ok and success
success, apps_not_valid = self._check_kube_applications()
output += _(
'All kubernetes applications are in a valid state: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
if not success:
output += _('Kubernetes applications not in a valid state: %s\n') \
% ', '.join(apps_not_valid)
health_ok = health_ok and success
# The load is only imported to controller-0. An upgrade can only
# be started when controller-0 is active.
is_controller_0 = self._check_active_is_controller_0()
success = is_controller_0
output += \
_('Active controller is controller-0: [%s]\n') \
% (Health.SUCCESS_MSG if success else Health.FAIL_MSG)
health_ok = health_ok and success
return health_ok, output