def test_get_subordinate_units(self):
juju_status = mock.MagicMock()
juju_status.applications = {
'nova-compute': {
'units': {
'nova-compute/0': {
'subordinates': {
'neutron-openvswitch/2': {
'charm': 'cs:neutron-openvswitch-22'}}}}},
'cinder': {
'units': {
'cinder/1': {
'subordinates': {
'cinder-hacluster/0': {
'charm': 'cs:hacluster-42'},
'cinder-ceph/3': {
'charm': 'cs:cinder-ceph-2'}}}}},
}
self.model.get_status.return_Value = juju_status
self.assertEqual(
sorted(juju_utils.get_subordinate_units(
['nova-compute/0', 'cinder/1'],
status=juju_status)),
sorted(['neutron-openvswitch/2', 'cinder-hacluster/0',
'cinder-ceph/3']))
self.assertEqual(
juju_utils.get_subordinate_units(
['nova-compute/0', 'cinder/1'],
charm_name='ceph',
status=juju_status),
['cinder-ceph/3'])
def test_930_scaleback(self):
"""Remove a unit and add a new one."""
principle_units = sorted(zaza.model.get_status().applications[
self._principle_app_name]['units'].keys())
self.assertEqual(len(principle_units), 3)
doomed_principle_unit = principle_units[0]
other_principle_unit = principle_units[1]
doomed_hacluster_unit = juju_utils.get_subordinate_units(
[doomed_principle_unit], charm_name=self._hacluster_charm_name)[0]
other_hacluster_unit = juju_utils.get_subordinate_units(
[other_principle_unit], charm_name=self._hacluster_charm_name)[0]
logging.info('Pausing unit {}'.format(doomed_hacluster_unit))
zaza.model.run_action(doomed_hacluster_unit,
'pause',
raise_on_failure=True)
logging.info('OK')
logging.info('Removing {}'.format(doomed_principle_unit))
zaza.model.destroy_unit(self._principle_app_name,
doomed_principle_unit,
wait_disappear=True)
logging.info('OK')
logging.info('Waiting for model to settle')
zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked')
zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked')
zaza.model.block_until_all_units_idle()
logging.info('OK')
logging.info('Adding an hacluster unit')
zaza.model.add_unit(self._principle_app_name, wait_appear=True)
logging.info('OK')
logging.info('Waiting for model to settle')
zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active')
# NOTE(lourot): the principle application sometimes remain blocked
# after scaling back up until lp:1400481 is solved.
# zaza.model.block_until_unit_wl_status(other_principle_unit, 'active')
zaza.model.block_until_all_units_idle()
logging.debug('OK')
def action_upgrade_apps(applications, model_name=None):
"""Upgrade units in the applications using action managed upgrades.
Upgrade all units of the given applications using action managed upgrades.
This involves the following process:
1) Take a unit from each application which has not been upgraded yet.
2) Pause all hacluster units assocaiated with units to be upgraded.
3) Pause target units.
4) Upgrade target units.
5) Resume target units.
6) Resume hacluster units paused in step 2.
7) Repeat until all units are upgraded.
:param applications: List of application names.
:type applications: []
:param model_name: Name of model to query.
:type model_name: str
"""
status = zaza.model.get_status(model_name=model_name)
done = []
while True:
target = []
for app in applications:
for unit in zaza.model.get_units(app, model_name=model_name):
if unit.entity_id not in done:
target.append(unit.entity_id)
break
else:
logging.info("All units of {} upgraded".format(app))
if not target:
break
hacluster_units = juju_utils.get_subordinate_units(
target, 'hacluster', status=status, model_name=model_name)
# NOTE(lourot): we're more likely to time out while waiting for the
# action's result if we launch an action while the model is still
# executing. Thus it's safer to wait for the model to settle between
# actions.
zaza.model.block_until_all_units_idle(model_name)
pause_units(hacluster_units, model_name=model_name)
zaza.model.block_until_all_units_idle(model_name)
pause_units(target, model_name=model_name)
zaza.model.block_until_all_units_idle(model_name)
action_unit_upgrade(target, model_name=model_name)
zaza.model.block_until_all_units_idle(model_name)
resume_units(target, model_name=model_name)
zaza.model.block_until_all_units_idle(model_name)
resume_units(hacluster_units, model_name=model_name)
done.extend(target)
# Ensure that mysql-innodb-cluster has at least one R/W group (it can get
# into a state where all are R/O whilst it is sorting itself out after an
# openstack_upgrade
if "mysql-innodb-cluster" in applications:
block_until_mysql_innodb_cluster_has_rw(model_name)
# Now we need to wait for the model to go back to idle.
zaza.model.block_until_all_units_idle(model_name)
def test_930_scaleback(self):
"""Remove one unit, recalculate quorum and re-add one unit.
NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't
recalculated when removing units. So within a cluster of 3 units,
removing a unit and re-adding one led to a situation where corosync
considers having 3 nodes online out of 4, instead of just 3 out of 3.
This test covers this scenario.
"""
principle_units = sorted(zaza.model.get_status().applications[
self._principle_app_name]['units'].keys())
self.assertEqual(len(principle_units), 3)
surviving_principle_unit = principle_units[0]
doomed_principle_unit = principle_units[1]
surviving_hacluster_unit = juju_utils.get_subordinate_units(
[surviving_principle_unit],
charm_name=self._hacluster_charm_name)[0]
doomed_hacluster_unit = juju_utils.get_subordinate_units(
[doomed_principle_unit], charm_name=self._hacluster_charm_name)[0]
logging.info('Pausing unit {}'.format(doomed_hacluster_unit))
zaza.model.run_action(doomed_hacluster_unit,
'pause',
raise_on_failure=True)
logging.info('Removing {}'.format(doomed_principle_unit))
zaza.model.destroy_unit(self._principle_app_name,
doomed_principle_unit,
wait_disappear=True)
logging.info('Waiting for model to settle')
zaza.model.block_until_unit_wl_status(surviving_hacluster_unit,
'blocked')
# NOTE(lourot): the surviving principle units (usually keystone units)
# aren't guaranteed to be blocked, so we don't validate that here.
zaza.model.block_until_all_units_idle()
# At this point the corosync ring hasn't been updated yet, so it should
# still remember the deleted unit:
self.__assert_some_corosync_nodes_are_offline(surviving_hacluster_unit)
logging.info('Updating corosync ring')
hacluster_app_name = zaza.model.get_unit_from_name(
surviving_hacluster_unit).application
zaza.model.run_action_on_leader(
hacluster_app_name,
'update-ring',
action_params={'i-really-mean-it': True},
raise_on_failure=True)
# At this point if the corosync ring has been properly updated, there
# shouldn't be any trace of the deleted unit anymore:
self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit)
logging.info('Re-adding an hacluster unit')
zaza.model.add_unit(self._principle_app_name, wait_appear=True)
logging.info('Waiting for model to settle')
# NOTE(lourot): the principle charm may remain blocked here. This seems
# to happen often when it is keystone and has a mysql-router as other
# subordinate charm. The keystone units seems to often remain blocked
# with 'Database not initialised'. This is not the hacluster charm's
# fault and this is why we don't validate here that the entire model
# goes back to active/idle.
zaza.model.block_until_unit_wl_status(surviving_hacluster_unit,
'active')
zaza.model.block_until_all_units_idle()
# Because of lp:1874719 the corosync ring may show a mysterious offline
# 'node1' node. We clean up the ring by re-running the 'update-ring'
# action:
logging.info('Updating corosync ring - workaround for lp:1874719')
zaza.model.run_action_on_leader(
hacluster_app_name,
'update-ring',
action_params={'i-really-mean-it': True},
raise_on_failure=True)
# At this point the corosync ring should not contain any offline node:
self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit)