def test_cache_device(self):
"""Test replacing a disk in use."""
logging.info('Running add-disk action with a caching device')
mon = next(iter(zaza_model.get_units('ceph-mon'))).entity_id
osds = [x.entity_id for x in zaza_model.get_units('ceph-osd')]
params = []
for unit in osds:
loop_dev = self.loop_devs[unit]
params.append({'unit': unit, 'device': loop_dev})
action_obj = zaza_model.run_action(unit_name=unit,
action_name='add-disk',
action_params={
'osd-devices': loop_dev,
'partition-size': 5
})
zaza_utils.assertActionRanOK(action_obj)
zaza_model.wait_for_application_states()
logging.info('Removing previously added disks')
for param in params:
osd_id = self.get_local_osd_id(param['unit'])
param.update({'osd-id': osd_id})
action_obj = zaza_model.run_action(unit_name=param['unit'],
action_name='remove-disk',
action_params={
'osd-ids': osd_id,
'timeout': 5,
'format': 'json',
'purge': False
})
zaza_utils.assertActionRanOK(action_obj)
results = json.loads(action_obj.data['results']['message'])
results = results[next(iter(results))]
self.assertEqual(results['osd-ids'], osd_id)
zaza_model.run_on_unit(param['unit'], 'partprobe')
zaza_model.wait_for_application_states()
logging.info('Recycling previously removed OSDs')
for param in params:
action_obj = zaza_model.run_action(unit_name=param['unit'],
action_name='add-disk',
action_params={
'osd-devices':
param['device'],
'osd-ids': param['osd-id'],
'partition-size': 4
})
zaza_utils.assertActionRanOK(action_obj)
zaza_model.wait_for_application_states()
self.assertEqual(len(osds) * 2, self.get_num_osds(mon))
# Finally, remove all the added OSDs that are backed by loop devices.
for param in params:
osd_id = self.get_local_osd_id(param['unit'])
zaza_model.run_action(unit_name=param['unit'],
action_name='remove-disk',
action_params={
'osd-ids': osd_id,
'purge': True
})
def osd_out_in(self, services):
"""Run OSD out and OSD in tests.
Remove OSDs and then add them back in on a unit checking that services
are in the required state after each action
:param services: Services expected to be restarted when config_file is
changed.
:type services: list
"""
zaza_model.block_until_service_status(self.lead_unit,
services,
'running',
model_name=self.model_name)
zaza_model.block_until_unit_wl_status(self.lead_unit,
'active',
model_name=self.model_name)
zaza_model.run_action(self.lead_unit,
'osd-out',
model_name=self.model_name)
zaza_model.block_until_unit_wl_status(self.lead_unit,
'maintenance',
model_name=self.model_name)
zaza_model.block_until_all_units_idle(model_name=self.model_name)
zaza_model.run_action(self.lead_unit,
'osd-in',
model_name=self.model_name)
zaza_model.block_until_unit_wl_status(self.lead_unit,
'active',
model_name=self.model_name)
zaza_model.block_until_all_units_idle(model_name=self.model_name)
zaza_model.block_until_service_status(self.lead_unit,
services,
'running',
model_name=self.model_name)
def test_run_action(self):
self.patch_object(model, 'Model')
self.patch_object(model, 'get_unit_from_name')
self.get_unit_from_name.return_value = self.unit1
self.Model.return_value = self.Model_mock
model.run_action('app/2', 'modelname', 'backup',
{'backup_dir': '/dev/null'})
self.unit1.run_action.assert_called_once_with(
'backup',
backup_dir='/dev/null')
def pause_resume(self, services, pgrep_full=False):
"""Run Pause and resume tests.
Pause and then resume a unit checking that services are in the
required state after each action
:param services: Services expected to be restarted when config_file is
changed.
:type services: list
:param pgrep_full: Should pgrep be used rather than pidof to identify
a service.
:type pgrep_full: bool
"""
model.block_until_service_status(
self.lead_unit,
services,
'running',
model_name=self.model_name,
pgrep_full=pgrep_full)
model.block_until_unit_wl_status(
self.lead_unit,
'active',
model_name=self.model_name)
model.run_action(
self.lead_unit,
'pause',
model_name=self.model_name)
model.block_until_unit_wl_status(
self.lead_unit,
'maintenance',
model_name=self.model_name)
model.block_until_all_units_idle(model_name=self.model_name)
model.block_until_service_status(
self.lead_unit,
services,
'stopped',
model_name=self.model_name,
pgrep_full=pgrep_full)
yield
model.run_action(
self.lead_unit,
'resume',
model_name=self.model_name)
model.block_until_unit_wl_status(
self.lead_unit,
'active',
model_name=self.model_name)
model.block_until_all_units_idle(model_name=self.model_name)
model.block_until_service_status(
self.lead_unit,
services,
'running',
model_name=self.model_name,
pgrep_full=pgrep_full)
def clear_hooks(self):
"""Clear and deferred hooks.
Run any deferred hooks.
"""
# Use action to run any deferred restarts
for unit in model.get_units(self.application_name):
logging.info("Running run-deferred-hooks on {}".format(
unit.entity_id))
model.run_action(
unit.entity_id,
'run-deferred-hooks',
raise_on_failure=True)
def test_ghost_nfs_share(self):
"""Ensure ghost-share action bind mounts NFS share."""
generic_utils.assertActionRanOK(
zaza_model.run_action(
self.lead_unit,
'ghost-share',
action_params={'nfs-shares': '10.20.0.1:/srv/testing'},
model_name=self.model_name))
def check_clear_restarts(self):
"""Clear and deferred restarts and check status.
Clear and deferred restarts and then check the workload status message
for each unit.
"""
# Use action to run any deferred restarts
for unit in model.get_units(self.application_name):
logging.info("Running restart-services on {}".format(
unit.entity_id))
model.run_action(
unit.entity_id,
'restart-services',
action_params={'deferred-only': True},
raise_on_failure=True)
# Check workload status no longer shows deferred restarts.
self.check_status_message_is_clear()
def test_blacklist(self):
"""Check the blacklist action.
The blacklist actions execute and behave as expected.
"""
logging.info('Checking blacklist-add-disk and'
'blacklist-remove-disk actions...')
unit_name = 'ceph-osd/0'
zaza_model.block_until_unit_wl_status(unit_name, 'active')
# Attempt to add device with non-absolute path should fail
action_obj = zaza_model.run_action(
unit_name=unit_name,
action_name='blacklist-add-disk',
action_params={'osd-devices': 'vda'})
self.assertTrue(action_obj.status != 'completed')
zaza_model.block_until_unit_wl_status(unit_name, 'active')
# Attempt to add device with non-existent path should fail
action_obj = zaza_model.run_action(
unit_name=unit_name,
action_name='blacklist-add-disk',
action_params={'osd-devices': '/non-existent'})
self.assertTrue(action_obj.status != 'completed')
zaza_model.block_until_unit_wl_status(unit_name, 'active')
# Attempt to add device with existent path should succeed
action_obj = zaza_model.run_action(
unit_name=unit_name,
action_name='blacklist-add-disk',
action_params={'osd-devices': '/dev/vda'})
self.assertEqual('completed', action_obj.status)
zaza_model.block_until_unit_wl_status(unit_name, 'active')
# Attempt to remove listed device should always succeed
action_obj = zaza_model.run_action(
unit_name=unit_name,
action_name='blacklist-remove-disk',
action_params={'osd-devices': '/dev/vda'})
self.assertEqual('completed', action_obj.status)
zaza_model.block_until_unit_wl_status(unit_name, 'active')
logging.debug('OK')
def test_501_security_checklist_action(self):
"""Verify expected result on a default install.
Ported from amulet tests.
"""
logging.info("Testing security-checklist")
unit_name = zaza_model.get_lead_unit_name('openstack-dashboard')
action = zaza_model.run_action(unit_name, 'security-checklist')
assert action.data.get(u"status") == "failed", \
"Security check is expected to not pass by default"
def run_show_deferred_events_action(self):
"""Run show-deferred-events and return results.
:returns: Data from action run
:rtype: Dict
"""
unit = model.get_units(self.application_name)[0]
action = model.run_action(
unit.entity_id,
'show-deferred-events',
raise_on_failure=True)
return yaml.safe_load(action.data['results']['output'])
def test_update_trilio_action(self):
"""Test that the action runs succesfully."""
action_name = 'update-trilio'
actions = zaza_model.get_actions(self.application_name)
if action_name not in actions:
raise unittest.SkipTest(
'Action {} not defined'.format(action_name))
generic_utils.assertActionRanOK(
zaza_model.run_action(self.lead_unit,
action_name,
action_params={},
model_name=self.model_name))
def pause_resume(self, services):
"""Run Pause and resume tests.
Override the default implementation since pausing ceph units
doesn't stop the services.
Pause and then resume a unit checking that services are in the
required state after each action
:param services: Services expected to be restarted when config_file is
changed.
:type services: list
"""
zaza_model.block_until_service_status(self.lead_unit,
services,
'running',
model_name=self.model_name)
zaza_model.block_until_unit_wl_status(self.lead_unit,
'active',
model_name=self.model_name)
zaza_model.run_action(self.lead_unit,
'pause',
model_name=self.model_name)
zaza_model.block_until_unit_wl_status(self.lead_unit,
'maintenance',
model_name=self.model_name)
zaza_model.block_until_all_units_idle(model_name=self.model_name)
zaza_model.run_action(self.lead_unit,
'resume',
model_name=self.model_name)
zaza_model.block_until_unit_wl_status(self.lead_unit,
'active',
model_name=self.model_name)
zaza_model.block_until_all_units_idle(model_name=self.model_name)
zaza_model.block_until_service_status(self.lead_unit,
services,
'running',
model_name=self.model_name)
def pause_resume(self, services):
"""Run Pause and resume tests.
Pause and then resume a unit checking that services are in the
required state after each action
:param services: Services expected to be restarted when config_file is
changed.
:type services: list
"""
model.block_until_service_status(self.lead_unit,
services,
'running',
model_name=self.model_name)
model.block_until_unit_wl_status(self.lead_unit,
'active',
model_name=self.model_name)
model.run_action(self.lead_unit, 'pause', model_name=self.model_name)
model.block_until_unit_wl_status(self.lead_unit,
'maintenance',
model_name=self.model_name)
model.block_until_all_units_idle(model_name=self.model_name)
model.block_until_service_status(self.lead_unit,
services,
'stopped',
model_name=self.model_name)
yield
model.run_action(self.lead_unit, 'resume', model_name=self.model_name)
model.block_until_unit_wl_status(self.lead_unit,
'active',
model_name=self.model_name)
model.block_until_all_units_idle(model_name=self.model_name)
model.block_until_service_status(self.lead_unit,
services,
'running',
model_name=self.model_name)
def test_list_disks(self):
"""Test the list-disks action.
The list-disks action execute.
"""
logging.info('Checking list-disks action...')
unit_name = 'ceph-osd/0'
zaza_model.block_until_unit_wl_status(unit_name, 'active')
action_obj = zaza_model.run_action(
unit_name=unit_name,
action_name='list-disks',
)
self.assertEqual('completed', action_obj.status)
zaza_model.block_until_unit_wl_status(unit_name, 'active')
logging.debug('OK')
def series_upgrade_application(application, pause_non_leader_primary=True,
pause_non_leader_subordinate=True,
from_series="trusty", to_series="xenial",
origin='openstack-origin',
completed_machines=[],
files=None, workaround_script=None,
post_upgrade_functions=None):
"""Series upgrade application.
Wrap all the functionality to handle series upgrade for a given
application. Including pausing non-leader units.
:param application: Name of application to upgrade series
:type application: str
:param pause_non_leader_primary: Whether the non-leader applications should
be paused
:type pause_non_leader_primary: bool
:param pause_non_leader_subordinate: Whether the non-leader subordinate
hacluster applications should be
paused
:type pause_non_leader_subordinate: bool
:param from_series: The series from which to upgrade
:type from_series: str
:param to_series: The series to which to upgrade
:type to_series: str
:param origin: The configuration setting variable name for changing origin
source. (openstack-origin or source)
:type origin: str
:param completed_machines: List of completed machines which do no longer
require series upgrade.
:type completed_machines: list
:param files: Workaround files to scp to unit under upgrade
:type files: list
:param workaround_script: Workaround script to run during series upgrade
:type workaround_script: str
:returns: None
:rtype: None
"""
status = model.get_status().applications[application]
# For some applications (percona-cluster) the leader unit must upgrade
# first. For API applications the non-leader haclusters must be paused
# before upgrade. Finally, for some applications this is arbitrary but
# generalized.
leader = None
non_leaders = []
for unit in status["units"]:
if status["units"][unit].get("leader"):
leader = unit
else:
non_leaders.append(unit)
# Pause the non-leaders
for unit in non_leaders:
if pause_non_leader_subordinate:
if status["units"][unit].get("subordinates"):
for subordinate in status["units"][unit]["subordinates"]:
_app = subordinate.split('/')[0]
if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST:
logging.info("Skipping pausing {} - blacklisted"
.format(subordinate))
else:
logging.info("Pausing {}".format(subordinate))
model.run_action(
subordinate, "pause", action_params={})
if pause_non_leader_primary:
logging.info("Pausing {}".format(unit))
model.run_action(unit, "pause", action_params={})
machine = status["units"][leader]["machine"]
# Series upgrade the leader
logging.info("Series upgrade leader: {}".format(leader))
if machine not in completed_machines:
series_upgrade(leader, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded."
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
set_origin(application, origin)
model.block_until_all_units_idle()
# Series upgrade the non-leaders
for unit in non_leaders:
machine = status["units"][unit]["machine"]
if machine not in completed_machines:
logging.info("Series upgrade non-leader unit: {}"
.format(unit))
series_upgrade(unit, machine,
from_series=from_series, to_series=to_series,
origin=origin, workaround_script=workaround_script,
files=files,
post_upgrade_functions=post_upgrade_functions)
completed_machines.append(machine)
else:
logging.info("Skipping unit: {}. Machine: {} already upgraded. "
"But setting origin on the application {}"
.format(unit, machine, application))
logging.info("Set origin on {}".format(application))
set_origin(application, origin)
model.block_until_all_units_idle()
def test_blocked_when_non_pristine_disk_appears(self):
"""Test blocked state with non-pristine disk.
Validate that charm goes into blocked state when it is presented with
new block devices that have foreign data on them.
Instances used in UOSCI has a flavour with ephemeral storage in
addition to the bootable instance storage. The ephemeral storage
device is partitioned, formatted and mounted early in the boot process
by cloud-init.
As long as the device is mounted the charm will not attempt to use it.
If we unmount it and trigger the config-changed hook the block device
will appear as a new and previously untouched device for the charm.
One of the first steps of device eligibility checks should be to make
sure we are seeing a pristine and empty device before doing any
further processing.
As the ephemeral device will have data on it we can use it to validate
that these checks work as intended.
"""
logging.info('Checking behaviour when non-pristine disks appear...')
logging.info('Configuring ephemeral-unmount...')
alternate_conf = {
'ephemeral-unmount': '/mnt',
'osd-devices': '/dev/vdb'
}
juju_service = 'ceph-osd'
zaza_model.set_application_config(juju_service, alternate_conf)
ceph_osd_states = {
'ceph-osd': {
'workload-status': 'blocked',
'workload-status-message': 'Non-pristine'
}
}
zaza_model.wait_for_application_states(states=ceph_osd_states)
logging.info('Units now in blocked state, running zap-disk action...')
unit_names = ['ceph-osd/0', 'ceph-osd/1', 'ceph-osd/2']
for unit_name in unit_names:
zap_disk_params = {
'devices': '/dev/vdb',
'i-really-mean-it': True,
}
action_obj = zaza_model.run_action(unit_name=unit_name,
action_name='zap-disk',
action_params=zap_disk_params)
logging.debug('Result of action: {}'.format(action_obj))
logging.info('Running add-disk action...')
for unit_name in unit_names:
add_disk_params = {
'osd-devices': '/dev/vdb',
}
action_obj = zaza_model.run_action(unit_name=unit_name,
action_name='add-disk',
action_params=add_disk_params)
logging.debug('Result of action: {}'.format(action_obj))
logging.info('Wait for idle/ready status...')
zaza_model.wait_for_application_states()
logging.info('OK')
set_default = {
'ephemeral-unmount': '',
'osd-devices': '/dev/vdb',
}
current_release = zaza_openstack.get_os_release()
bionic_train = zaza_openstack.get_os_release('bionic_train')
if current_release < bionic_train:
set_default['osd-devices'] = '/dev/vdb /srv/ceph'
logging.info('Restoring to default configuration...')
zaza_model.set_application_config(juju_service, set_default)
zaza_model.wait_for_application_states()
def test_blocked_when_non_pristine_disk_appears(self):
"""Test blocked state with non-pristine disk.
Validate that charm goes into blocked state when it is presented with
new block devices that have foreign data on them.
Instances used in UOSCI has a flavour with ephemeral storage in
addition to the bootable instance storage. The ephemeral storage
device is partitioned, formatted and mounted early in the boot process
by cloud-init.
As long as the device is mounted the charm will not attempt to use it.
If we unmount it and trigger the config-changed hook the block device
will appear as a new and previously untouched device for the charm.
One of the first steps of device eligibility checks should be to make
sure we are seeing a pristine and empty device before doing any
further processing.
As the ephemeral device will have data on it we can use it to validate
that these checks work as intended.
"""
current_release = zaza_openstack.get_os_release()
focal_ussuri = zaza_openstack.get_os_release('focal_ussuri')
if current_release >= focal_ussuri:
# NOTE(ajkavanagh) - focal (on ServerStack) is broken for /dev/vdb
# and so this test can't pass: LP#1842751 discusses the issue, but
# basically the snapd daemon along with lxcfs results in /dev/vdb
# being mounted in the lxcfs process namespace. If the charm
# 'tries' to umount it, it can (as root), but the mount is still
# 'held' by lxcfs and thus nothing else can be done with it. This
# is only a problem in serverstack with images with a default
# /dev/vdb ephemeral
logging.warn("Skipping pristine disk test for focal and higher")
return
logging.info('Checking behaviour when non-pristine disks appear...')
logging.info('Configuring ephemeral-unmount...')
alternate_conf = {
'ephemeral-unmount': '/mnt',
'osd-devices': '/dev/vdb'
}
juju_service = 'ceph-osd'
zaza_model.set_application_config(juju_service, alternate_conf)
ceph_osd_states = {
'ceph-osd': {
'workload-status': 'blocked',
'workload-status-message': 'Non-pristine'
}
}
zaza_model.wait_for_application_states(states=ceph_osd_states)
logging.info('Units now in blocked state, running zap-disk action...')
unit_names = ['ceph-osd/0', 'ceph-osd/1', 'ceph-osd/2']
for unit_name in unit_names:
zap_disk_params = {
'devices': '/dev/vdb',
'i-really-mean-it': True,
}
action_obj = zaza_model.run_action(
unit_name=unit_name,
action_name='zap-disk',
action_params=zap_disk_params
)
logging.debug('Result of action: {}'.format(action_obj))
logging.info('Running add-disk action...')
for unit_name in unit_names:
add_disk_params = {
'osd-devices': '/dev/vdb',
}
action_obj = zaza_model.run_action(
unit_name=unit_name,
action_name='add-disk',
action_params=add_disk_params
)
logging.debug('Result of action: {}'.format(action_obj))
logging.info('Wait for idle/ready status...')
zaza_model.wait_for_application_states()
logging.info('OK')
set_default = {
'ephemeral-unmount': '',
'osd-devices': '/dev/vdb',
}
current_release = zaza_openstack.get_os_release()
bionic_train = zaza_openstack.get_os_release('bionic_train')
if current_release < bionic_train:
set_default['osd-devices'] = '/dev/vdb /srv/ceph'
logging.info('Restoring to default configuration...')
zaza_model.set_application_config(juju_service, set_default)
zaza_model.wait_for_application_states()
