def test_cache_device(self):
"""Test replacing a disk in use."""
logging.info('Running add-disk action with a caching device')
mon = next(iter(zaza_model.get_units('ceph-mon'))).entity_id
osds = [x.entity_id for x in zaza_model.get_units('ceph-osd')]
params = []
for unit in osds:
loop_dev = self.loop_devs[unit]
params.append({'unit': unit, 'device': loop_dev})
action_obj = zaza_model.run_action(unit_name=unit,
action_name='add-disk',
action_params={
'osd-devices': loop_dev,
'partition-size': 5
})
zaza_utils.assertActionRanOK(action_obj)
zaza_model.wait_for_application_states()
logging.info('Removing previously added disks')
for param in params:
osd_id = self.get_local_osd_id(param['unit'])
param.update({'osd-id': osd_id})
action_obj = zaza_model.run_action(unit_name=param['unit'],
action_name='remove-disk',
action_params={
'osd-ids': osd_id,
'timeout': 5,
'format': 'json',
'purge': False
})
zaza_utils.assertActionRanOK(action_obj)
results = json.loads(action_obj.data['results']['message'])
results = results[next(iter(results))]
self.assertEqual(results['osd-ids'], osd_id)
zaza_model.run_on_unit(param['unit'], 'partprobe')
zaza_model.wait_for_application_states()
logging.info('Recycling previously removed OSDs')
for param in params:
action_obj = zaza_model.run_action(unit_name=param['unit'],
action_name='add-disk',
action_params={
'osd-devices':
param['device'],
'osd-ids': param['osd-id'],
'partition-size': 4
})
zaza_utils.assertActionRanOK(action_obj)
zaza_model.wait_for_application_states()
self.assertEqual(len(osds) * 2, self.get_num_osds(mon))
# Finally, remove all the added OSDs that are backed by loop devices.
for param in params:
osd_id = self.get_local_osd_id(param['unit'])
zaza_model.run_action(unit_name=param['unit'],
action_name='remove-disk',
action_params={
'osd-ids': osd_id,
'purge': True
})
def dist_upgrade(unit_name):
"""Run dist-upgrade on unit after update package db.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Updating package db ' + unit_name)
update_cmd = 'sudo apt update'
model.run_on_unit(unit_name, update_cmd)
logging.info('Updating existing packages ' + unit_name)
dist_upgrade_cmd = (
"""sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
model.run_on_unit(unit_name, dist_upgrade_cmd)
rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required")
if "Stdout" in rdict and "restart" in rdict["Stdout"].lower():
logging.info("dist-upgrade required reboot {}".format(unit_name))
os_utils.reboot(unit_name)
logging.info("Waiting for workload status 'unknown' on {}"
.format(unit_name))
model.block_until_unit_wl_status(unit_name, "unknown")
logging.info("Waiting for workload status to return to normal on {}"
.format(unit_name))
model.block_until_unit_wl_status(
unit_name, "unknown", negate_match=True)
logging.info("Waiting for model idleness")
# pause for a big
time.sleep(5.0)
model.block_until_all_units_idle()
def run_update_status_hooks(self, units):
"""Run update status hooks on units.
:param units: List of unit names or unit.entity_id
:type units: List[str]
:returns: None
:rtype: None
"""
for unit in units:
model.run_on_unit(unit, "hooks/update-status")
def restart_tvault_contego():
"""Workaround for Bug #1951999.
tvault-contego may need restarting if nova.conf is rendered after
systemd has timed out restarting tvault-contego.
"""
for unit in zaza_model.get_units('trilio-data-mover'):
zaza_model.run_on_unit(unit.entity_id,
("systemctl restart tvault-contego; "
"./hooks/update-status"))
def python2_workaround():
"""Workaround for Bug #1915914.
Trilio code currently has a bug which assumes an executable called 'python'
will be on the path. To workaround this install a package which adds a
symlink to python
"""
for unit in zaza_model.get_units('trilio-wlm'):
zaza_model.run_on_unit(unit.entity_id,
("apt install --yes python-is-python3; "
"systemctl restart wlm\\*.service"))
def set_dpkg_non_interactive_on_unit(
unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"):
"""Set dpkg options on unit.
:param unit_name: Unit Name
:type unit_name: str
:param apt_conf_d: Apt.conf file to update
:type apt_conf_d: str
"""
DPKG_NON_INTERACTIVE = 'DPkg::options { "--force-confdef"; };'
# Check if the option exists. If not, add it to the apt.conf.d file
cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}"
.format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d))
model.run_on_unit(unit_name, cmd)
def trigger_deferred_restart_via_package(self, restart_package):
"""Update a package which requires a service restart.
:param restart_package: Package that will be changed to trigger a
service restart.
:type restart_package: str
"""
logging.info("Triggering deferred restart via package change")
# Test restart requested by package
for unit in model.get_units(self.application_name):
model.run_on_unit(
unit.entity_id,
('dpkg-reconfigure {}; '
'JUJU_HOOK_NAME=update-status ./hooks/update-status').format(
restart_package))
def test_02_remove_check(self):
"""Verify swap check is removed."""
model.set_application_config(self.application_name, {"swap": ""})
model.block_until_all_units_idle()
cmd = "cat /etc/nagios/nrpe.d/check_swap.cfg"
result = model.run_on_unit(self.lead_unit_name, cmd)
self.assertTrue(result.get("Code") != 0)
def setUp(self):
"""Run test setup."""
# Skip 'service' action tests on systems without systemd
result = zaza_model.run_on_unit(self.TESTED_UNIT, 'which systemctl')
if not result['Stdout']:
raise unittest.SkipTest("'service' action is not supported on "
"systems without 'systemd'. Skipping "
"tests.")
# Note(mkalcok): This counter reset is needed because ceph-osd service
# is limited to 3 restarts per 30 mins which is insufficient
# when running functional tests for 'service' action. This
# limitation is defined in /lib/systemd/system/[email protected]
# in section [Service] with options 'StartLimitInterval' and
# 'StartLimitBurst'
reset_counter = 'systemctl reset-failed'
zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter)
def remote_run(unit, remote_cmd, timeout=None, fatal=None, model_name=None):
"""Run command on unit and return the output.
NOTE: This function is pre-deprecated. As soon as libjuju unit.run is able
to return output this functionality should move to model.run_on_unit.
:param remote_cmd: Command to execute on unit
:type remote_cmd: string
:param timeout: Timeout value for the command
:type arg: int
:param fatal: Command failure condidered fatal or not
:type fatal: bool
:param model_name: Name of model to query.
:type model_name: str
:returns: Juju run output
:rtype: string
:raises: model.CommandRunFailed
"""
if fatal is None:
fatal = True
result = model.run_on_unit(unit,
remote_cmd,
model_name=model_name,
timeout=timeout)
if result:
if int(result.get("Code")) == 0:
return result.get("Stdout")
else:
if fatal:
raise model.CommandRunFailed(remote_cmd, result)
return result.get("Stderr")
def assert_path_glob(test_case, unit, file_details, paths=None):
"""Verify all files in a given directory.
:param test_case: Test case that we are asserting in
:type test_case: unittest.TestCase
:param unit: Unit name to operate on
:type unit: str
:param file_details: Dictionary with details of the file
:type file_details: dict
:param paths: list of paths that are explicitly tested
:type paths: list
:returns: Nothing
:rtype: None
"""
if not paths:
paths = []
result = model.run_on_unit(
unit, 'bash -c "'
'shopt -s -q globstar; '
'stat -c "%n %U %G %a" {}"'.format(file_details['path']))
files = result['Stdout']
for file in files.splitlines():
file, owner, group, mode = file.split()
if file not in paths and file not in ['.', '..']:
_verify_file(test_case,
unit,
file_details,
owner,
group,
mode,
path=file)
def get_ceph_pool_details(query_leader=True, unit_name=None, model_name=None):
"""Get ceph pool details.
Return a list of ceph pools details dicts.
:param query_leader: Whether to query the leader for pool details.
:type query_leader: bool
:param unit_name: Name of unit to get the pools on if query_leader is False
:type unit_name: string
:param model_name: Name of model to operate in
:type model_name: str
:returns: Dict of ceph pools
:rtype: List[Dict,]
:raise: zaza_model.CommandRunFailed
"""
cmd = 'sudo ceph osd pool ls detail -f json'
if query_leader and unit_name:
raise ValueError("Cannot set query_leader and unit_name")
if query_leader:
result = zaza_model.run_on_leader(
'ceph-mon',
cmd,
model_name=model_name)
else:
result = zaza_model.run_on_unit(
unit_name,
cmd,
model_name=model_name)
if int(result.get('Code')) != 0:
raise zaza_model.CommandRunFailed(cmd, result)
return json.loads(result.get('Stdout'))
def remote_run(unit, remote_cmd, timeout=None, fatal=None):
"""Run command on unit and return the output
NOTE: This function is pre-deprecated. As soon as libjuju unit.run is able
to return output this functionality should move to model.run_on_unit.
:param remote_cmd: Command to execute on unit
:type remote_cmd: string
:param timeout: Timeout value for the command
:type arg: int
:param fatal: Command failure condidered fatal or not
:type fatal: boolean
:returns: Juju run output
:rtype: string
"""
if fatal is None:
fatal = True
result = model.run_on_unit(lifecycle_utils.get_juju_model(),
unit,
remote_cmd,
timeout=timeout)
if result:
if int(result.get('Code')) == 0:
return result.get('Stdout')
else:
if fatal:
raise Exception('Error running remote command: {}'.format(
result.get('Stderr')))
return result.get('Stderr')
def _ceph_to_ceph_osd_relation(self, remote_unit_name):
"""Verify the cephX to ceph-osd relation data.
Helper function to test the relation.
"""
logging.info('Checking {}:ceph-osd mon relation data...'.format(
remote_unit_name))
unit_name = 'ceph-osd/0'
relation_name = 'osd'
remote_unit = zaza_model.get_unit_from_name(remote_unit_name)
remote_ip = remote_unit.public_address
cmd = 'leader-get fsid'
result = zaza_model.run_on_unit(remote_unit_name, cmd)
fsid = result.get('Stdout').strip()
expected = {
'private-address': remote_ip,
'ceph-public-address': remote_ip,
'fsid': fsid,
}
relation = zaza_juju.get_relation_from_unit(unit_name,
remote_unit_name,
relation_name)
for e_key, e_value in expected.items():
a_value = relation[e_key]
self.assertEqual(e_value, a_value)
self.assertTrue(relation['osd_bootstrap_key'] is not None)
def get_process_id_list(unit_name, process_name,
expect_success=True):
"""Get a list of process ID(s).
Get a list of process ID(s) from a single sentry juju unit
for a single process name.
:param unit_name: Amulet sentry instance (juju unit)
:param process_name: Process name
:param expect_success: If False, expect the PID to be missing,
raise if it is present.
:returns: List of process IDs
:raises: zaza_exceptions.ProcessIdsFailed
"""
cmd = 'pidof -x "{}"'.format(process_name)
if not expect_success:
cmd += " || exit 0 && exit 1"
results = model.run_on_unit(unit_name=unit_name, command=cmd)
code = results.get("Code", 1)
try:
code = int(code)
except ValueError:
code = 1
error = results.get("Stderr")
output = results.get("Stdout")
if code != 0:
msg = ('{} `{}` returned {} '
'{} with error {}'.format(unit_name, cmd, code, output, error))
raise zaza_exceptions.ProcessIdsFailed(msg)
return str(output).split()
def test_cinder_ceph_restrict_pool_setup(self):
"""Make sure cinder-ceph restrict pool was created successfully."""
logging.info('Wait for idle/ready status...')
zaza_model.wait_for_application_states()
pools = zaza_ceph.get_ceph_pools('ceph-mon/0')
if 'cinder-ceph' not in pools:
msg = 'cinder-ceph pool was not found upon querying ceph-mon/0'
raise zaza_exceptions.CephPoolNotFound(msg)
# Checking for cinder-ceph specific permissions makes
# the test more rugged when we add additional relations
# to ceph for other applications (such as glance and nova).
expected_permissions = [
"allow rwx pool=cinder-ceph",
"allow class-read object_prefix rbd_children",
]
cmd = "sudo ceph auth get client.cinder-ceph"
result = zaza_model.run_on_unit('ceph-mon/0', cmd)
output = result.get('Stdout').strip()
for expected in expected_permissions:
if expected not in output:
msg = ('cinder-ceph pool restriction ({}) was not'
' configured correctly.'
' Found: {}'.format(expected, output))
raise zaza_exceptions.CephPoolNotConfigured(msg)
def get_relation_from_unit(entity, remote_entity, remote_interface_name):
"""Get relation data passed between two units.
Get relation data for relation with `remote_interface_name` between
`entity` and `remote_entity` from the perspective of `entity`.
`entity` and `remote_entity` may refer to either a application or a
specific unit. If application name is given first unit is found in model.
:param entity: Application or unit to get relation data from
:type entity: str
:param remote_entity: Application or Unit in the other end of the relation
we want to query
:type remote_entity: str
:param remote_interface_name: Name of interface to query on remote end of
relation
:type remote_interface_name: str
:returns: dict with relation data
:rtype: dict
:raises: model.CommandRunFailed
"""
application = entity.split('/')[0]
remote_application = remote_entity.split('/')[0]
rid = model.get_relation_id(application,
remote_application,
remote_interface_name=remote_interface_name)
(unit, remote_unit) = _get_unit_names([entity, remote_entity])
cmd = 'relation-get --format=yaml -r "{}" - "{}"'.format(rid, remote_unit)
result = model.run_on_unit(unit, cmd)
if result and int(result.get('Code')) == 0:
return yaml.safe_load(result.get('Stdout'))
else:
raise model.CommandRunFailed(cmd, result)
def test_ceph_pool_creation_with_text_file(self):
"""Check the creation of a pool and a text file.
Create a pool, add a text file to it and retrieve its content.
Verify that the content matches the original file.
"""
issue = 'github.com/openstack-charmers/zaza-openstack-tests/issues/647'
current_release = zaza_openstack.get_os_release()
focal_victoria = zaza_openstack.get_os_release('focal_victoria')
if current_release >= focal_victoria:
logging.warn('Skipping test_ceph_pool_creation_with_text_file due'
' to issue {}'.format(issue))
return
unit_name = 'ceph-mon/0'
cmd = 'sudo ceph osd pool create test 128; \
echo 123456789 > /tmp/input.txt; \
rados put -p test test_input /tmp/input.txt; \
rados get -p test test_input /dev/stdout'
logging.debug('Creating test pool and putting test file in pool...')
result = zaza_model.run_on_unit(unit_name, cmd)
code = result.get('Code')
if code != '0':
raise zaza_model.CommandRunFailed(cmd, result)
output = result.get('Stdout').strip()
logging.debug('Output received: {}'.format(output))
self.assertEqual(output, '123456789')
def test_pg_tuning(self):
"""Verify that auto PG tuning is enabled for Nautilus+."""
unit_name = 'ceph-mon/0'
cmd = "ceph osd pool autoscale-status --format=json"
result = zaza_model.run_on_unit(unit_name, cmd)
self.assertEqual(result['Code'], '0')
for pool in json.loads(result['Stdout']):
self.assertEqual(pool['pg_autoscale_mode'], 'on')
def get_unit_hostnames(units):
"""Return a dict of juju unit names to hostnames."""
host_names = {}
for unit in units:
output = model.run_on_unit(unit.entity_id, 'hostname')
hostname = output['Stdout'].strip()
host_names[unit.entity_id] = hostname
return host_names
def dist_upgrade(unit_name):
"""Run dist-upgrade on unit after update package db.
:param unit_name: Unit Name
:type unit_name: str
:returns: None
:rtype: None
"""
logging.info('Updating package db ' + unit_name)
update_cmd = 'sudo apt update'
model.run_on_unit(unit_name, update_cmd)
logging.info('Updating existing packages ' + unit_name)
dist_upgrade_cmd = (
"""sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """
"""-o "Dpkg::Options::=--force-confdef" """
"""-o "Dpkg::Options::=--force-confold" dist-upgrade""")
model.run_on_unit(unit_name, dist_upgrade_cmd)
def test_run_on_unit(self):
self.patch_object(model, 'get_juju_model', return_value='mname')
expected = {'Code': '0', 'Stderr': '', 'Stdout': 'RESULT'}
self.cmd = cmd = 'somecommand someargument'
self.patch_object(model, 'Model')
self.patch_object(model, 'get_unit_from_name')
self.get_unit_from_name.return_value = self.unit1
self.Model.return_value = self.Model_mock
self.assertEqual(model.run_on_unit('app/2', cmd), expected)
self.unit1.run.assert_called_once_with(cmd, timeout=None)
def get_unit_hostnames(units, fqdn=False):
"""Return a dict of juju unit names to hostnames."""
host_names = {}
for unit in units:
cmd = 'hostname'
if fqdn:
cmd = cmd + ' -f'
output = model.run_on_unit(unit.entity_id, cmd)
hostname = output['Stdout'].strip()
host_names[unit.entity_id] = hostname
return host_names
def test_start_specific(self):
"""Start only specified ceph-osd service."""
if len(self.available_services) < 2:
raise unittest.SkipTest('This test can be performed only if '
'there\'s more than one ceph-osd service '
'present on the tested unit')
service_names = [service.name for service in self.available_services]
should_stop = deepcopy(self.available_services)
to_start = should_stop.pop()
should_stop = [service.name for service in should_stop]
# Note: can't stop ceph-osd.target as restarting a single OSD will
# cause this to start all of the OSDs when a single one starts.
logging.info("Stopping all running ceph-osd services")
service_stop_cmd = '; '.join(
['systemctl stop {}'.format(service) for service in service_names])
zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd)
wait_for_service(unit_name=self.TESTED_UNIT,
services=service_names,
target_status='stopped')
logging.info("Running 'service start={} on {} "
"unit".format(to_start.id, self.TESTED_UNIT))
zaza_model.run_action_on_units([
self.TESTED_UNIT,
],
'start',
action_params={'osds': to_start.id})
wait_for_service(unit_name=self.TESTED_UNIT,
services=[
to_start.name,
],
target_status='running')
wait_for_service(unit_name=self.TESTED_UNIT,
services=should_stop,
target_status='stopped')
def test_01_nrpe_check(self):
"""Verify nrpe check exists."""
logging.debug(
"Verify the nrpe checks are created and have the required content..."
)
check_mysql_content = (
"command[check_mysql]=/usr/local/lib/nagios/plugins/check_systemd.py mysql"
)
machine = list(juju_utils.get_machines_for_application("mysql"))[0]
machine_series = juju_utils.get_machine_series(machine)
if machine_series == "trusty":
check_mysql_content = (
"command[check_mysql]=/usr/lib/nagios/plugins/check_mysql -u nagios"
)
nrpe_checks = {
"check_conntrack.cfg":
"command[check_conntrack]=/usr/local/lib/nagios/plugins/"
"check_conntrack.sh",
"check_disk_root.cfg":
"command[check_disk_root]=/usr/lib/nagios/plugins/check_disk",
"check_load.cfg":
"command[check_load]=/usr/lib/nagios/plugins/check_load",
"check_mem.cfg":
"command[check_mem]=/usr/local/lib/nagios/plugins/check_mem.pl",
"check_mysql.cfg":
check_mysql_content,
"check_mysql_proc.cfg":
"command[check_mysql_proc]=/usr/lib/nagios/plugins/"
"check_procs -c 1:1 -C mysqld",
"check_swap_activity.cfg":
"command[check_swap_activity]="
"/usr/local/lib/nagios/plugins/check_swap_activity",
"check_swap.cfg":
"command[check_swap]=/usr/lib/nagios/plugins/check_swap",
}
for nrpe_check in nrpe_checks:
logging.info(
"Checking content of '{}' nrpe check".format(nrpe_check))
cmd = "cat /etc/nagios/nrpe.d/" + nrpe_check
result = model.run_on_unit(self.lead_unit_name, cmd)
code = result.get("Code")
if code != "0":
logging.warning(
"Unable to find nrpe check {} at /etc/nagios/nrpe.d/".
format(nrpe_check))
raise model.CommandRunFailed(cmd, result)
content = result.get("Stdout")
self.assertTrue(nrpe_checks[nrpe_check] in content)
def wait_for_sync(self, application):
"""Wait for slave to secondary to show it is in sync."""
juju_units = zaza_model.get_units(application)
unit_hostnames = generic_utils.get_unit_hostnames(juju_units)
sync_states = []
sync_check_str = 'data is caught up with source'
for unit_name, hostname in unit_hostnames.items():
key_name = "rgw.{}".format(hostname)
cmd = 'radosgw-admin --id={} zone sync status'.format(key_name)
stdout = zaza_model.run_on_unit(unit_name, cmd).get('Stdout', '')
sync_states.append(sync_check_str in stdout)
assert all(sync_states)
def _fetch_osd_services(self):
"""Fetch all ceph-osd services present on the TESTED_UNIT."""
service_list = []
service_list_cmd = 'systemctl list-units --full --all ' \
'--no-pager -t service'
result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd)
for line in result['Stdout'].split('\n'):
service_name = self.SERVICE_PATTERN.search(line)
if service_name:
service_id = int(service_name.group('service_id'))
service_list.append(OsdService(service_id))
return service_list
def test_04_check_nagios_ip_is_allowed(self):
"""Verify nagios ip is allowed in nrpe.cfg."""
nagios_ip = model.get_app_ips("nagios")[0]
line = "allowed_hosts=127.0.0.1,{}/32".format(nagios_ip)
cmd = "cat /etc/nagios/nrpe.cfg"
result = model.run_on_unit(self.lead_unit_name, cmd)
code = result.get("Code")
if code != "0":
logging.warning(
"Unable to find nrpe config file at /etc/nagios/nrpe.cfg")
raise model.CommandRunFailed(cmd, result)
content = result.get("Stdout")
self.assertTrue(line in content)
def remove_loop_device(unit, device, name='loop.img'):
"""Remove a loopback device from a Juju unit.
:param unit: The unit name from which to remove the device.
:type unit: str
:param device: The loop device path to be removed.
:type unit: str
:param name: The name of the file used for the loop device.
:type name: str
"""
cmd = "sudo sh -c 'losetup -d {} && rm {}'".format(device, name)
return model.run_on_unit(unit, cmd)
def get_ceph_df(unit_name, model_name=None):
"""Return dict of ceph df json output, including ceph pool state.
:param unit_name: Name of the unit to get ceph df
:type unit_name: string
:param model_name: Name of model to operate in
:type model_name: str
:returns: Dict of ceph df output
:rtype: dict
:raise: zaza.model.CommandRunFailed
"""
cmd = 'sudo ceph df --format=json'
result = zaza_model.run_on_unit(unit_name, cmd, model_name=model_name)
if result.get('Code') != '0':
raise zaza_model.CommandRunFailed(cmd, result)
return json.loads(result.get('Stdout'))
