def test_cache_device(self): """Test replacing a disk in use.""" logging.info('Running add-disk action with a caching device') mon = next(iter(zaza_model.get_units('ceph-mon'))).entity_id osds = [x.entity_id for x in zaza_model.get_units('ceph-osd')] params = [] for unit in osds: loop_dev = self.loop_devs[unit] params.append({'unit': unit, 'device': loop_dev}) action_obj = zaza_model.run_action(unit_name=unit, action_name='add-disk', action_params={ 'osd-devices': loop_dev, 'partition-size': 5 }) zaza_utils.assertActionRanOK(action_obj) zaza_model.wait_for_application_states() logging.info('Removing previously added disks') for param in params: osd_id = self.get_local_osd_id(param['unit']) param.update({'osd-id': osd_id}) action_obj = zaza_model.run_action(unit_name=param['unit'], action_name='remove-disk', action_params={ 'osd-ids': osd_id, 'timeout': 5, 'format': 'json', 'purge': False }) zaza_utils.assertActionRanOK(action_obj) results = json.loads(action_obj.data['results']['message']) results = results[next(iter(results))] self.assertEqual(results['osd-ids'], osd_id) zaza_model.run_on_unit(param['unit'], 'partprobe') zaza_model.wait_for_application_states() logging.info('Recycling previously removed OSDs') for param in params: action_obj = zaza_model.run_action(unit_name=param['unit'], action_name='add-disk', action_params={ 'osd-devices': param['device'], 'osd-ids': param['osd-id'], 'partition-size': 4 }) zaza_utils.assertActionRanOK(action_obj) zaza_model.wait_for_application_states() self.assertEqual(len(osds) * 2, self.get_num_osds(mon)) # Finally, remove all the added OSDs that are backed by loop devices. for param in params: osd_id = self.get_local_osd_id(param['unit']) zaza_model.run_action(unit_name=param['unit'], action_name='remove-disk', action_params={ 'osd-ids': osd_id, 'purge': True })
def dist_upgrade(unit_name): """Run dist-upgrade on unit after update package db. :param unit_name: Unit Name :type unit_name: str :returns: None :rtype: None """ logging.info('Updating package db ' + unit_name) update_cmd = 'sudo apt update' model.run_on_unit(unit_name, update_cmd) logging.info('Updating existing packages ' + unit_name) dist_upgrade_cmd = ( """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") model.run_on_unit(unit_name, dist_upgrade_cmd) rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required") if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): logging.info("dist-upgrade required reboot {}".format(unit_name)) os_utils.reboot(unit_name) logging.info("Waiting for workload status 'unknown' on {}" .format(unit_name)) model.block_until_unit_wl_status(unit_name, "unknown") logging.info("Waiting for workload status to return to normal on {}" .format(unit_name)) model.block_until_unit_wl_status( unit_name, "unknown", negate_match=True) logging.info("Waiting for model idleness") # pause for a big time.sleep(5.0) model.block_until_all_units_idle()
def run_update_status_hooks(self, units): """Run update status hooks on units. :param units: List of unit names or unit.entity_id :type units: List[str] :returns: None :rtype: None """ for unit in units: model.run_on_unit(unit, "hooks/update-status")
def restart_tvault_contego(): """Workaround for Bug #1951999. tvault-contego may need restarting if nova.conf is rendered after systemd has timed out restarting tvault-contego. """ for unit in zaza_model.get_units('trilio-data-mover'): zaza_model.run_on_unit(unit.entity_id, ("systemctl restart tvault-contego; " "./hooks/update-status"))
def python2_workaround(): """Workaround for Bug #1915914. Trilio code currently has a bug which assumes an executable called 'python' will be on the path. To workaround this install a package which adds a symlink to python """ for unit in zaza_model.get_units('trilio-wlm'): zaza_model.run_on_unit(unit.entity_id, ("apt install --yes python-is-python3; " "systemctl restart wlm\\*.service"))
def set_dpkg_non_interactive_on_unit( unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"): """Set dpkg options on unit. :param unit_name: Unit Name :type unit_name: str :param apt_conf_d: Apt.conf file to update :type apt_conf_d: str """ DPKG_NON_INTERACTIVE = 'DPkg::options { "--force-confdef"; };' # Check if the option exists. If not, add it to the apt.conf.d file cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}" .format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d)) model.run_on_unit(unit_name, cmd)
def trigger_deferred_restart_via_package(self, restart_package): """Update a package which requires a service restart. :param restart_package: Package that will be changed to trigger a service restart. :type restart_package: str """ logging.info("Triggering deferred restart via package change") # Test restart requested by package for unit in model.get_units(self.application_name): model.run_on_unit( unit.entity_id, ('dpkg-reconfigure {}; ' 'JUJU_HOOK_NAME=update-status ./hooks/update-status').format( restart_package))
def test_02_remove_check(self): """Verify swap check is removed.""" model.set_application_config(self.application_name, {"swap": ""}) model.block_until_all_units_idle() cmd = "cat /etc/nagios/nrpe.d/check_swap.cfg" result = model.run_on_unit(self.lead_unit_name, cmd) self.assertTrue(result.get("Code") != 0)
def setUp(self): """Run test setup.""" # Skip 'service' action tests on systems without systemd result = zaza_model.run_on_unit(self.TESTED_UNIT, 'which systemctl') if not result['Stdout']: raise unittest.SkipTest("'service' action is not supported on " "systems without 'systemd'. Skipping " "tests.") # Note(mkalcok): This counter reset is needed because ceph-osd service # is limited to 3 restarts per 30 mins which is insufficient # when running functional tests for 'service' action. This # limitation is defined in /lib/systemd/system/[email protected] # in section [Service] with options 'StartLimitInterval' and # 'StartLimitBurst' reset_counter = 'systemctl reset-failed' zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter)
def remote_run(unit, remote_cmd, timeout=None, fatal=None, model_name=None): """Run command on unit and return the output. NOTE: This function is pre-deprecated. As soon as libjuju unit.run is able to return output this functionality should move to model.run_on_unit. :param remote_cmd: Command to execute on unit :type remote_cmd: string :param timeout: Timeout value for the command :type arg: int :param fatal: Command failure condidered fatal or not :type fatal: bool :param model_name: Name of model to query. :type model_name: str :returns: Juju run output :rtype: string :raises: model.CommandRunFailed """ if fatal is None: fatal = True result = model.run_on_unit(unit, remote_cmd, model_name=model_name, timeout=timeout) if result: if int(result.get("Code")) == 0: return result.get("Stdout") else: if fatal: raise model.CommandRunFailed(remote_cmd, result) return result.get("Stderr")
def assert_path_glob(test_case, unit, file_details, paths=None): """Verify all files in a given directory. :param test_case: Test case that we are asserting in :type test_case: unittest.TestCase :param unit: Unit name to operate on :type unit: str :param file_details: Dictionary with details of the file :type file_details: dict :param paths: list of paths that are explicitly tested :type paths: list :returns: Nothing :rtype: None """ if not paths: paths = [] result = model.run_on_unit( unit, 'bash -c "' 'shopt -s -q globstar; ' 'stat -c "%n %U %G %a" {}"'.format(file_details['path'])) files = result['Stdout'] for file in files.splitlines(): file, owner, group, mode = file.split() if file not in paths and file not in ['.', '..']: _verify_file(test_case, unit, file_details, owner, group, mode, path=file)
def get_ceph_pool_details(query_leader=True, unit_name=None, model_name=None): """Get ceph pool details. Return a list of ceph pools details dicts. :param query_leader: Whether to query the leader for pool details. :type query_leader: bool :param unit_name: Name of unit to get the pools on if query_leader is False :type unit_name: string :param model_name: Name of model to operate in :type model_name: str :returns: Dict of ceph pools :rtype: List[Dict,] :raise: zaza_model.CommandRunFailed """ cmd = 'sudo ceph osd pool ls detail -f json' if query_leader and unit_name: raise ValueError("Cannot set query_leader and unit_name") if query_leader: result = zaza_model.run_on_leader( 'ceph-mon', cmd, model_name=model_name) else: result = zaza_model.run_on_unit( unit_name, cmd, model_name=model_name) if int(result.get('Code')) != 0: raise zaza_model.CommandRunFailed(cmd, result) return json.loads(result.get('Stdout'))
def remote_run(unit, remote_cmd, timeout=None, fatal=None): """Run command on unit and return the output NOTE: This function is pre-deprecated. As soon as libjuju unit.run is able to return output this functionality should move to model.run_on_unit. :param remote_cmd: Command to execute on unit :type remote_cmd: string :param timeout: Timeout value for the command :type arg: int :param fatal: Command failure condidered fatal or not :type fatal: boolean :returns: Juju run output :rtype: string """ if fatal is None: fatal = True result = model.run_on_unit(lifecycle_utils.get_juju_model(), unit, remote_cmd, timeout=timeout) if result: if int(result.get('Code')) == 0: return result.get('Stdout') else: if fatal: raise Exception('Error running remote command: {}'.format( result.get('Stderr'))) return result.get('Stderr')
def _ceph_to_ceph_osd_relation(self, remote_unit_name): """Verify the cephX to ceph-osd relation data. Helper function to test the relation. """ logging.info('Checking {}:ceph-osd mon relation data...'.format( remote_unit_name)) unit_name = 'ceph-osd/0' relation_name = 'osd' remote_unit = zaza_model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address cmd = 'leader-get fsid' result = zaza_model.run_on_unit(remote_unit_name, cmd) fsid = result.get('Stdout').strip() expected = { 'private-address': remote_ip, 'ceph-public-address': remote_ip, 'fsid': fsid, } relation = zaza_juju.get_relation_from_unit(unit_name, remote_unit_name, relation_name) for e_key, e_value in expected.items(): a_value = relation[e_key] self.assertEqual(e_value, a_value) self.assertTrue(relation['osd_bootstrap_key'] is not None)
def get_process_id_list(unit_name, process_name, expect_success=True): """Get a list of process ID(s). Get a list of process ID(s) from a single sentry juju unit for a single process name. :param unit_name: Amulet sentry instance (juju unit) :param process_name: Process name :param expect_success: If False, expect the PID to be missing, raise if it is present. :returns: List of process IDs :raises: zaza_exceptions.ProcessIdsFailed """ cmd = 'pidof -x "{}"'.format(process_name) if not expect_success: cmd += " || exit 0 && exit 1" results = model.run_on_unit(unit_name=unit_name, command=cmd) code = results.get("Code", 1) try: code = int(code) except ValueError: code = 1 error = results.get("Stderr") output = results.get("Stdout") if code != 0: msg = ('{} `{}` returned {} ' '{} with error {}'.format(unit_name, cmd, code, output, error)) raise zaza_exceptions.ProcessIdsFailed(msg) return str(output).split()
def test_cinder_ceph_restrict_pool_setup(self): """Make sure cinder-ceph restrict pool was created successfully.""" logging.info('Wait for idle/ready status...') zaza_model.wait_for_application_states() pools = zaza_ceph.get_ceph_pools('ceph-mon/0') if 'cinder-ceph' not in pools: msg = 'cinder-ceph pool was not found upon querying ceph-mon/0' raise zaza_exceptions.CephPoolNotFound(msg) # Checking for cinder-ceph specific permissions makes # the test more rugged when we add additional relations # to ceph for other applications (such as glance and nova). expected_permissions = [ "allow rwx pool=cinder-ceph", "allow class-read object_prefix rbd_children", ] cmd = "sudo ceph auth get client.cinder-ceph" result = zaza_model.run_on_unit('ceph-mon/0', cmd) output = result.get('Stdout').strip() for expected in expected_permissions: if expected not in output: msg = ('cinder-ceph pool restriction ({}) was not' ' configured correctly.' ' Found: {}'.format(expected, output)) raise zaza_exceptions.CephPoolNotConfigured(msg)
def get_relation_from_unit(entity, remote_entity, remote_interface_name): """Get relation data passed between two units. Get relation data for relation with `remote_interface_name` between `entity` and `remote_entity` from the perspective of `entity`. `entity` and `remote_entity` may refer to either a application or a specific unit. If application name is given first unit is found in model. :param entity: Application or unit to get relation data from :type entity: str :param remote_entity: Application or Unit in the other end of the relation we want to query :type remote_entity: str :param remote_interface_name: Name of interface to query on remote end of relation :type remote_interface_name: str :returns: dict with relation data :rtype: dict :raises: model.CommandRunFailed """ application = entity.split('/')[0] remote_application = remote_entity.split('/')[0] rid = model.get_relation_id(application, remote_application, remote_interface_name=remote_interface_name) (unit, remote_unit) = _get_unit_names([entity, remote_entity]) cmd = 'relation-get --format=yaml -r "{}" - "{}"'.format(rid, remote_unit) result = model.run_on_unit(unit, cmd) if result and int(result.get('Code')) == 0: return yaml.safe_load(result.get('Stdout')) else: raise model.CommandRunFailed(cmd, result)
def test_ceph_pool_creation_with_text_file(self): """Check the creation of a pool and a text file. Create a pool, add a text file to it and retrieve its content. Verify that the content matches the original file. """ issue = 'github.com/openstack-charmers/zaza-openstack-tests/issues/647' current_release = zaza_openstack.get_os_release() focal_victoria = zaza_openstack.get_os_release('focal_victoria') if current_release >= focal_victoria: logging.warn('Skipping test_ceph_pool_creation_with_text_file due' ' to issue {}'.format(issue)) return unit_name = 'ceph-mon/0' cmd = 'sudo ceph osd pool create test 128; \ echo 123456789 > /tmp/input.txt; \ rados put -p test test_input /tmp/input.txt; \ rados get -p test test_input /dev/stdout' logging.debug('Creating test pool and putting test file in pool...') result = zaza_model.run_on_unit(unit_name, cmd) code = result.get('Code') if code != '0': raise zaza_model.CommandRunFailed(cmd, result) output = result.get('Stdout').strip() logging.debug('Output received: {}'.format(output)) self.assertEqual(output, '123456789')
def test_pg_tuning(self): """Verify that auto PG tuning is enabled for Nautilus+.""" unit_name = 'ceph-mon/0' cmd = "ceph osd pool autoscale-status --format=json" result = zaza_model.run_on_unit(unit_name, cmd) self.assertEqual(result['Code'], '0') for pool in json.loads(result['Stdout']): self.assertEqual(pool['pg_autoscale_mode'], 'on')
def get_unit_hostnames(units): """Return a dict of juju unit names to hostnames.""" host_names = {} for unit in units: output = model.run_on_unit(unit.entity_id, 'hostname') hostname = output['Stdout'].strip() host_names[unit.entity_id] = hostname return host_names
def dist_upgrade(unit_name): """Run dist-upgrade on unit after update package db. :param unit_name: Unit Name :type unit_name: str :returns: None :rtype: None """ logging.info('Updating package db ' + unit_name) update_cmd = 'sudo apt update' model.run_on_unit(unit_name, update_cmd) logging.info('Updating existing packages ' + unit_name) dist_upgrade_cmd = ( """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") model.run_on_unit(unit_name, dist_upgrade_cmd)
def test_run_on_unit(self): self.patch_object(model, 'get_juju_model', return_value='mname') expected = {'Code': '0', 'Stderr': '', 'Stdout': 'RESULT'} self.cmd = cmd = 'somecommand someargument' self.patch_object(model, 'Model') self.patch_object(model, 'get_unit_from_name') self.get_unit_from_name.return_value = self.unit1 self.Model.return_value = self.Model_mock self.assertEqual(model.run_on_unit('app/2', cmd), expected) self.unit1.run.assert_called_once_with(cmd, timeout=None)
def get_unit_hostnames(units, fqdn=False): """Return a dict of juju unit names to hostnames.""" host_names = {} for unit in units: cmd = 'hostname' if fqdn: cmd = cmd + ' -f' output = model.run_on_unit(unit.entity_id, cmd) hostname = output['Stdout'].strip() host_names[unit.entity_id] = hostname return host_names
def test_start_specific(self): """Start only specified ceph-osd service.""" if len(self.available_services) < 2: raise unittest.SkipTest('This test can be performed only if ' 'there\'s more than one ceph-osd service ' 'present on the tested unit') service_names = [service.name for service in self.available_services] should_stop = deepcopy(self.available_services) to_start = should_stop.pop() should_stop = [service.name for service in should_stop] # Note: can't stop ceph-osd.target as restarting a single OSD will # cause this to start all of the OSDs when a single one starts. logging.info("Stopping all running ceph-osd services") service_stop_cmd = '; '.join( ['systemctl stop {}'.format(service) for service in service_names]) zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) wait_for_service(unit_name=self.TESTED_UNIT, services=service_names, target_status='stopped') logging.info("Running 'service start={} on {} " "unit".format(to_start.id, self.TESTED_UNIT)) zaza_model.run_action_on_units([ self.TESTED_UNIT, ], 'start', action_params={'osds': to_start.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[ to_start.name, ], target_status='running') wait_for_service(unit_name=self.TESTED_UNIT, services=should_stop, target_status='stopped')
def test_01_nrpe_check(self): """Verify nrpe check exists.""" logging.debug( "Verify the nrpe checks are created and have the required content..." ) check_mysql_content = ( "command[check_mysql]=/usr/local/lib/nagios/plugins/check_systemd.py mysql" ) machine = list(juju_utils.get_machines_for_application("mysql"))[0] machine_series = juju_utils.get_machine_series(machine) if machine_series == "trusty": check_mysql_content = ( "command[check_mysql]=/usr/lib/nagios/plugins/check_mysql -u nagios" ) nrpe_checks = { "check_conntrack.cfg": "command[check_conntrack]=/usr/local/lib/nagios/plugins/" "check_conntrack.sh", "check_disk_root.cfg": "command[check_disk_root]=/usr/lib/nagios/plugins/check_disk", "check_load.cfg": "command[check_load]=/usr/lib/nagios/plugins/check_load", "check_mem.cfg": "command[check_mem]=/usr/local/lib/nagios/plugins/check_mem.pl", "check_mysql.cfg": check_mysql_content, "check_mysql_proc.cfg": "command[check_mysql_proc]=/usr/lib/nagios/plugins/" "check_procs -c 1:1 -C mysqld", "check_swap_activity.cfg": "command[check_swap_activity]=" "/usr/local/lib/nagios/plugins/check_swap_activity", "check_swap.cfg": "command[check_swap]=/usr/lib/nagios/plugins/check_swap", } for nrpe_check in nrpe_checks: logging.info( "Checking content of '{}' nrpe check".format(nrpe_check)) cmd = "cat /etc/nagios/nrpe.d/" + nrpe_check result = model.run_on_unit(self.lead_unit_name, cmd) code = result.get("Code") if code != "0": logging.warning( "Unable to find nrpe check {} at /etc/nagios/nrpe.d/". format(nrpe_check)) raise model.CommandRunFailed(cmd, result) content = result.get("Stdout") self.assertTrue(nrpe_checks[nrpe_check] in content)
def wait_for_sync(self, application): """Wait for slave to secondary to show it is in sync.""" juju_units = zaza_model.get_units(application) unit_hostnames = generic_utils.get_unit_hostnames(juju_units) sync_states = [] sync_check_str = 'data is caught up with source' for unit_name, hostname in unit_hostnames.items(): key_name = "rgw.{}".format(hostname) cmd = 'radosgw-admin --id={} zone sync status'.format(key_name) stdout = zaza_model.run_on_unit(unit_name, cmd).get('Stdout', '') sync_states.append(sync_check_str in stdout) assert all(sync_states)
def _fetch_osd_services(self): """Fetch all ceph-osd services present on the TESTED_UNIT.""" service_list = [] service_list_cmd = 'systemctl list-units --full --all ' \ '--no-pager -t service' result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd) for line in result['Stdout'].split('\n'): service_name = self.SERVICE_PATTERN.search(line) if service_name: service_id = int(service_name.group('service_id')) service_list.append(OsdService(service_id)) return service_list
def test_04_check_nagios_ip_is_allowed(self): """Verify nagios ip is allowed in nrpe.cfg.""" nagios_ip = model.get_app_ips("nagios")[0] line = "allowed_hosts=127.0.0.1,{}/32".format(nagios_ip) cmd = "cat /etc/nagios/nrpe.cfg" result = model.run_on_unit(self.lead_unit_name, cmd) code = result.get("Code") if code != "0": logging.warning( "Unable to find nrpe config file at /etc/nagios/nrpe.cfg") raise model.CommandRunFailed(cmd, result) content = result.get("Stdout") self.assertTrue(line in content)
def remove_loop_device(unit, device, name='loop.img'): """Remove a loopback device from a Juju unit. :param unit: The unit name from which to remove the device. :type unit: str :param device: The loop device path to be removed. :type unit: str :param name: The name of the file used for the loop device. :type name: str """ cmd = "sudo sh -c 'losetup -d {} && rm {}'".format(device, name) return model.run_on_unit(unit, cmd)
def get_ceph_df(unit_name, model_name=None): """Return dict of ceph df json output, including ceph pool state. :param unit_name: Name of the unit to get ceph df :type unit_name: string :param model_name: Name of model to operate in :type model_name: str :returns: Dict of ceph df output :rtype: dict :raise: zaza.model.CommandRunFailed """ cmd = 'sudo ceph df --format=json' result = zaza_model.run_on_unit(unit_name, cmd, model_name=model_name) if result.get('Code') != '0': raise zaza_model.CommandRunFailed(cmd, result) return json.loads(result.get('Stdout'))