def _on_update_status(self, _):
container = self.unit.get_container("splunk")
if not self.state.auto_start:
self.unit.status = MaintenanceStatus("splunk service is paused")
elif not container.get_service("splunk").is_running():
self.unit.status = BlockedStatus("splunk service isn't running")
else:
self.unit.status = ActiveStatus("ready")
def test_not_kubeflow_model():
# Tests that unit will BlockStatus if deployed outside a model named kubeflow
# Remove when this bug is resolved: https://github.com/kubeflow/kubeflow/issues/6136
harness = Harness(Operator)
harness.begin_with_initial_hooks()
assert harness.charm.model.unit.status == BlockedStatus(
"kubeflow-dashboard must be deployed to model named `kubeflow`:"
" https://git.io/J6d35")
def _on_install(self, event):
"""Perform installation operations for slurmctld."""
self.unit.set_workload_version(Path("version").read_text().strip())
self.unit.status = WaitingStatus("Installing slurmctld")
custom_repo = self.config.get("custom-slurm-repo")
successful_installation = self._slurm_manager.install(custom_repo)
if successful_installation:
self._stored.slurm_installed = True
# Store the munge_key and jwt_rsa key in the stored state.
# NOTE: Use leadership settings instead of stored state when
# leadership settings support becomes available in the framework.
if self._is_leader():
# NOTE the backup controller should also have the jwt and munge
# keys configured. We should move these information to the
# peer relation.
self._stored.jwt_rsa = self._slurm_manager.generate_jwt_rsa()
self._stored.munge_key = self._slurm_manager.get_munge_key()
self._slurm_manager.configure_jwt_rsa(self.get_jwt_rsa())
else:
# NOTE: the secondary slurmctld should get the jwt and munge
# keys from the peer relation here
logger.debug("secondary slurmctld")
# all slurmctld should restart munged here, as it would assure
# munge is working
self._slurm_manager.restart_munged()
else:
self.unit.status = BlockedStatus("Error installing slurmctld")
event.defer()
logger.debug("## Retrieving etcd resource to install it")
try:
etcd_path = self.model.resources.fetch("etcd")
logger.debug(f"## Found etcd resource: {etcd_path}")
except ModelError:
logger.error("## Missing etcd resource")
self.unit.status = BlockedStatus("Missing etcd resource")
event.defer()
return
self._etcd.install(etcd_path)
self._check_status()
def _on_update_status(self, _):
path = self._image_publish_dir() + '/.data'
if os.path.isdir(path):
stat = os.stat(path)
self.model.unit.status = \
ActiveStatus("Publishes: {}".format(time.ctime(stat.st_mtime)))
else:
self.model.unit.status = BlockedStatus("Images not synchronized")
def test_custom_status_check_default_config(self):
self.harness.disable_hooks()
self.harness.begin()
self.assertFalse(self.harness.charm.custom_status_check())
expected_status = BlockedStatus('Missing configs: {}'.format(
list(self.REQUIRED_CHARM_CONFIG_BY_DEFAULT.keys())))
self.assertEqual(self.harness.charm.unit.status, expected_status)
def _on_config_changed(self, _):
"""Reconfigure the Grafana tester."""
container = self.unit.get_container(self._name)
if not container.can_connect():
self.unit.status = BlockedStatus("Waiting for Pebble ready")
return
self.unit.status = ActiveStatus()
def verify_credentials(self):
proxy = self.get_ssh_proxy()
verified, _ = proxy.verify_credentials()
if verified:
self.unit.status = ActiveStatus()
else:
self.unit.status = BlockedStatus("Invalid SSH credentials.")
return verified
def test_start_no_certificate(self):
charm._host = Mock()
harness = Harness(charm.CertbotCharm)
self.addCleanup(harness.cleanup)
harness.begin()
harness.update_config(self._config(harness.charm))
harness.charm.on.start.emit()
self.assertEqual(harness.charm.model.unit.status,
BlockedStatus('certificate not yet acquired.'))
def _on_check_status_and_write_config(self, event):
slurm_installed = self._stored.slurm_installed
slurm_config = self._stored.config_available
logger.debug("##### inside check status and write config ######")
if not (slurm_installed and slurm_config):
if not slurm_config:
self.unit.status = BlockedStatus(
"NEED RELATION TO SLURM-CONFIGURATOR")
else:
self.unit.status = BlockedStatus("SLURM NOT INSTALLED")
event.defer()
return
else:
logger.debug("##### STATUS CONFIRMED ######")
config = dict(self._slurmrestd.get_slurm_config())
logger.debug(config)
self.slurm_manager.render_config_and_restart(config)
self.unit.status = ActiveStatus("Slurmrestd Available")
def start(self, event):
"""Event handler for StartEevnt."""
if self.model.config['kind'] not in ('ingress', 'egress'):
self.model.unit.status = BlockedStatus(
'Config item `kind` must be set')
return
if not self.model.relations['istio-pilot']:
self.model.unit.status = BlockedStatus(
"Waiting for istio-pilot relation")
return
if not ((pilot := self.interfaces["istio-pilot"])
and pilot.get_data()):
self.model.unit.status = WaitingStatus(
"Waiting for istio-pilot relation data")
return
def _on_check_status_and_write_config(self, event):
slurm_installed = self._stored.slurm_installed
slurmctld_acquired = self._stored.slurmctld_available
slurm_config = self._stored.slurm_config
if not (slurm_installed and slurmctld_acquired and slurm_config):
if not slurmctld_acquired:
self.unit.status = BlockedStatus("NEED RELATION TO SLURMCTLD")
elif not slurm_config:
self.unit.status = BlockedStatus("NEED SLURM CONFIG")
else:
self.unit.status = BlockedStatus("SLURM NOT INSTALLED")
event.defer()
return
else:
config = dict(self._stored.slurm_config)
self.slurm_manager.render_config_and_restart(config)
self.unit.status = ActiveStatus("Slurmrestd Available")
def _on_start(self, _):
self.model.unit.status = BlockedStatus("certificate not yet acquired.")
try:
# attempt to get a certificate using the defaults, don't
# worry if it fails.
self._get_certificate()
except Exception as err:
logger.info("could not automatically acquire certificate",
exc_info=err)
def _on_check_status_and_write_config(self, event):
slurmdbd_acquired = self.slurmdbd.slurmdbd_acquired
slurmd_acquired = self.slurmd.slurmd_acquired
if not (slurmdbd_acquired and slurmd_acquired):
if not slurmdbd_acquired:
self.unit.status = BlockedStatus("NEED RELATION TO SLURMDBD")
else:
self.unit.status = BlockedStatus("NEED RELATION TO SLURMD")
event.defer()
else:
try:
slurm_config = json.loads(self.slurmd.get_slurm_config())
except json.JSONDecodeError as e:
logger.debug(e)
self.slurm_ops_manager.render_config_and_restart(slurm_config)
logger.debug(slurm_config)
self.unit.status = ActiveStatus("Slurmctld Available")
def _check_mandatory_config(self):
charm_config = self.framework.model.config
missing_config = []
for config in self.MANDATORY_CONFIG:
if charm_config.get(config) is None:
missing_config.append(config)
if missing_config:
self.unit.status = BlockedStatus("Missing mandatory configuration option {}".format(missing_config))
return False
return True
def _get_peers_for_context(self, peers):
try:
peers = yaml.load(io.StringIO(peers))
except yaml.YAMLError:
self.unit.status = BlockedStatus(
"'peers' config property is not valid YAML.")
return None
invalid_peers = [
p for p in peers if not self._is_peer_config_valid(peers[p])
]
if invalid_peers:
self.unit.status = BlockedStatus(
f"Peers with invalid configuration: {invalid_peers}")
return None
peers = list(peers.values())
for p in peers:
p['thruk_id'] = hashlib.md5(
p['nagios_context'].encode('utf-8')).hexdigest()
return peers
def _check_status(self):
slurm_installed = self._stored.slurm_installed
config_available = self._stored.config_available
if not (slurm_installed and config_available):
self.unit.status = BlockedStatus(
"NEED RELATION TO SLURM CONFIGURATOR")
return False
else:
return True
def test_check_config_with_missing_option(self):
self.harness.set_leader(True)
missing_password_config = {'port': 9000, 'admin-password': ''}
with self.assertLogs(level='WARNING') as logger:
self.harness.update_config(missing_password_config)
msg = 'ERROR:charm:Need admin-password config option before setting pod spec.'
self.assertEqual(sorted(logger.output), [msg])
self.assertEqual(
self.harness.model.unit.status,
BlockedStatus("Need 'admin-password' config option."))
def test_validate_config_missing_product_id(self):
self.harness.update_config({'product-id': ''})
self.harness.disable_hooks()
self.harness.begin()
self.assertFalse(self.harness.charm._validate_config())
expected_status = BlockedStatus('Missing configuration: {}'.format(
['product-id']))
self.assertEqual(self.harness.charm.unit.status, expected_status)
def configure_pod(self, _=None) -> NoReturn:
"""Assemble the pod spec and apply it, if possible."""
missing = self._missing_relations()
if missing:
status = "Waiting for {0} relation{1}"
self.unit.status = BlockedStatus(
status.format(missing, "s" if "," in missing else ""))
return
if not self.unit.is_leader():
self.unit.status = ActiveStatus("ready")
return
self.unit.status = MaintenanceStatus("Assembling pod spec")
# Fetch image information
try:
self.unit.status = MaintenanceStatus("Fetching image information")
image_info = self.image.fetch()
except OCIImageResourceError:
self.unit.status = BlockedStatus(
"Error fetching image information")
return
if self.state.upf_host:
try:
pod_spec = make_pod_spec(
image_info,
self.model.config,
self.relation_state,
self.model.app.name,
)
except ValueError as exc:
logger.exception("Config/Relation data validation error")
self.unit.status = BlockedStatus(str(exc))
return
if self.state.pod_spec != pod_spec:
self.model.pod.set_spec(pod_spec)
self.state.pod_spec = pod_spec
self.unit.status = ActiveStatus("ready")
def test_install_apt_packages(self, _add_package, _update):
# Call the method with some packages to install
self.harness.charm._install_apt_packages(["curl", "vim"])
# Check that apt is called with the correct arguments
_update.assert_called_once()
_add_package.assert_called_with(["curl", "vim"])
# Now check that if an exception is raised we do the right logging
_add_package.reset_mock()
_add_package.return_value = 1
_add_package.side_effect = apt.PackageNotFoundError
self.harness.charm._install_apt_packages(["curl", "vim"])
self.assertEqual(self.harness.charm.unit.status,
BlockedStatus("Failed to install packages"))
# Now check that if an exception is raised we do the right logging
_add_package.reset_mock()
_add_package.return_value = 1
_add_package.side_effect = apt.PackageError
self.harness.charm._install_apt_packages(["curl", "vim"])
self.assertEqual(self.harness.charm.unit.status,
BlockedStatus("Failed to install packages"))
def _on_config_changed(self, event):
if self.model.config['protocol'] != 'layer2':
self.unit.status = BlockedStatus(
'Invalid protocol; '
'only "layer2" currently supported')
return
current_config_hash = self._config_hash()
if current_config_hash != self._stored.config_hash:
self._stored.started = False
self._stored.config_hash = current_config_hash
self._on_start(event)
def _on_upgrade(self, event):
"""Upgrade the charm."""
slurm_config = self._assemble_slurm_config()
if not slurm_config:
self.unit.status = BlockedStatus(
"Cannot generate slurm_config, defering upgrade.")
event.defer()
return
self._slurm_manager.upgrade(slurm_config)
def test_on_config_changed_when_unit_is_leader_but_image_fetch_breaks(
self, fetch):
# Given
self.harness.set_leader(True)
fetch.side_effect = OCIImageResourceError("redis-image")
# When
self.harness.charm.on.config_changed.emit()
# Then
fetch.assert_called_once_with()
self.assertEqual(self.harness.charm.unit.status,
BlockedStatus("Error fetching image information."))
def _install_prometheus_client(self):
"""Install Prometheus tester dependencies."""
container = self.unit.get_container(self._name)
if not container.can_connect():
self.unit.status = BlockedStatus("Waiting for Pebble ready")
return
process = container.exec(
[self._pip_path, "install", "prometheus_client"])
process.wait()
logger.debug("Installed prometheus client")
def main(self, event):
self.model.unit.status = MaintenanceStatus("Calculating manifests")
self.ensure_state()
try:
manifest = self.get_manifest()
except Exception as err:
self.model.unit.status = BlockedStatus(str(err))
return
self.model.unit.status = MaintenanceStatus("Applying manifests")
errors = self.set_manifest(manifest)
if errors:
self.model.unit.status = BlockedStatus(
f"There were {len(errors)} errors while applying manifests.")
for error in errors:
self.logger.error(error)
else:
# Ensure requested resources are up
try:
for attempt in Retrying(
retry=retry_if_exception_type(CheckFailedError),
stop=stop_after_delay(
max_delay=self._max_time_checking_resources),
wait=wait_exponential(multiplier=0.1, min=0.1, max=15),
reraise=True,
):
with attempt:
self.logger.info(
f"Checking status of requested resources (attempt "
f"{attempt.retry_state.attempt_number})")
self._check_deployed_resources()
except CheckFailedError:
self.unit.status = BlockedStatus(
"Some Kubernetes resources did not start correctly during install"
)
return
# Otherwise, application is working as expected
self.model.unit.status = ActiveStatus()
def on_upgrade_charm(self, event):
if not hasattr(self.state, 'installed'):
self.on_install(event)
return
try:
check_call(['apt-get', 'update', '-qq'])
check_call(['apt-get', 'dist-upgrade', '-y'])
event.framework.model.unit.status = ActiveStatus()
except Exception as e:
event.framework.model.unit.status = BlockedStatus('{}: {}'.format(
"upgrade failed", e))
event.defer()
def test_validate_config_eula_invalid_product_id(self):
self.harness.update_config({
'accept-eula': True,
'product-id': 'Invalid Product ID'
})
self.harness.disable_hooks()
self.harness.begin()
self.assertFalse(self.harness.charm._validate_config())
self.assertEqual(self.harness.charm.unit.status,
BlockedStatus('Invalid MSSQL product id'))
def _on_relation_changed(self, event):
"""Check for slurmdbd and slurmd, write config, set relation data."""
logger.debug('_on_relation_changed(): entering')
if len(self.framework.model.relations['slurmd']) > 0:
if not self._charm.is_slurmd_available():
self._charm.set_slurmd_available(True)
self.on.slurmd_available.emit()
else:
self._charm.unit.status = BlockedStatus("Need > 0 units of slurmd")
event.defer()
return
def sanitized_container_config(self):
"""Uninterpolated container config without secrets"""
config = self.framework.model.config
if config["container_config"].strip() == "":
container_config = {}
else:
container_config = yaml.safe_load(self.framework.model.config["container_config"])
if not isinstance(container_config, dict):
self.framework.model.unit.status = BlockedStatus("container_config is not a YAML mapping")
return None
return container_config
def configure_pod(self, event):
# Continue only if the unit is the leader
if not self.unit.is_leader():
self.unit.status = ActiveStatus()
return
# Check problems in the settings
problems = self._check_settings()
if problems:
self.unit.status = BlockedStatus(problems)
return
self.unit.status = BlockedStatus("Assembling pod spec")
image_details = self._make_pod_image_details()
ports = self._make_pod_ports()
env_config = self._make_pod_envconfig()
command = self._make_pod_command()
volume_config = self._make_pod_volume_config()
ingress_resources = self._make_pod_ingress_resources()
secrets = self._make_pod_secrets()
pod_spec = {
"version": 3,
"containers": [
{
"name": self.framework.model.app.name,
"imageDetails": image_details,
"ports": ports,
"envConfig": env_config,
"command": command,
"volumeConfig": volume_config,
}
],
"kubernetesResources": {
"ingressResources": ingress_resources or [],
"secrets": secrets,
},
}
self.model.pod.set_spec(pod_spec)
self.unit.status = ActiveStatus()
