def configure_control(experiment: Experiment, configuration: Configuration,
secrets: Secrets, settings: Settings):
if configuration:
experiment["control-value"] = configuration.get("dummy-key", "default")
elif settings:
experiment["control-value"] = settings.get("dummy-key", "default")
def notify(
settings: Settings,
event: FlowEvent,
payload: Any = None, # noqa: C901
error: Any = None,
):
"""
Go through all the notification channels declared in the settings and
call them one by one. Only call those matching the current event.
As this function is blocking, make sure none of your channels take too
long to run.
Whenever an error happened in the notification, a debug message is logged
into the chaostoolkit log for review but this should not impact the
experiment itself.
When no settings were provided, no notifications are sent. Equally, if the
settings do not define a `notifications` entry. Here is an example of
settings:
```yaml
notifications:
-
type: plugin
module: somepackage.somemodule
events:
- init-failed
- run-failed
-
type: http
url: http://example.com
headers:
Authorization: "Bearer token"
-
type: http
url: https://private.com
verify_tls: false
forward_event_payload: false
headers:
Authorization: "Bearer token"
events:
- discovery-completed
- run-failed
```
In this sample, the first channel will be the `notify` function of the
`somepackage.somemopdule` Python module. The other two notifications will
be sent over HTTP with the third one not forwarding the event payload
itself (hence being a GET rather than a POST).
Notice how the first and third channels take an `events` sequence. That
list represents the events which those endpoints are interested in. In
other words, they will only be called for those specific events. The second
channel will be applied to all events.
The payload event is a dictionary made of the following entries:
- `"event"`: the event name
- `"payload"`: the payload associated to this event (may be None)
- `"phase"`: which phase this event was raised from
- `"error"`: if an error was passed on to the function
- `"ts"`: a UTC timestamp of when the event was raised
"""
if not settings:
return
notification_channels = settings.get("notifications")
if not notification_channels:
return
event_payload = {
"name": event.value,
"payload": payload,
"phase": "unknown",
"ts": datetime.utcnow().replace(tzinfo=timezone.utc).timestamp(),
}
if error:
event_payload["error"] = error
event_class = event.__class__
if event_class is DiscoverFlowEvent:
event_payload["phase"] = "discovery"
elif event_class is InitFlowEvent:
event_payload["phase"] = "init"
elif event_class is RunFlowEvent:
event_payload["phase"] = "run"
elif event_class is ValidateFlowEvent:
event_payload["phase"] = "validate"
for channel in notification_channels:
events = channel.get("events")
if events and event.value not in events:
continue
channel_type = channel.get("type")
if channel_type == "http":
notify_with_http(channel, event_payload)
elif channel_type == "plugin":
notify_via_plugin(channel, event_payload)
def _run(
self,
strategy: Strategy,
schedule: Schedule, # noqa: C901
experiment: Experiment,
journal: Journal,
configuration: Configuration,
secrets: Secrets,
settings: Settings,
experiment_vars: Dict[str, Any],
event_registry: EventHandlerRegistry) -> None:
experiment["title"] = substitute(experiment["title"], configuration,
secrets)
logger.info("Running experiment: {t}".format(t=experiment["title"]))
started_at = time.time()
journal = journal or initialize_run_journal(experiment)
event_registry.started(experiment, journal)
config_vars, secret_vars = experiment_vars or (None, None)
control = Control()
activity_pool, rollback_pool = get_background_pools(experiment)
hypo_pool = get_hypothesis_pool()
continous_hypo_event = threading.Event()
dry = experiment.get("dry", False)
if dry:
logger.warning("Dry mode enabled")
initialize_global_controls(experiment, configuration, secrets,
settings)
initialize_controls(experiment, configuration, secrets)
logger.info("Steady-state strategy: {}".format(strategy.value))
rollback_strategy = settings.get("runtime",
{}).get("rollbacks", {}).get(
"strategy", "default")
logger.info("Rollbacks strategy: {}".format(rollback_strategy))
exit_gracefully_with_rollbacks = True
with_ssh = has_steady_state_hypothesis_with_probes(experiment)
if not with_ssh:
logger.info("No steady state hypothesis defined. That's ok, just "
"exploring.")
try:
try:
control.begin("experiment", experiment, experiment,
configuration, secrets)
state = object()
if with_ssh and should_run_before_method(strategy):
state = run_gate_hypothesis(experiment, journal,
configuration, secrets,
event_registry, dry)
if state is not None:
if with_ssh and should_run_during_method(strategy):
run_hypothesis_during_method(hypo_pool,
continous_hypo_event,
strategy, schedule,
experiment, journal,
configuration, secrets,
event_registry, dry)
state = run_method(strategy, activity_pool, experiment,
journal, configuration, secrets,
event_registry, dry)
continous_hypo_event.set()
if journal["status"] not in ["interrupted", "aborted"]:
if with_ssh and (state is not None) and \
should_run_after_method(strategy):
run_deviation_validation_hypothesis(
experiment, journal, configuration, secrets,
event_registry, dry)
except InterruptExecution as i:
journal["status"] = "interrupted"
logger.fatal(str(i))
event_registry.interrupted(experiment, journal)
except KeyboardInterrupt:
journal["status"] = "interrupted"
logger.warning("Received a termination signal (Ctrl-C)...")
event_registry.signal_exit()
except SystemExit as x:
journal["status"] = "interrupted"
logger.warning("Received the exit signal: {}".format(x.code))
exit_gracefully_with_rollbacks = x.code != 30
if not exit_gracefully_with_rollbacks:
logger.warning("Ignoring rollbacks as per signal")
event_registry.signal_exit()
finally:
hypo_pool.shutdown(wait=True)
# just in case a signal overrode everything else to tell us not to
# play them anyway (see the exit.py module)
if exit_gracefully_with_rollbacks:
run_rollback(rollback_strategy, rollback_pool, experiment,
journal, configuration, secrets, event_registry,
dry)
journal["end"] = datetime.utcnow().isoformat()
journal["duration"] = time.time() - started_at
# the spec only allows these statuses, so if it's anything else
# we override to "completed"
if journal["status"] not in ("completed", "failed", "aborted",
"interrupted"):
journal["status"] = "completed"
has_deviated = journal["deviated"]
status = "deviated" if has_deviated else journal["status"]
logger.info("Experiment ended with status: {s}".format(s=status))
if has_deviated:
logger.info(
"The steady-state has deviated, a weakness may have been "
"discovered")
control.with_state(journal)
try:
control.end("experiment", experiment, experiment,
configuration, secrets)
except ChaosException:
logger.debug("Failed to close controls", exc_info=True)
finally:
try:
cleanup_controls(experiment)
cleanup_global_controls()
finally:
event_registry.finish(journal)
return journal