def configure_control(experiment: Experiment, configuration: Configuration, secrets: Secrets, settings: Settings): if configuration: experiment["control-value"] = configuration.get("dummy-key", "default") elif settings: experiment["control-value"] = settings.get("dummy-key", "default")
def notify( settings: Settings, event: FlowEvent, payload: Any = None, # noqa: C901 error: Any = None, ): """ Go through all the notification channels declared in the settings and call them one by one. Only call those matching the current event. As this function is blocking, make sure none of your channels take too long to run. Whenever an error happened in the notification, a debug message is logged into the chaostoolkit log for review but this should not impact the experiment itself. When no settings were provided, no notifications are sent. Equally, if the settings do not define a `notifications` entry. Here is an example of settings: ```yaml notifications: - type: plugin module: somepackage.somemodule events: - init-failed - run-failed - type: http url: http://example.com headers: Authorization: "Bearer token" - type: http url: https://private.com verify_tls: false forward_event_payload: false headers: Authorization: "Bearer token" events: - discovery-completed - run-failed ``` In this sample, the first channel will be the `notify` function of the `somepackage.somemopdule` Python module. The other two notifications will be sent over HTTP with the third one not forwarding the event payload itself (hence being a GET rather than a POST). Notice how the first and third channels take an `events` sequence. That list represents the events which those endpoints are interested in. In other words, they will only be called for those specific events. The second channel will be applied to all events. The payload event is a dictionary made of the following entries: - `"event"`: the event name - `"payload"`: the payload associated to this event (may be None) - `"phase"`: which phase this event was raised from - `"error"`: if an error was passed on to the function - `"ts"`: a UTC timestamp of when the event was raised """ if not settings: return notification_channels = settings.get("notifications") if not notification_channels: return event_payload = { "name": event.value, "payload": payload, "phase": "unknown", "ts": datetime.utcnow().replace(tzinfo=timezone.utc).timestamp(), } if error: event_payload["error"] = error event_class = event.__class__ if event_class is DiscoverFlowEvent: event_payload["phase"] = "discovery" elif event_class is InitFlowEvent: event_payload["phase"] = "init" elif event_class is RunFlowEvent: event_payload["phase"] = "run" elif event_class is ValidateFlowEvent: event_payload["phase"] = "validate" for channel in notification_channels: events = channel.get("events") if events and event.value not in events: continue channel_type = channel.get("type") if channel_type == "http": notify_with_http(channel, event_payload) elif channel_type == "plugin": notify_via_plugin(channel, event_payload)
def _run( self, strategy: Strategy, schedule: Schedule, # noqa: C901 experiment: Experiment, journal: Journal, configuration: Configuration, secrets: Secrets, settings: Settings, experiment_vars: Dict[str, Any], event_registry: EventHandlerRegistry) -> None: experiment["title"] = substitute(experiment["title"], configuration, secrets) logger.info("Running experiment: {t}".format(t=experiment["title"])) started_at = time.time() journal = journal or initialize_run_journal(experiment) event_registry.started(experiment, journal) config_vars, secret_vars = experiment_vars or (None, None) control = Control() activity_pool, rollback_pool = get_background_pools(experiment) hypo_pool = get_hypothesis_pool() continous_hypo_event = threading.Event() dry = experiment.get("dry", False) if dry: logger.warning("Dry mode enabled") initialize_global_controls(experiment, configuration, secrets, settings) initialize_controls(experiment, configuration, secrets) logger.info("Steady-state strategy: {}".format(strategy.value)) rollback_strategy = settings.get("runtime", {}).get("rollbacks", {}).get( "strategy", "default") logger.info("Rollbacks strategy: {}".format(rollback_strategy)) exit_gracefully_with_rollbacks = True with_ssh = has_steady_state_hypothesis_with_probes(experiment) if not with_ssh: logger.info("No steady state hypothesis defined. That's ok, just " "exploring.") try: try: control.begin("experiment", experiment, experiment, configuration, secrets) state = object() if with_ssh and should_run_before_method(strategy): state = run_gate_hypothesis(experiment, journal, configuration, secrets, event_registry, dry) if state is not None: if with_ssh and should_run_during_method(strategy): run_hypothesis_during_method(hypo_pool, continous_hypo_event, strategy, schedule, experiment, journal, configuration, secrets, event_registry, dry) state = run_method(strategy, activity_pool, experiment, journal, configuration, secrets, event_registry, dry) continous_hypo_event.set() if journal["status"] not in ["interrupted", "aborted"]: if with_ssh and (state is not None) and \ should_run_after_method(strategy): run_deviation_validation_hypothesis( experiment, journal, configuration, secrets, event_registry, dry) except InterruptExecution as i: journal["status"] = "interrupted" logger.fatal(str(i)) event_registry.interrupted(experiment, journal) except KeyboardInterrupt: journal["status"] = "interrupted" logger.warning("Received a termination signal (Ctrl-C)...") event_registry.signal_exit() except SystemExit as x: journal["status"] = "interrupted" logger.warning("Received the exit signal: {}".format(x.code)) exit_gracefully_with_rollbacks = x.code != 30 if not exit_gracefully_with_rollbacks: logger.warning("Ignoring rollbacks as per signal") event_registry.signal_exit() finally: hypo_pool.shutdown(wait=True) # just in case a signal overrode everything else to tell us not to # play them anyway (see the exit.py module) if exit_gracefully_with_rollbacks: run_rollback(rollback_strategy, rollback_pool, experiment, journal, configuration, secrets, event_registry, dry) journal["end"] = datetime.utcnow().isoformat() journal["duration"] = time.time() - started_at # the spec only allows these statuses, so if it's anything else # we override to "completed" if journal["status"] not in ("completed", "failed", "aborted", "interrupted"): journal["status"] = "completed" has_deviated = journal["deviated"] status = "deviated" if has_deviated else journal["status"] logger.info("Experiment ended with status: {s}".format(s=status)) if has_deviated: logger.info( "The steady-state has deviated, a weakness may have been " "discovered") control.with_state(journal) try: control.end("experiment", experiment, experiment, configuration, secrets) except ChaosException: logger.debug("Failed to close controls", exc_info=True) finally: try: cleanup_controls(experiment) cleanup_global_controls() finally: event_registry.finish(journal) return journal