def run_rollbacks(experiment: Experiment, configuration: Configuration, secrets: Secrets, pool: ThreadPoolExecutor, dry: bool = False) -> Iterator[Run]: """ Run all rollbacks declared in the experiment in their order. Wait for each rollback activity to complete before to the next unless the activity is declared with the `background` flag. """ rollbacks = experiment.get("rollbacks", []) if not rollbacks: logger.info("No declared rollbacks, let's move on.") for activity in rollbacks: logger.info("Rollback: {t}".format(t=activity.get("name"))) if activity.get("background"): logger.debug("rollback activity will run in the background") yield pool.submit(execute_activity, experiment=experiment, activity=activity, configuration=configuration, secrets=secrets, dry=dry) else: yield execute_activity(experiment, activity, configuration=configuration, secrets=secrets, dry=dry)
def run_steady_state_hypothesis(experiment: Experiment, configuration: Configuration, secrets: Secrets, dry: bool = False): """ Run all probes in the hypothesis and fail the experiment as soon as any of the probe fails or is outside the tolerance zone. """ state = { "steady_state_met": None, "probes": [] } hypo = experiment.get("steady-state-hypothesis") if not hypo: logger.info( "No steady state hypothesis defined. That's ok, just exploring.") return logger.info("Steady state hypothesis: {h}".format(h=hypo.get("title"))) with controls(level="hypothesis", experiment=experiment, context=hypo, configuration=configuration, secrets=secrets) as control: probes = hypo.get("probes", []) control.with_state(state) for activity in probes: run = execute_activity( experiment=experiment, activity=activity, configuration=configuration, secrets=secrets, dry=dry) state["probes"].append(run) if run["status"] == "failed": run["tolerance_met"] = False state["steady_state_met"] = False logger.warn("Probe terminated unexpectedly, " "so its tolerance could not be validated") return state run["tolerance_met"] = True if dry: # do not check for tolerance when dry mode is on continue tolerance = activity.get("tolerance") logger.debug("allowed tolerance is {t}".format(t=str(tolerance))) checked = within_tolerance( tolerance, run["output"], configuration=configuration, secrets=secrets) if not checked: run["tolerance_met"] = False state["steady_state_met"] = False return state state["steady_state_met"] = True logger.info("Steady state hypothesis is met!") return state
def test_controls_are_applied_before_and_after_activities(): exp = deepcopy(experiments.ExperimentWithControls) exp["dry"] = True activities = get_all_activities(exp) for activity in activities: with controls("activity", exp, context=activity): assert activity["before_activity_control"] is True run = execute_activity(exp, activity, None, None, dry=False) assert "after_activity_control" in activity assert activity["after_activity_control"] is True assert run["after_activity_control"] is True
def run_steady_state_hypothesis(experiment: Experiment, configuration: Configuration, secrets: Secrets, dry: bool = False): """ Run all probes in the hypothesis and fail the experiment as soon as any of the probe fails or is outside the tolerance zone. """ state = { "steady_state_met": None, "probes": [] } hypo = experiment.get("steady-state-hypothesis") if not hypo: logger.info( "No steady state hypothesis defined. That's ok, just exploring.") return logger.info("Steady state hypothesis: {h}".format(h=hypo.get("title"))) probes = hypo.get("probes", []) for activity in probes: run = execute_activity( activity, configuration=configuration, secrets=secrets, dry=dry) run["tolerance_met"] = True state["probes"].append(run) if dry: # do not check for tolerance when dry mode is on continue tolerance = activity.get("tolerance") logger.debug("allowed tolerance is {t}".format(t=str(tolerance))) if not within_tolerance(tolerance, run["output"]): run["tolerance_met"] = False state["steady_state_met"] = False return state state["steady_state_met"] = True logger.info("Steady state hypothesis is met!") return state