def run_rollbacks(experiment: Experiment,
configuration: Configuration,
secrets: Secrets,
pool: ThreadPoolExecutor,
dry: bool = False) -> Iterator[Run]:
"""
Run all rollbacks declared in the experiment in their order. Wait for
each rollback activity to complete before to the next unless the activity
is declared with the `background` flag.
"""
rollbacks = experiment.get("rollbacks", [])
if not rollbacks:
logger.info("No declared rollbacks, let's move on.")
for activity in rollbacks:
logger.info("Rollback: {t}".format(t=activity.get("name")))
if activity.get("background"):
logger.debug("rollback activity will run in the background")
yield pool.submit(execute_activity,
experiment=experiment,
activity=activity,
configuration=configuration,
secrets=secrets,
dry=dry)
else:
yield execute_activity(experiment,
activity,
configuration=configuration,
secrets=secrets,
dry=dry)
def run_steady_state_hypothesis(experiment: Experiment,
configuration: Configuration, secrets: Secrets,
dry: bool = False):
"""
Run all probes in the hypothesis and fail the experiment as soon as any of
the probe fails or is outside the tolerance zone.
"""
state = {
"steady_state_met": None,
"probes": []
}
hypo = experiment.get("steady-state-hypothesis")
if not hypo:
logger.info(
"No steady state hypothesis defined. That's ok, just exploring.")
return
logger.info("Steady state hypothesis: {h}".format(h=hypo.get("title")))
with controls(level="hypothesis", experiment=experiment, context=hypo,
configuration=configuration, secrets=secrets) as control:
probes = hypo.get("probes", [])
control.with_state(state)
for activity in probes:
run = execute_activity(
experiment=experiment, activity=activity,
configuration=configuration, secrets=secrets, dry=dry)
state["probes"].append(run)
if run["status"] == "failed":
run["tolerance_met"] = False
state["steady_state_met"] = False
logger.warn("Probe terminated unexpectedly, "
"so its tolerance could not be validated")
return state
run["tolerance_met"] = True
if dry:
# do not check for tolerance when dry mode is on
continue
tolerance = activity.get("tolerance")
logger.debug("allowed tolerance is {t}".format(t=str(tolerance)))
checked = within_tolerance(
tolerance, run["output"], configuration=configuration,
secrets=secrets)
if not checked:
run["tolerance_met"] = False
state["steady_state_met"] = False
return state
state["steady_state_met"] = True
logger.info("Steady state hypothesis is met!")
return state
def test_controls_are_applied_before_and_after_activities():
exp = deepcopy(experiments.ExperimentWithControls)
exp["dry"] = True
activities = get_all_activities(exp)
for activity in activities:
with controls("activity", exp, context=activity):
assert activity["before_activity_control"] is True
run = execute_activity(exp, activity, None, None, dry=False)
assert "after_activity_control" in activity
assert activity["after_activity_control"] is True
assert run["after_activity_control"] is True
def run_steady_state_hypothesis(experiment: Experiment,
configuration: Configuration, secrets: Secrets,
dry: bool = False):
"""
Run all probes in the hypothesis and fail the experiment as soon as any of
the probe fails or is outside the tolerance zone.
"""
state = {
"steady_state_met": None,
"probes": []
}
hypo = experiment.get("steady-state-hypothesis")
if not hypo:
logger.info(
"No steady state hypothesis defined. That's ok, just exploring.")
return
logger.info("Steady state hypothesis: {h}".format(h=hypo.get("title")))
probes = hypo.get("probes", [])
for activity in probes:
run = execute_activity(
activity, configuration=configuration, secrets=secrets, dry=dry)
run["tolerance_met"] = True
state["probes"].append(run)
if dry:
# do not check for tolerance when dry mode is on
continue
tolerance = activity.get("tolerance")
logger.debug("allowed tolerance is {t}".format(t=str(tolerance)))
if not within_tolerance(tolerance, run["output"]):
run["tolerance_met"] = False
state["steady_state_met"] = False
return state
state["steady_state_met"] = True
logger.info("Steady state hypothesis is met!")
return state
