def test_no_controls_get_applied_when_none_defined():
exp = deepcopy(experiments.ExperimentWithoutControls)
exp["dry"] = True
with controls("experiment", exp, context=exp) as c:
assert "before_experiment_control" not in exp
exp["dry"] = True
journal = run_experiment(exp)
assert "after_experiment_control" not in exp
def test_controls_are_applied_before_and_but_not_after_experiment():
exp = deepcopy(experiments.ExperimentWithControls)
exp["controls"][0]["scope"] = "before"
with controls("experiment", exp, context=exp) as c:
assert "before_experiment_control" in exp
assert exp["before_experiment_control"] is True
exp["dry"] = True
journal = run_experiment(exp)
assert "after_experiment_control" not in exp
def test_control_cleanup_cannot_fail_the_experiment():
exp = deepcopy(experiments.ExperimentNoControls)
try:
run_experiment(
exp,
settings={
"dummy-key": "hello there",
"controls": {
"dummy": {
"provider": {
"type":
"python",
"module":
"fixtures.controls.dummy_with_failing_cleanup",
}
}
},
},
)
except Exception:
pytest.fail("Failed to run experiment with a broken cleanup control")
def run_chaos_engine(self, file, env_params: dict, report: str, report_endpoint: str, engineName) -> bool:
"""
Runs chaos engine programmatically instead of using chaos binary
:param file:
:param env_params:
:param report:
:param report_endpoint:
:return:
"""
settings = ({}, os.environ.get("settings_path"))[os.environ.get("settings_path") is not None]
has_deviated = False
has_failed = False
load_global_controls(settings)
jornal_file_suffix = file
try:
try:
with open(file, "r"):
logger.info("File exists in local")
except FileNotFoundError:
logger.info("File is not available in the current directory, looking inside site packages")
location = site.getsitepackages().__getitem__(0)
file_found = False
for root, dirs, files in os.walk(location):
if file in files:
file_found = True
file = os.path.join(root, file)
break
if not file_found:
logger.error("File " + file + " not found in site packages too, quitting")
raise FileNotFoundError("Chaos file is not found")
experiment = load_experiment(
click.format_filename(file), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
logger.info("chaos json file found, proceeding with test")
journal = run_experiment(experiment, settings=settings)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
json_file_name = "journal" + "-" + jornal_file_suffix
with open(json_file_name, "w") as r:
json.dump(
journal, r, indent=2, ensure_ascii=False, default=encoder)
r.close()
if report == 'true':
self.create_report(os.environ, journal, report_endpoint)
if has_failed or has_deviated:
logger.error("Test Failed")
return has_failed and has_deviated
else:
logger.info("Test Passed")
return True
def test_controls_are_applied_before_and_after_experiment():
exp = deepcopy(experiments.ExperimentWithControls)
with controls("experiment", exp, context=exp):
assert "before_experiment_control" in exp
assert exp["before_experiment_control"] is True
exp["dry"] = True
journal = run_experiment(exp)
assert "after_experiment_control" in exp
assert exp["after_experiment_control"] is True
assert journal["after_experiment_control"] is True
def test_controls_are_applied_before_and_after_rollbacks():
exp = deepcopy(experiments.ExperimentWithControls)
with controls("rollback", exp, context=exp):
assert "before_rollback_control" in exp
assert exp["before_rollback_control"] is True
exp["dry"] = Dry.ACTIVITIES
journal = run_experiment(exp)
assert "after_rollback_control" in exp
assert exp["after_rollback_control"] is True
assert "after_rollback_control" in journal["rollbacks"]
def run(ctx: click.Context, source: str, journal_path: str = "./journal.json",
dry: bool = False, no_validation: bool = False,
no_exit: bool = False) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"]) or {}
initialize_global_controls(settings)
has_deviated = False
has_failed = False
try:
try:
experiment = load_experiment(
click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
with io.open(journal_path, "w") as r:
json.dump(
journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
finally:
cleanup_global_controls()
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def test_can_interrupt_rollbacks_on_SIGINT():
def handler(signum, frame):
raise KeyboardInterrupt()
signal.signal(signal.SIGALRM, handler)
signal.alarm(1)
try:
journal = run_experiment(experiments.ExperimentWithRollbackLongPause)
assert isinstance(journal, dict)
assert journal["status"] == "interrupted"
except SystemExit:
pytest.fail("we should have swallowed the KeyboardInterrupt exception")
def test_no_rollback_even_on_SystemExit():
def handler(signum, frame):
raise SystemExit()
signal.signal(signal.SIGALRM, handler)
signal.alarm(1)
try:
journal = run_experiment(experiments.ExperimentWithLongPause)
assert isinstance(journal, dict)
assert journal["status"] == "interrupted"
except SystemExit:
pytest.fail("we should have swalled the SystemExit exception")
def test_can_interrupt_rollbacks():
def handler(signum, frame):
raise InterruptExecution("boom")
signal.signal(signal.SIGALRM, handler)
signal.alarm(1)
try:
journal = run_experiment(experiments.ExperimentWithRollbackLongPause)
assert isinstance(journal, dict)
assert journal["status"] == "interrupted"
except Exception:
pytest.fail("we should have swalled the InterruptExecution exception")
def test_controls_are_applied_before_and_after_hypothesis():
exp = deepcopy(experiments.ExperimentWithControls)
hypo = exp["steady-state-hypothesis"]
with controls("hypothesis", exp, context=hypo):
assert "before_hypothesis_control" in hypo
assert hypo["before_hypothesis_control"] is True
exp["dry"] = True
journal = run_experiment(exp)
assert "after_hypothesis_control" in hypo
assert hypo["after_hypothesis_control"] is True
assert journal["steady_states"]["before"][
"after_hypothesis_control"] is True
def run(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
fail_fast: bool = True) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"])
try:
experiment = load_experiment(click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
else:
notify(settings, RunFlowEvent.RunFailed, journal)
if journal.get("deviated", False):
notify(settings, RunFlowEvent.RunDeviated, journal)
# when set (default) we exit this command immediatly if the execution
# failed, was aborted or interrupted
# when unset, plugins can continue the processing. In that case, they
# are responsible to set the process exit code accordingly.
if fail_fast:
sys.exit(1)
return journal
def test_do_not_exit_when_continous_ssh_fails_and_no_failfast():
experiment = experiments.SimpleExperimentWithSSHFailingAtSomePoint.copy()
journal = run_experiment(
experiment,
strategy=Strategy.CONTINOUS,
schedule=Schedule(continous_hypothesis_frequency=0.1, fail_fast=False),
settings={"runtime": {
"rollbacks": {
"strategy": "always"
}
}})
assert journal is not None
assert journal["steady_states"]["before"] is not None
assert journal["steady_states"]["after"] is not None
assert journal["steady_states"]["during"] is not None
assert journal["status"] == "failed"
assert journal["deviated"] == True
assert len(journal["run"]) == 2
def run(path: str,
report_path: str = "./chaos-report.json",
dry: bool = False,
no_validation: bool = False):
"""Run the plan given at PATH."""
experiment = load_experiment(click.format_filename(path))
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
journal = run_experiment(experiment)
with io.open(report_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False)
def test_exit_continous_ssh_continous_when_activity_raises_unknown_exception():
experiment = experiments.SimpleExperimentWithException.copy()
journal = run_experiment(
experiment,
strategy=Strategy.CONTINOUS,
schedule=Schedule(continous_hypothesis_frequency=0.1),
settings={"runtime": {
"rollbacks": {
"strategy": "always"
}
}})
assert journal is not None
assert journal["steady_states"]["before"] is not None
assert journal["steady_states"]["after"] is not None
assert journal["steady_states"]["during"] is not None
assert journal["deviated"] == False
assert journal["status"] == "completed"
assert len(journal["run"]) == 2
assert journal["run"][-1]["status"] == "failed"
assert "oops" in journal["run"][-1]["exception"][-1]
def run(path: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False) -> Journal:
"""Run the experiment given at PATH."""
experiment = load_experiment(click.format_filename(path))
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False)
return journal
def test_control_must_not_rest_state_before_calling_the_after_side():
exp = deepcopy(experiments.ExperimentNoControlsWithDeviation)
settings = {
"controls": {
"dummy": {
"provider": {
"type": "python",
"module":
"fixtures.controls.dummy_need_access_to_end_state"
}
}
}
}
load_global_controls(settings)
journal = run_experiment(exp, settings)
before_hypo_result = journal["steady_states"]["before"]
assert "after_hypothesis_control" in before_hypo_result
assert before_hypo_result["after_hypothesis_control"] == True
assert "after_experiment_control" in journal
assert journal["after_experiment_control"] == True
def test_exit_immediately_when_continuous_ssh_fails_and_failfast_when_background_activity(
): # noqa E501
experiment = (
experiments.
SimpleExperimentWithSSHFailingAtSomePointWithBackgroundActivity.copy(
) # noqa E501
)
journal = run_experiment(
experiment,
strategy=Strategy.CONTINUOUS,
schedule=Schedule(continuous_hypothesis_frequency=0.1, fail_fast=True),
settings={"runtime": {
"rollbacks": {
"strategy": "always"
}
}},
)
assert journal is not None
assert journal["steady_states"]["before"] is not None
assert journal["steady_states"]["after"] is not None
assert journal["steady_states"]["during"] is not None
assert journal["status"] == "failed"
assert journal["deviated"] is True
assert len(journal["run"]) == 2
def test_can_run_experiment_in_probeless_mode():
experiment = experiments.Experiment.copy()
experiment["dry"] = Dry.PROBES
journal = run_experiment(experiment)
assert isinstance(journal, dict)
def test_controls_may_interrupt_experiment():
exp = deepcopy(experiments.ExperimentCanBeInterruptedByControl)
with controls("experiment", exp, context=exp):
exp["dry"] = True
journal = run_experiment(exp)
assert journal["status"] == "interrupted"
def test_can_run_experiment_with_activity_in_dry_mode():
experiment = experiments.ExperimentWithBypassedActivity.copy()
journal = run_experiment(experiment)
assert isinstance(journal, dict)
assert journal["run"][0]["output"] is None
def test_control_can_update_secrets():
exp = deepcopy(experiments.ExperimentWithControlsThatUpdatedSecrets)
state = run_experiment(exp)
assert state["run"][0]["output"] != "UNSET"
def test_control_can_update_configuration():
exp = deepcopy(experiments.ExperimentWithControlsThatUpdatedConfiguration)
state = run_experiment(exp)
assert state["run"][0]["output"] != "UNSET"
def test_can_run_experiment_in_dry_mode():
experiment = experiments.Experiment.copy()
experiment["dry"] = True
journal = run_experiment(experiment)
assert isinstance(journal, dict)
def test_can_run_experiment_in_actionless_mode():
experiment = experiments.ExperimentWithLongPauseAction.copy()
experiment["dry"] = Dry.ACTIONS
journal = run_experiment(experiment)
assert isinstance(journal, dict)
def test_probes_missing_ref_should_fail_the_experiment():
experiment = experiments.MissingRefProbeExperiment.copy()
experiment["dry"] = True
journal = run_experiment(experiment)
assert journal["status"] == "aborted"
def test_probes_can_reference_each_other():
experiment = experiments.RefProbeExperiment.copy()
try:
run_experiment(experiment)
except Exception:
pytest.fail("experiment should not have failed")
def test_can_run_experiment_in_pauseless_mode():
experiment = experiments.ExperimentWithLongPause.copy()
experiment["dry"] = Dry.PAUSE
journal = run_experiment(experiment)
assert isinstance(journal, dict)
def test_experiment_title_substitution():
journal = run_experiment(experiments.ExperimentWithInterpolatedTitle)
assert journal["experiment"][
"title"] in "Cats in space: do cats live in the Internet?"
def test_experiment_may_run_without_steady_state():
experiment = experiments.Experiment.copy()
experiment.pop("steady-state-hypothesis")
journal = run_experiment(experiment)
assert journal is not None
