def run(path: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False) -> Journal:
"""Run the experiment given at PATH."""
experiment = load_experiment(click.format_filename(path))
settings = load_settings()
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
else:
notify(settings, RunFlowEvent.RunFailed, journal)
return journal
def run(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
no_exit: bool = False,
no_verify_tls: bool = False,
rollback_strategy: str = "default") -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource. SOURCE can be formatted as JSON or YAML."""
settings = load_settings(ctx.obj["settings_path"]) or {}
has_deviated = False
has_failed = False
load_global_controls(settings)
try:
experiment = load_experiment(source,
settings,
verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = dry
settings.setdefault("runtime",
{}).setdefault("rollbacks",
{}).setdefault("strategy",
rollback_strategy)
journal = run_experiment(experiment, settings=settings)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def validate(
ctx: click.Context, source: str, no_verify_tls: bool = False
) -> Experiment:
"""Validate the experiment at SOURCE."""
settings = load_settings(ctx.obj["settings_path"])
try:
experiment = load_experiment(source, settings, verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
try:
notify(settings, ValidateFlowEvent.ValidateStarted, experiment)
ensure_experiment_is_valid(experiment)
notify(settings, ValidateFlowEvent.ValidateCompleted, experiment)
logger.info("experiment syntax and semantic look valid")
except ChaosException as x:
notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x)
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
return experiment
def validate(path: str):
"""Validate the experiment at PATH."""
experiment = load_experiment(click.format_filename(path))
try:
ensure_experiment_is_valid(experiment)
logger.info("experiment syntax and semantic look valid")
except ChaosException as x:
logger.error(str(x))
sys.exit(1)
def run(ctx: click.Context, source: str, journal_path: str = "./journal.json",
dry: bool = False, no_validation: bool = False,
no_exit: bool = False) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"]) or {}
initialize_global_controls(settings)
has_deviated = False
has_failed = False
try:
try:
experiment = load_experiment(
click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
with io.open(journal_path, "w") as r:
json.dump(
journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
finally:
cleanup_global_controls()
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def run(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
fail_fast: bool = True) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"])
try:
experiment = load_experiment(click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
else:
notify(settings, RunFlowEvent.RunFailed, journal)
if journal.get("deviated", False):
notify(settings, RunFlowEvent.RunDeviated, journal)
# when set (default) we exit this command immediatly if the execution
# failed, was aborted or interrupted
# when unset, plugins can continue the processing. In that case, they
# are responsible to set the process exit code accordingly.
if fail_fast:
sys.exit(1)
return journal
def validate(ctx: click.Context, path: str) -> Experiment:
"""Validate the experiment at PATH."""
settings = load_settings(ctx.obj["settings_path"])
experiment = load_experiment(click.format_filename(path))
try:
notify(settings, ValidateFlowEvent.ValidateStarted, experiment)
ensure_experiment_is_valid(experiment)
notify(settings, ValidateFlowEvent.ValidateCompleted, experiment)
logger.info("experiment syntax and semantic look valid")
except ChaosException as x:
notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x)
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
return experiment
def run(path: str,
report_path: str = "./chaos-report.json",
dry: bool = False,
no_validation: bool = False):
"""Run the plan given at PATH."""
experiment = load_experiment(click.format_filename(path))
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
journal = run_experiment(experiment)
with io.open(report_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False)
def run(path: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False) -> Journal:
"""Run the experiment given at PATH."""
experiment = load_experiment(click.format_filename(path))
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False)
return journal
def ensure_verification_is_valid(experiment: Experiment):
ensure_experiment_is_valid(experiment)
extensions = experiment.get("extensions")
if extensions is None:
raise InvalidVerification(
"a verification must have an extensions block")
chaosiq_blocks = list(
filter(lambda extension: extension.get("name", "") == "chaosiq",
extensions))
if not len(chaosiq_blocks) == 1:
raise InvalidVerification(
"a verification must have a single chaosiq extension block")
verification = chaosiq_blocks[0].get("verification")
if verification is None:
raise InvalidVerification(
"a verification must have a verification block")
id = verification.get("id")
if id is None:
raise InvalidVerification("a verification must have an id")
frequency_of_measurement = verification.get("frequency-of-measurement")
if frequency_of_measurement is None:
raise InvalidVerification(
"a verification must have a frequency-of-measurement block")
duration_of_conditions = verification.get("duration-of-conditions")
if duration_of_conditions is None:
raise InvalidVerification(
"a verification must have a duration-of-conditions block")
logger.info("Verification looks valid")
def test_experiment_must_have_a_title():
with pytest.raises(InvalidExperiment) as exc:
ensure_experiment_is_valid(experiments.MissingTitleExperiment)
assert "experiment requires a title" in str(exc.value)
def test_experiment_method_without_steps():
ensure_experiment_is_valid(experiments.NoStepsMethodExperiment)
def test_experiment_must_have_a_method():
with pytest.raises(InvalidExperiment) as exc:
ensure_experiment_is_valid(experiments.MissingMethodExperiment)
assert "an experiment requires a method" in str(exc.value)
def test_validate_all_tolerance_probes():
with requests_mock.mock() as m:
m.get("http://example.com", text="you are number 87")
ensure_experiment_is_valid(experiments.ExperimentWithVariousTolerances)
def test_empty_experiment_is_invalid():
with pytest.raises(InvalidExperiment) as exc:
ensure_experiment_is_valid(experiments.EmptyExperiment)
assert "an empty experiment is not an experiment" in str(exc.value)
def test_valid_experiment_from_yaml():
with tempfile.NamedTemporaryFile(suffix=".yaml") as f:
f.write(yaml.dump(experiments.Experiment).encode('utf-8'))
f.seek(0)
doc = load_experiment(f.name)
assert ensure_experiment_is_valid(doc) is None
def test_experiment_hypothesis_must_have_a_valid_probe():
with pytest.raises(InvalidActivity) as exc:
ensure_experiment_is_valid(experiments.ExperimentWithInvalidHypoProbe)
assert "required argument 'path' is missing from activity" in str(
exc.value)
def test_control_can_validate_itself():
exp = deepcopy(experiments.ExperimentWithInvalidControls)
with pytest.raises(InvalidActivity):
ensure_experiment_is_valid(exp)
def test_valid_experiment_from_json():
with tempfile.NamedTemporaryFile(suffix=".json") as f:
f.write(json.dumps(experiments.Experiment).encode("utf-8"))
f.seek(0)
doc = load_experiment(f.name)
assert ensure_experiment_is_valid(doc) is None
def test_experiment_must_have_a_method():
with pytest.raises(InvalidExperiment) as exc:
ensure_experiment_is_valid(experiments.MissingMethodExperiment)
assert "an experiment requires a method with "\
"at least one activity" in str(exc)
def test_experiment_must_have_at_least_one_step():
with pytest.raises(InvalidExperiment) as exc:
ensure_experiment_is_valid(experiments.NoStepsMethodExperiment)
assert "an experiment requires a method with "\
"at least one activity" in str(exc.value)
def test_experiment_hypothesis_must_have_a_title():
with pytest.raises(InvalidExperiment) as exc:
ensure_experiment_is_valid(
experiments.MissingHypothesisTitleExperiment)
assert "hypothesis requires a title" in str(exc)
def test_experiment_must_have_a_description():
with pytest.raises(InvalidExperiment) as exc:
ensure_experiment_is_valid(experiments.MissingDescriptionExperiment)
assert "experiment requires a description" in str(exc.value)
def test_valid_experiment():
assert ensure_experiment_is_valid(experiments.Experiment) is None
def test_experiment_may_not_have_a_hypothesis():
assert ensure_experiment_is_valid(
experiments.MissingHypothesisExperiment) is None
def run(
ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: Optional[str] = None,
no_validation: bool = False,
no_exit: bool = False,
no_verify_tls: bool = False,
rollback_strategy: str = "default",
var: Dict[str, Any] = None,
var_file: List[str] = None,
hypothesis_strategy: str = "default",
hypothesis_frequency: float = 1.0,
fail_fast: bool = False,
) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource. SOURCE can be formatted as JSON or YAML."""
settings = load_settings(ctx.obj["settings_path"]) or {}
has_deviated = False
has_failed = False
experiment_vars = merge_vars(var, var_file)
load_global_controls(settings)
try:
experiment = load_experiment(source, settings, verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = Dry.from_string(dry)
settings.setdefault("runtime", {}).setdefault("rollbacks", {}).setdefault(
"strategy", rollback_strategy
)
hypothesis_strategy = check_hypothesis_strategy_spelling(hypothesis_strategy)
schedule = Schedule(
continuous_hypothesis_frequency=hypothesis_frequency, fail_fast=fail_fast
)
journal = run_experiment(
experiment,
settings=settings,
strategy=hypothesis_strategy,
schedule=schedule,
experiment_vars=experiment_vars,
)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
if "dry" in journal["experiment"]:
journal["experiment"]["dry"] = dry
with open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
