def run(path: str, journal_path: str = "./journal.json", dry: bool = False, no_validation: bool = False) -> Journal: """Run the experiment given at PATH.""" experiment = load_experiment(click.format_filename(path)) settings = load_settings() notify(settings, RunFlowEvent.RunStarted, experiment) if not no_validation: try: ensure_experiment_is_valid(experiment) except ChaosException as x: logger.error(str(x)) logger.debug(x) sys.exit(1) experiment["dry"] = dry journal = run_experiment(experiment) with io.open(journal_path, "w") as r: json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder) if journal["status"] == "completed": notify(settings, RunFlowEvent.RunCompleted, journal) else: notify(settings, RunFlowEvent.RunFailed, journal) return journal
def run(ctx: click.Context, source: str, journal_path: str = "./journal.json", dry: bool = False, no_validation: bool = False, no_exit: bool = False, no_verify_tls: bool = False, rollback_strategy: str = "default") -> Journal: """Run the experiment loaded from SOURCE, either a local file or a HTTP resource. SOURCE can be formatted as JSON or YAML.""" settings = load_settings(ctx.obj["settings_path"]) or {} has_deviated = False has_failed = False load_global_controls(settings) try: experiment = load_experiment(source, settings, verify_tls=not no_verify_tls) except InvalidSource as x: logger.error(str(x)) logger.debug(x) ctx.exit(1) notify(settings, RunFlowEvent.RunStarted, experiment) if not no_validation: try: ensure_experiment_is_valid(experiment) except ChaosException as x: logger.error(str(x)) logger.debug(x) ctx.exit(1) experiment["dry"] = dry settings.setdefault("runtime", {}).setdefault("rollbacks", {}).setdefault("strategy", rollback_strategy) journal = run_experiment(experiment, settings=settings) has_deviated = journal.get("deviated", False) has_failed = journal["status"] != "completed" with io.open(journal_path, "w") as r: json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder) if journal["status"] == "completed": notify(settings, RunFlowEvent.RunCompleted, journal) elif has_failed: notify(settings, RunFlowEvent.RunFailed, journal) if has_deviated: notify(settings, RunFlowEvent.RunDeviated, journal) if (has_failed or has_deviated) and not no_exit: ctx.exit(1) return journal
def validate( ctx: click.Context, source: str, no_verify_tls: bool = False ) -> Experiment: """Validate the experiment at SOURCE.""" settings = load_settings(ctx.obj["settings_path"]) try: experiment = load_experiment(source, settings, verify_tls=not no_verify_tls) except InvalidSource as x: logger.error(str(x)) logger.debug(x) ctx.exit(1) try: notify(settings, ValidateFlowEvent.ValidateStarted, experiment) ensure_experiment_is_valid(experiment) notify(settings, ValidateFlowEvent.ValidateCompleted, experiment) logger.info("experiment syntax and semantic look valid") except ChaosException as x: notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x) logger.error(str(x)) logger.debug(x) ctx.exit(1) return experiment
def validate(path: str): """Validate the experiment at PATH.""" experiment = load_experiment(click.format_filename(path)) try: ensure_experiment_is_valid(experiment) logger.info("experiment syntax and semantic look valid") except ChaosException as x: logger.error(str(x)) sys.exit(1)
def run(ctx: click.Context, source: str, journal_path: str = "./journal.json", dry: bool = False, no_validation: bool = False, no_exit: bool = False) -> Journal: """Run the experiment loaded from SOURCE, either a local file or a HTTP resource.""" settings = load_settings(ctx.obj["settings_path"]) or {} initialize_global_controls(settings) has_deviated = False has_failed = False try: try: experiment = load_experiment( click.format_filename(source), settings) except InvalidSource as x: logger.error(str(x)) logger.debug(x) ctx.exit(1) notify(settings, RunFlowEvent.RunStarted, experiment) if not no_validation: try: ensure_experiment_is_valid(experiment) except ChaosException as x: logger.error(str(x)) logger.debug(x) ctx.exit(1) experiment["dry"] = dry journal = run_experiment(experiment) has_deviated = journal.get("deviated", False) has_failed = journal["status"] != "completed" with io.open(journal_path, "w") as r: json.dump( journal, r, indent=2, ensure_ascii=False, default=encoder) if journal["status"] == "completed": notify(settings, RunFlowEvent.RunCompleted, journal) elif has_failed: notify(settings, RunFlowEvent.RunFailed, journal) if has_deviated: notify(settings, RunFlowEvent.RunDeviated, journal) finally: cleanup_global_controls() if (has_failed or has_deviated) and not no_exit: ctx.exit(1) return journal
def run(ctx: click.Context, source: str, journal_path: str = "./journal.json", dry: bool = False, no_validation: bool = False, fail_fast: bool = True) -> Journal: """Run the experiment loaded from SOURCE, either a local file or a HTTP resource.""" settings = load_settings(ctx.obj["settings_path"]) try: experiment = load_experiment(click.format_filename(source), settings) except InvalidSource as x: logger.error(str(x)) logger.debug(x) sys.exit(1) notify(settings, RunFlowEvent.RunStarted, experiment) if not no_validation: try: ensure_experiment_is_valid(experiment) except ChaosException as x: logger.error(str(x)) logger.debug(x) sys.exit(1) experiment["dry"] = dry journal = run_experiment(experiment) with io.open(journal_path, "w") as r: json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder) if journal["status"] == "completed": notify(settings, RunFlowEvent.RunCompleted, journal) else: notify(settings, RunFlowEvent.RunFailed, journal) if journal.get("deviated", False): notify(settings, RunFlowEvent.RunDeviated, journal) # when set (default) we exit this command immediatly if the execution # failed, was aborted or interrupted # when unset, plugins can continue the processing. In that case, they # are responsible to set the process exit code accordingly. if fail_fast: sys.exit(1) return journal
def validate(ctx: click.Context, path: str) -> Experiment: """Validate the experiment at PATH.""" settings = load_settings(ctx.obj["settings_path"]) experiment = load_experiment(click.format_filename(path)) try: notify(settings, ValidateFlowEvent.ValidateStarted, experiment) ensure_experiment_is_valid(experiment) notify(settings, ValidateFlowEvent.ValidateCompleted, experiment) logger.info("experiment syntax and semantic look valid") except ChaosException as x: notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x) logger.error(str(x)) logger.debug(x) ctx.exit(1) return experiment
def run(path: str, report_path: str = "./chaos-report.json", dry: bool = False, no_validation: bool = False): """Run the plan given at PATH.""" experiment = load_experiment(click.format_filename(path)) if not no_validation: try: ensure_experiment_is_valid(experiment) except ChaosException as x: logger.error(str(x)) logger.debug(x) sys.exit(1) journal = run_experiment(experiment) with io.open(report_path, "w") as r: json.dump(journal, r, indent=2, ensure_ascii=False)
def run(path: str, journal_path: str = "./journal.json", dry: bool = False, no_validation: bool = False) -> Journal: """Run the experiment given at PATH.""" experiment = load_experiment(click.format_filename(path)) if not no_validation: try: ensure_experiment_is_valid(experiment) except ChaosException as x: logger.error(str(x)) logger.debug(x) sys.exit(1) experiment["dry"] = dry journal = run_experiment(experiment) with io.open(journal_path, "w") as r: json.dump(journal, r, indent=2, ensure_ascii=False) return journal
def ensure_verification_is_valid(experiment: Experiment): ensure_experiment_is_valid(experiment) extensions = experiment.get("extensions") if extensions is None: raise InvalidVerification( "a verification must have an extensions block") chaosiq_blocks = list( filter(lambda extension: extension.get("name", "") == "chaosiq", extensions)) if not len(chaosiq_blocks) == 1: raise InvalidVerification( "a verification must have a single chaosiq extension block") verification = chaosiq_blocks[0].get("verification") if verification is None: raise InvalidVerification( "a verification must have a verification block") id = verification.get("id") if id is None: raise InvalidVerification("a verification must have an id") frequency_of_measurement = verification.get("frequency-of-measurement") if frequency_of_measurement is None: raise InvalidVerification( "a verification must have a frequency-of-measurement block") duration_of_conditions = verification.get("duration-of-conditions") if duration_of_conditions is None: raise InvalidVerification( "a verification must have a duration-of-conditions block") logger.info("Verification looks valid")
def test_experiment_must_have_a_title(): with pytest.raises(InvalidExperiment) as exc: ensure_experiment_is_valid(experiments.MissingTitleExperiment) assert "experiment requires a title" in str(exc.value)
def test_experiment_method_without_steps(): ensure_experiment_is_valid(experiments.NoStepsMethodExperiment)
def test_experiment_must_have_a_method(): with pytest.raises(InvalidExperiment) as exc: ensure_experiment_is_valid(experiments.MissingMethodExperiment) assert "an experiment requires a method" in str(exc.value)
def test_validate_all_tolerance_probes(): with requests_mock.mock() as m: m.get("http://example.com", text="you are number 87") ensure_experiment_is_valid(experiments.ExperimentWithVariousTolerances)
def test_empty_experiment_is_invalid(): with pytest.raises(InvalidExperiment) as exc: ensure_experiment_is_valid(experiments.EmptyExperiment) assert "an empty experiment is not an experiment" in str(exc.value)
def test_valid_experiment_from_yaml(): with tempfile.NamedTemporaryFile(suffix=".yaml") as f: f.write(yaml.dump(experiments.Experiment).encode('utf-8')) f.seek(0) doc = load_experiment(f.name) assert ensure_experiment_is_valid(doc) is None
def test_experiment_hypothesis_must_have_a_valid_probe(): with pytest.raises(InvalidActivity) as exc: ensure_experiment_is_valid(experiments.ExperimentWithInvalidHypoProbe) assert "required argument 'path' is missing from activity" in str( exc.value)
def test_control_can_validate_itself(): exp = deepcopy(experiments.ExperimentWithInvalidControls) with pytest.raises(InvalidActivity): ensure_experiment_is_valid(exp)
def test_valid_experiment_from_json(): with tempfile.NamedTemporaryFile(suffix=".json") as f: f.write(json.dumps(experiments.Experiment).encode("utf-8")) f.seek(0) doc = load_experiment(f.name) assert ensure_experiment_is_valid(doc) is None
def test_experiment_must_have_a_method(): with pytest.raises(InvalidExperiment) as exc: ensure_experiment_is_valid(experiments.MissingMethodExperiment) assert "an experiment requires a method with "\ "at least one activity" in str(exc)
def test_experiment_must_have_at_least_one_step(): with pytest.raises(InvalidExperiment) as exc: ensure_experiment_is_valid(experiments.NoStepsMethodExperiment) assert "an experiment requires a method with "\ "at least one activity" in str(exc.value)
def test_experiment_hypothesis_must_have_a_title(): with pytest.raises(InvalidExperiment) as exc: ensure_experiment_is_valid( experiments.MissingHypothesisTitleExperiment) assert "hypothesis requires a title" in str(exc)
def test_experiment_must_have_a_description(): with pytest.raises(InvalidExperiment) as exc: ensure_experiment_is_valid(experiments.MissingDescriptionExperiment) assert "experiment requires a description" in str(exc.value)
def test_valid_experiment(): assert ensure_experiment_is_valid(experiments.Experiment) is None
def test_experiment_may_not_have_a_hypothesis(): assert ensure_experiment_is_valid( experiments.MissingHypothesisExperiment) is None
def run( ctx: click.Context, source: str, journal_path: str = "./journal.json", dry: Optional[str] = None, no_validation: bool = False, no_exit: bool = False, no_verify_tls: bool = False, rollback_strategy: str = "default", var: Dict[str, Any] = None, var_file: List[str] = None, hypothesis_strategy: str = "default", hypothesis_frequency: float = 1.0, fail_fast: bool = False, ) -> Journal: """Run the experiment loaded from SOURCE, either a local file or a HTTP resource. SOURCE can be formatted as JSON or YAML.""" settings = load_settings(ctx.obj["settings_path"]) or {} has_deviated = False has_failed = False experiment_vars = merge_vars(var, var_file) load_global_controls(settings) try: experiment = load_experiment(source, settings, verify_tls=not no_verify_tls) except InvalidSource as x: logger.error(str(x)) logger.debug(x) ctx.exit(1) notify(settings, RunFlowEvent.RunStarted, experiment) if not no_validation: try: ensure_experiment_is_valid(experiment) except ChaosException as x: logger.error(str(x)) logger.debug(x) ctx.exit(1) experiment["dry"] = Dry.from_string(dry) settings.setdefault("runtime", {}).setdefault("rollbacks", {}).setdefault( "strategy", rollback_strategy ) hypothesis_strategy = check_hypothesis_strategy_spelling(hypothesis_strategy) schedule = Schedule( continuous_hypothesis_frequency=hypothesis_frequency, fail_fast=fail_fast ) journal = run_experiment( experiment, settings=settings, strategy=hypothesis_strategy, schedule=schedule, experiment_vars=experiment_vars, ) has_deviated = journal.get("deviated", False) has_failed = journal["status"] != "completed" if "dry" in journal["experiment"]: journal["experiment"]["dry"] = dry with open(journal_path, "w") as r: json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder) if journal["status"] == "completed": notify(settings, RunFlowEvent.RunCompleted, journal) elif has_failed: notify(settings, RunFlowEvent.RunFailed, journal) if has_deviated: notify(settings, RunFlowEvent.RunDeviated, journal) if (has_failed or has_deviated) and not no_exit: ctx.exit(1) return journal