def test_load_configuration_should_raise_exception(): os.environ.clear() with pytest.raises(InvalidExperiment) as x: load_configuration( { "token1": "value1", "token2": {"type": "env", "key": "KUBE_TOKEN"}, "token3": {"type": "env", "key": "UNDEFINED", "default": ""}, } ) assert str(x.value) == ( "Configuration makes reference to an environment key that does not exist:" " KUBE_TOKEN" )
def test_that_environment_variables_are_typed_correctly(): config = load_configuration({ "token1": { "type": "env", "key": "TEST_ENV_VAR_NO_TYPE" }, "token2": { "type": "env", "key": "TEST_ENV_VAR_STRING", "env_var_type": "str", }, "token3": { "type": "env", "key": "TEST_ENV_VAR_INT", "env_var_type": "int" }, "token4": { "type": "env", "key": "TEST_ENV_VAR_FLOAT", "env_var_type": "float", }, "token5": { "type": "env", "key": "TEST_ENV_VAR_BYTES", "env_var_type": "bytes", }, }) assert config["token1"] == "should_be_a_string" assert config["token2"] == "should_also_be_a_string" assert config["token3"] == int(1000) assert config["token4"] == 30.54321 assert config["token5"] == b"these_are_bytes"
def configure(self, experiment: Experiment, settings: Settings, experiment_vars: Dict[str, Any]) -> None: config_vars, secret_vars = experiment_vars or (None, None) self.settings = settings if settings is not None else \ get_loaded_settings() self.config = load_configuration(experiment.get("configuration", {}), config_vars) self.secrets = load_secrets(experiment.get("secrets", {}), self.config, secret_vars)
def test_load_nested_object_configuration(): os.environ.clear() config = load_configuration( {"nested": {"onea": "fdsfdsf", "lol": {"haha": [1, 2, 3]}}} ) assert isinstance(config["nested"], dict) assert config["nested"]["onea"] == "fdsfdsf" assert config["nested"]["lol"] == {"haha": [1, 2, 3]}
def test_should_load_configuration(): os.environ["KUBE_TOKEN"] = "value2" config = load_configuration({ "token1": "value1", "token2": { "type": "env", "key": "KUBE_TOKEN" } }) assert config["token1"] == "value1" assert config["token2"] == "value2"
def test_use_nested_object_as_substitution(): config = load_configuration( {"nested": { "onea": "fdsfdsf", "lol": { "haha": [1, 2, 3] } }}) result = substitute("${nested}", configuration=config, secrets=None) assert isinstance(result, dict) assert result == {"onea": "fdsfdsf", "lol": {"haha": [1, 2, 3]}}
def test_env_var_can_be_used_with_loading_dynamic_config(fixtures_dir: str): env_file = os.path.join(fixtures_dir, "env_vars_issue252.json") cfg_vars, _ = merge_vars(None, [env_file]) cfg = load_configuration( { "some_config_1": "hello", "some_config_2": "there" }, extra_vars=cfg_vars) dcfg = load_dynamic_configuration(cfg) assert dcfg["some_config_1"] == os.getcwd() assert dcfg["some_config_2"] is True
def test_should_load_configuration(): os.environ["KUBE_TOKEN"] = "value2" config = load_configuration( { "token1": "value1", "token2": {"type": "env", "key": "KUBE_TOKEN"}, "token3": {"type": "env", "key": "UNDEFINED", "default": "value3"}, } ) assert config["token1"] == "value1" assert config["token2"] == "value2" assert config["token3"] == "value3"
def test_should_load_configuration_with_empty_string_as_input_while_default_is_define(): os.environ.clear() os.environ["KUBE_TOKEN"] = "" config = load_configuration( { "token1": "value1", "token2": {"type": "env", "key": "KUBE_TOKEN", "default": "value2"}, "token3": {"type": "env", "key": "UNDEFINED", "default": "value3"}, } ) assert config["token1"] == "value1" assert config["token2"] == "" assert config["token3"] == "value3"
def test_should_override_load_configuration_with_var(): os.environ["KUBE_TOKEN"] = "value2" config = load_configuration( { "token1": "value1", "token2": {"type": "env", "key": "KUBE_TOKEN"}, "token3": {"type": "env", "key": "UNDEFINED", "default": "value3"}, }, {"token1": "othervalue1", "token2": "othervalue2"}, ) assert config["token1"] == "othervalue1" assert config["token2"] == "othervalue2" assert config["token3"] == "value3"
def test_can_override_experiment_inline_config_keys(): os.environ["KUBE_TOKEN"] = "value2" config = load_configuration( { "token1": "value1", "token2": {"type": "env", "key": "KUBE_TOKEN"}, "token3": {"type": "env", "key": "UNDEFINED", "default": "value3"}, }, extra_vars={"token1": "extravalue"}, ) assert config["token1"] == "extravalue" assert config["token2"] == "value2" assert config["token3"] == "value3"
def test_should_load_configuration_with_empty_string_as_input(): config = load_configuration({ "token1": "value1", "token2": { "type": "env", "key": "KUBE_TOKEN" }, "token3": { "type": "env", "key": "UNDEFINED", "default": "value3" }, }) assert config["token1"] == "value1" assert config["token2"] == "" assert config["token3"] == "value3"
def test_default_value_is_overriden_in_inline_config_keys(): config = load_configuration( { "token1": "value1", "token2": { "type": "env", "key": "KUBE_TOKEN" }, "token3": { "type": "env", "key": "UNDEFINED", "default": "value3" }, }, extra_vars={"token3": "extravalue"}, ) assert config["token1"] == "value1" assert config["token2"] == "value2" assert config["token3"] == "extravalue"
def test_always_return_to_string_when_pattern_is_not_alone(): config = load_configuration({"value": 8}) result = substitute("hello ${value}", configuration=config, secrets=None) assert isinstance(result, str) assert result == "hello 8"
def test_use_integer_as_substitution(): config = load_configuration({"value": 8}) result = substitute("${value}", configuration=config, secrets=None) assert isinstance(result, int) assert result == 8
def configure(self, experiment: Experiment, settings: Settings) -> None: self.settings = settings if settings is not None else \ get_loaded_settings() self.config = load_configuration(experiment.get("configuration", {})) self.secrets = load_secrets(experiment.get("secrets", {}), self.config)
def ensure_experiment_is_valid(experiment: Experiment): """ A chaos experiment consists of a method made of activities to carry sequentially. There are two kinds of activities: * probe: detecting the state of a resource in your system or external to it There are two kinds of probes: `steady` and `close` * action: an operation to apply against your system Usually, an experiment is made of a set of `steady` probes that ensure the system is sound to carry further the experiment. Then, an action before another set of of ̀close` probes to sense the state of the system post-action. This function raises :exc:`InvalidExperiment`, :exc:`InvalidProbe` or :exc:`InvalidAction` depending on where it fails. """ logger.info("Validating the experiment's syntax") if not experiment: raise InvalidExperiment("an empty experiment is not an experiment") if not experiment.get("title"): raise InvalidExperiment("experiment requires a title") if not experiment.get("description"): raise InvalidExperiment("experiment requires a description") tags = experiment.get("tags") if tags: if list(filter(lambda t: t == '' or not isinstance(t, str), tags)): raise InvalidExperiment( "experiment tags must be a non-empty string") validate_extensions(experiment) config = load_configuration(experiment.get("configuration", {})) load_secrets(experiment.get("secrets", {}), config) ensure_hypothesis_is_valid(experiment) method = experiment.get("method") if not method: raise InvalidExperiment("an experiment requires a method with " "at least one activity") for activity in method: ensure_activity_is_valid(activity) # let's see if a ref is indeed found in the experiment ref = activity.get("ref") if ref and not lookup_activity(ref): raise InvalidActivity("referenced activity '{r}' could not be " "found in the experiment".format(r=ref)) rollbacks = experiment.get("rollbacks", []) for activity in rollbacks: ensure_activity_is_valid(activity) warn_about_deprecated_features(experiment) validate_controls(experiment) logger.info("Experiment looks valid")
def run_experiment(experiment: Experiment, settings: Settings = None) -> Journal: """ Run the given `experiment` method step by step, in the following sequence: steady probe, action, close probe. Activities can be executed in background when they have the `"background"` property set to `true`. In that case, the activity is run in a thread. By the end of runs, those threads block until they are all complete. If the experiment has the `"dry"` property set to `False`, the experiment runs without actually executing the activities. NOTE: Tricky to make a decision whether we should rollback when exiting abnormally (Ctrl-C, SIGTERM...). Afterall, there is a chance we actually cannot afford to rollback properly. Better bailing to a conservative approach. This means we swallow :exc:`KeyboardInterrupt` and :exc:`SystemExit` and do not bubble it back up to the caller. We when were interrupted, we set the `interrupted` flag of the result accordingly to notify the caller this was indeed not terminated properly. """ logger.info("Running experiment: {t}".format(t=experiment["title"])) dry = experiment.get("dry", False) if dry: logger.warning("Dry mode enabled") started_at = time.time() settings = settings if settings is not None else get_loaded_settings() config = load_configuration(experiment.get("configuration", {})) secrets = load_secrets(experiment.get("secrets", {}), config) initialize_global_controls(experiment, config, secrets, settings) initialize_controls(experiment, config, secrets) activity_pool, rollback_pool = get_background_pools(experiment) control = Control() journal = initialize_run_journal(experiment) try: try: control.begin("experiment", experiment, experiment, config, secrets) # this may fail the entire experiment right there if any of the # probes fail or fall out of their tolerance zone try: state = run_steady_state_hypothesis(experiment, config, secrets, dry=dry) journal["steady_states"]["before"] = state if state is not None and not state["steady_state_met"]: p = state["probes"][-1] raise ActivityFailed( "Steady state probe '{p}' is not in the given " "tolerance so failing this experiment".format( p=p["activity"]["name"])) except ActivityFailed as a: journal["steady_states"]["before"] = state journal["status"] = "failed" logger.fatal(str(a)) else: try: journal["run"] = apply_activities(experiment, config, secrets, activity_pool, dry) except Exception: journal["status"] = "aborted" logger.fatal( "Experiment ran into an un expected fatal error, " "aborting now.", exc_info=True) else: try: state = run_steady_state_hypothesis(experiment, config, secrets, dry=dry) journal["steady_states"]["after"] = state if state is not None and not state["steady_state_met"]: journal["deviated"] = True p = state["probes"][-1] raise ActivityFailed( "Steady state probe '{p}' is not in the given " "tolerance so failing this experiment".format( p=p["activity"]["name"])) except ActivityFailed as a: journal["status"] = "failed" logger.fatal(str(a)) except InterruptExecution as i: journal["status"] = "interrupted" logger.fatal(str(i)) except (KeyboardInterrupt, SystemExit): journal["status"] = "interrupted" logger.warn("Received an exit signal, " "leaving without applying rollbacks.") else: journal["status"] = journal["status"] or "completed" journal["rollbacks"] = apply_rollbacks(experiment, config, secrets, rollback_pool, dry) journal["end"] = datetime.utcnow().isoformat() journal["duration"] = time.time() - started_at has_deviated = journal["deviated"] status = "deviated" if has_deviated else journal["status"] logger.info("Experiment ended with status: {s}".format(s=status)) if has_deviated: logger.info( "The steady-state has deviated, a weakness may have been " "discovered") control.with_state(journal) try: control.end("experiment", experiment, experiment, config, secrets) except ChaosException: logger.debug("Failed to close controls", exc_info=True) finally: cleanup_controls(experiment) cleanup_global_controls() return journal