def test_load_configuration_should_raise_exception():
os.environ.clear()
with pytest.raises(InvalidExperiment) as x:
load_configuration(
{
"token1": "value1",
"token2": {"type": "env", "key": "KUBE_TOKEN"},
"token3": {"type": "env", "key": "UNDEFINED", "default": ""},
}
)
assert str(x.value) == (
"Configuration makes reference to an environment key that does not exist:"
" KUBE_TOKEN"
)
def test_that_environment_variables_are_typed_correctly():
config = load_configuration({
"token1": {
"type": "env",
"key": "TEST_ENV_VAR_NO_TYPE"
},
"token2": {
"type": "env",
"key": "TEST_ENV_VAR_STRING",
"env_var_type": "str",
},
"token3": {
"type": "env",
"key": "TEST_ENV_VAR_INT",
"env_var_type": "int"
},
"token4": {
"type": "env",
"key": "TEST_ENV_VAR_FLOAT",
"env_var_type": "float",
},
"token5": {
"type": "env",
"key": "TEST_ENV_VAR_BYTES",
"env_var_type": "bytes",
},
})
assert config["token1"] == "should_be_a_string"
assert config["token2"] == "should_also_be_a_string"
assert config["token3"] == int(1000)
assert config["token4"] == 30.54321
assert config["token5"] == b"these_are_bytes"
def configure(self, experiment: Experiment, settings: Settings,
experiment_vars: Dict[str, Any]) -> None:
config_vars, secret_vars = experiment_vars or (None, None)
self.settings = settings if settings is not None else \
get_loaded_settings()
self.config = load_configuration(experiment.get("configuration", {}),
config_vars)
self.secrets = load_secrets(experiment.get("secrets", {}), self.config,
secret_vars)
def test_load_nested_object_configuration():
os.environ.clear()
config = load_configuration(
{"nested": {"onea": "fdsfdsf", "lol": {"haha": [1, 2, 3]}}}
)
assert isinstance(config["nested"], dict)
assert config["nested"]["onea"] == "fdsfdsf"
assert config["nested"]["lol"] == {"haha": [1, 2, 3]}
def test_should_load_configuration():
os.environ["KUBE_TOKEN"] = "value2"
config = load_configuration({
"token1": "value1",
"token2": {
"type": "env",
"key": "KUBE_TOKEN"
}
})
assert config["token1"] == "value1"
assert config["token2"] == "value2"
def test_use_nested_object_as_substitution():
config = load_configuration(
{"nested": {
"onea": "fdsfdsf",
"lol": {
"haha": [1, 2, 3]
}
}})
result = substitute("${nested}", configuration=config, secrets=None)
assert isinstance(result, dict)
assert result == {"onea": "fdsfdsf", "lol": {"haha": [1, 2, 3]}}
def test_env_var_can_be_used_with_loading_dynamic_config(fixtures_dir: str):
env_file = os.path.join(fixtures_dir, "env_vars_issue252.json")
cfg_vars, _ = merge_vars(None, [env_file])
cfg = load_configuration(
{
"some_config_1": "hello",
"some_config_2": "there"
},
extra_vars=cfg_vars)
dcfg = load_dynamic_configuration(cfg)
assert dcfg["some_config_1"] == os.getcwd()
assert dcfg["some_config_2"] is True
def test_should_load_configuration():
os.environ["KUBE_TOKEN"] = "value2"
config = load_configuration(
{
"token1": "value1",
"token2": {"type": "env", "key": "KUBE_TOKEN"},
"token3": {"type": "env", "key": "UNDEFINED", "default": "value3"},
}
)
assert config["token1"] == "value1"
assert config["token2"] == "value2"
assert config["token3"] == "value3"
def test_should_load_configuration_with_empty_string_as_input_while_default_is_define():
os.environ.clear()
os.environ["KUBE_TOKEN"] = ""
config = load_configuration(
{
"token1": "value1",
"token2": {"type": "env", "key": "KUBE_TOKEN", "default": "value2"},
"token3": {"type": "env", "key": "UNDEFINED", "default": "value3"},
}
)
assert config["token1"] == "value1"
assert config["token2"] == ""
assert config["token3"] == "value3"
def test_should_override_load_configuration_with_var():
os.environ["KUBE_TOKEN"] = "value2"
config = load_configuration(
{
"token1": "value1",
"token2": {"type": "env", "key": "KUBE_TOKEN"},
"token3": {"type": "env", "key": "UNDEFINED", "default": "value3"},
},
{"token1": "othervalue1", "token2": "othervalue2"},
)
assert config["token1"] == "othervalue1"
assert config["token2"] == "othervalue2"
assert config["token3"] == "value3"
def test_can_override_experiment_inline_config_keys():
os.environ["KUBE_TOKEN"] = "value2"
config = load_configuration(
{
"token1": "value1",
"token2": {"type": "env", "key": "KUBE_TOKEN"},
"token3": {"type": "env", "key": "UNDEFINED", "default": "value3"},
},
extra_vars={"token1": "extravalue"},
)
assert config["token1"] == "extravalue"
assert config["token2"] == "value2"
assert config["token3"] == "value3"
def test_should_load_configuration_with_empty_string_as_input():
config = load_configuration({
"token1": "value1",
"token2": {
"type": "env",
"key": "KUBE_TOKEN"
},
"token3": {
"type": "env",
"key": "UNDEFINED",
"default": "value3"
},
})
assert config["token1"] == "value1"
assert config["token2"] == ""
assert config["token3"] == "value3"
def test_default_value_is_overriden_in_inline_config_keys():
config = load_configuration(
{
"token1": "value1",
"token2": {
"type": "env",
"key": "KUBE_TOKEN"
},
"token3": {
"type": "env",
"key": "UNDEFINED",
"default": "value3"
},
},
extra_vars={"token3": "extravalue"},
)
assert config["token1"] == "value1"
assert config["token2"] == "value2"
assert config["token3"] == "extravalue"
def test_always_return_to_string_when_pattern_is_not_alone():
config = load_configuration({"value": 8})
result = substitute("hello ${value}", configuration=config, secrets=None)
assert isinstance(result, str)
assert result == "hello 8"
def test_use_integer_as_substitution():
config = load_configuration({"value": 8})
result = substitute("${value}", configuration=config, secrets=None)
assert isinstance(result, int)
assert result == 8
def configure(self, experiment: Experiment, settings: Settings) -> None:
self.settings = settings if settings is not None else \
get_loaded_settings()
self.config = load_configuration(experiment.get("configuration", {}))
self.secrets = load_secrets(experiment.get("secrets", {}), self.config)
def ensure_experiment_is_valid(experiment: Experiment):
"""
A chaos experiment consists of a method made of activities to carry
sequentially.
There are two kinds of activities:
* probe: detecting the state of a resource in your system or external to it
There are two kinds of probes: `steady` and `close`
* action: an operation to apply against your system
Usually, an experiment is made of a set of `steady` probes that ensure the
system is sound to carry further the experiment. Then, an action before
another set of of ̀close` probes to sense the state of the system
post-action.
This function raises :exc:`InvalidExperiment`, :exc:`InvalidProbe` or
:exc:`InvalidAction` depending on where it fails.
"""
logger.info("Validating the experiment's syntax")
if not experiment:
raise InvalidExperiment("an empty experiment is not an experiment")
if not experiment.get("title"):
raise InvalidExperiment("experiment requires a title")
if not experiment.get("description"):
raise InvalidExperiment("experiment requires a description")
tags = experiment.get("tags")
if tags:
if list(filter(lambda t: t == '' or not isinstance(t, str), tags)):
raise InvalidExperiment(
"experiment tags must be a non-empty string")
validate_extensions(experiment)
config = load_configuration(experiment.get("configuration", {}))
load_secrets(experiment.get("secrets", {}), config)
ensure_hypothesis_is_valid(experiment)
method = experiment.get("method")
if not method:
raise InvalidExperiment("an experiment requires a method with "
"at least one activity")
for activity in method:
ensure_activity_is_valid(activity)
# let's see if a ref is indeed found in the experiment
ref = activity.get("ref")
if ref and not lookup_activity(ref):
raise InvalidActivity("referenced activity '{r}' could not be "
"found in the experiment".format(r=ref))
rollbacks = experiment.get("rollbacks", [])
for activity in rollbacks:
ensure_activity_is_valid(activity)
warn_about_deprecated_features(experiment)
validate_controls(experiment)
logger.info("Experiment looks valid")
def run_experiment(experiment: Experiment,
settings: Settings = None) -> Journal:
"""
Run the given `experiment` method step by step, in the following sequence:
steady probe, action, close probe.
Activities can be executed in background when they have the
`"background"` property set to `true`. In that case, the activity is run in
a thread. By the end of runs, those threads block until they are all
complete.
If the experiment has the `"dry"` property set to `False`, the experiment
runs without actually executing the activities.
NOTE: Tricky to make a decision whether we should rollback when exiting
abnormally (Ctrl-C, SIGTERM...). Afterall, there is a chance we actually
cannot afford to rollback properly. Better bailing to a conservative
approach. This means we swallow :exc:`KeyboardInterrupt` and
:exc:`SystemExit` and do not bubble it back up to the caller. We when were
interrupted, we set the `interrupted` flag of the result accordingly to
notify the caller this was indeed not terminated properly.
"""
logger.info("Running experiment: {t}".format(t=experiment["title"]))
dry = experiment.get("dry", False)
if dry:
logger.warning("Dry mode enabled")
started_at = time.time()
settings = settings if settings is not None else get_loaded_settings()
config = load_configuration(experiment.get("configuration", {}))
secrets = load_secrets(experiment.get("secrets", {}), config)
initialize_global_controls(experiment, config, secrets, settings)
initialize_controls(experiment, config, secrets)
activity_pool, rollback_pool = get_background_pools(experiment)
control = Control()
journal = initialize_run_journal(experiment)
try:
try:
control.begin("experiment", experiment, experiment, config,
secrets)
# this may fail the entire experiment right there if any of the
# probes fail or fall out of their tolerance zone
try:
state = run_steady_state_hypothesis(experiment,
config,
secrets,
dry=dry)
journal["steady_states"]["before"] = state
if state is not None and not state["steady_state_met"]:
p = state["probes"][-1]
raise ActivityFailed(
"Steady state probe '{p}' is not in the given "
"tolerance so failing this experiment".format(
p=p["activity"]["name"]))
except ActivityFailed as a:
journal["steady_states"]["before"] = state
journal["status"] = "failed"
logger.fatal(str(a))
else:
try:
journal["run"] = apply_activities(experiment, config,
secrets, activity_pool,
dry)
except Exception:
journal["status"] = "aborted"
logger.fatal(
"Experiment ran into an un expected fatal error, "
"aborting now.",
exc_info=True)
else:
try:
state = run_steady_state_hypothesis(experiment,
config,
secrets,
dry=dry)
journal["steady_states"]["after"] = state
if state is not None and not state["steady_state_met"]:
journal["deviated"] = True
p = state["probes"][-1]
raise ActivityFailed(
"Steady state probe '{p}' is not in the given "
"tolerance so failing this experiment".format(
p=p["activity"]["name"]))
except ActivityFailed as a:
journal["status"] = "failed"
logger.fatal(str(a))
except InterruptExecution as i:
journal["status"] = "interrupted"
logger.fatal(str(i))
except (KeyboardInterrupt, SystemExit):
journal["status"] = "interrupted"
logger.warn("Received an exit signal, "
"leaving without applying rollbacks.")
else:
journal["status"] = journal["status"] or "completed"
journal["rollbacks"] = apply_rollbacks(experiment, config, secrets,
rollback_pool, dry)
journal["end"] = datetime.utcnow().isoformat()
journal["duration"] = time.time() - started_at
has_deviated = journal["deviated"]
status = "deviated" if has_deviated else journal["status"]
logger.info("Experiment ended with status: {s}".format(s=status))
if has_deviated:
logger.info(
"The steady-state has deviated, a weakness may have been "
"discovered")
control.with_state(journal)
try:
control.end("experiment", experiment, experiment, config, secrets)
except ChaosException:
logger.debug("Failed to close controls", exc_info=True)
finally:
cleanup_controls(experiment)
cleanup_global_controls()
return journal
