def ensure_hypothesis_tolerance_is_valid(tolerance: Tolerance):
"""
Validate the tolerance of the hypothesis probe and raises
:exc:`InvalidActivity` if it isn't valid.
"""
if not isinstance(tolerance, (bool, int, list, str, dict)):
raise InvalidActivity(
"hypothesis probe tolerance must either be an integer, "
"a string, a boolean or a pair of values for boundaries. "
"It can also be a dictionary which is a probe activity "
"definition that takes an argument called `value` with "
"the value of the probe itself to be validated")
if isinstance(tolerance, dict):
tolerance_type = tolerance.get("type")
if tolerance_type == "probe":
ensure_activity_is_valid(tolerance)
elif tolerance_type == "regex":
check_regex_pattern(tolerance)
elif tolerance_type == "jsonpath":
check_json_path(tolerance)
elif tolerance_type == "range":
check_range(tolerance)
else:
raise InvalidActivity(
"hypothesis probe tolerance type '{}' is unsupported".format(
tolerance_type))
def ensure_hypothesis_is_valid(experiment: Experiment):
"""
Validates that the steady state hypothesis entry has the expected schema
or raises :exc:`InvalidExperiment` or :exc:`InvalidProbe`.
"""
hypo = experiment.get("steady-state-hypothesis")
if hypo is None:
return
if not hypo.get("title"):
raise InvalidExperiment("hypothesis requires a title")
probes = hypo.get("probes")
if probes:
for probe in probes:
ensure_activity_is_valid(probe)
if "tolerance" not in probe:
raise InvalidActivity(
"hypothesis probe must have a tolerance entry")
if not isinstance(probe["tolerance"], (
bool, int, list, str, dict)):
raise InvalidActivity(
"hypothesis probe tolerance must either be an integer, "
"a string, a boolean or a pair of values for boundaries. "
"It can also be a dictionary which is a probe activity "
"definition that takes an argument called `value` with "
"the value of the probe itself to be validated")
if isinstance(probe, dict):
ensure_activity_is_valid(probe)
def ensure_hypothesis_is_valid(experiment: Experiment):
"""
Validates that the steady state hypothesis entry has the expected schema
or raises :exc:`InvalidExperiment` or :exc:`InvalidActivity`.
"""
hypo = experiment.get("steady-state-hypothesis")
if hypo is None:
return
if not hypo.get("title"):
raise InvalidExperiment("hypothesis requires a title")
probes = hypo.get("probes")
if probes:
for probe in probes:
ensure_activity_is_valid(probe)
if "tolerance" not in probe:
raise InvalidActivity("hypothesis probe must have a tolerance entry")
ensure_hypothesis_tolerance_is_valid(probe["tolerance"])
def ensure_experiment_is_valid(experiment: Experiment):
"""
A chaos experiment consists of a method made of activities to carry
sequentially.
There are two kinds of activities:
* probe: detecting the state of a resource in your system or external to it
There are two kinds of probes: `steady` and `close`
* action: an operation to apply against your system
Usually, an experiment is made of a set of `steady` probes that ensure the
system is sound to carry further the experiment. Then, an action before
another set of of ̀close` probes to sense the state of the system
post-action.
This function raises :exc:`InvalidExperiment`, :exc:`InvalidProbe` or
:exc:`InvalidAction` depending on where it fails.
"""
logger.info("Validating the experiment's syntax")
if not experiment:
raise InvalidExperiment("an empty experiment is not an experiment")
if not experiment.get("title"):
raise InvalidExperiment("experiment requires a title")
if not experiment.get("description"):
raise InvalidExperiment("experiment requires a description")
tags = experiment.get("tags")
if tags:
if list(filter(lambda t: t == '' or not isinstance(t, str), tags)):
raise InvalidExperiment(
"experiment tags must be a non-empty string")
validate_extensions(experiment)
config = load_configuration(experiment.get("configuration", {}))
load_secrets(experiment.get("secrets", {}), config)
ensure_hypothesis_is_valid(experiment)
method = experiment.get("method")
if not method:
raise InvalidExperiment("an experiment requires a method with "
"at least one activity")
for activity in method:
ensure_activity_is_valid(activity)
# let's see if a ref is indeed found in the experiment
ref = activity.get("ref")
if ref and not lookup_activity(ref):
raise InvalidActivity("referenced activity '{r}' could not be "
"found in the experiment".format(r=ref))
rollbacks = experiment.get("rollbacks", [])
for activity in rollbacks:
ensure_activity_is_valid(activity)
warn_about_deprecated_features(experiment)
validate_controls(experiment)
logger.info("Experiment looks valid")
def test_http_probe_must_have_a_url():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.MissingHTTPUrlProbe)
assert "a HTTP activity must have a URL" in str(exc.value)
def test_process_probe_path_must_exist():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.ProcessPathDoesNotExistProbe)
assert "path 'somewhere/not/here' cannot be found, in activity" in str(
exc.value)
def test_process_probe_have_a_path():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.MissingProcessPathProbe)
assert "a process activity must have a path" in str(exc.value)
def test_empty_probe_is_invalid():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.EmptyProbe)
assert "empty activity is no activity" in str(exc.value)
def test_python_probe_func_must_have_enough_args():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.MissingFuncArgProbe)
assert "required argument 'path' is missing" in str(exc.value)
def test_python_probe_must_be_importable():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.NotImportableModuleProbe)
assert "could not find Python module 'fake.module'" in str(exc.value)
def test_python_probe_must_have_a_function_name():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.MissingFunctionProbe)
assert "a Python activity must have a function name" in str(exc.value)
def test_python_probe_must_have_a_module_path():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.MissingModuleProbe)
assert "a Python activity must have a module path" in str(exc.value)
def test_probe_provider_must_have_a_known_type():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.UnknownProviderTypeProbe)
assert "unknown provider type 'pizza'" in str(exc.value)
def test_probe_must_have_a_known_type():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.UnknownTypeProbe)
assert "'whatever' is not a supported activity type" in str(exc.value)
def test_probe_must_have_a_type():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.MissingTypeProbe)
assert "an activity must have a type" in str(exc.value)
def test_pauses_must_be_numbers_or_substitution_pattern():
try:
ensure_activity_is_valid({
"name": "can-use-numbers",
"type": "probe",
"provider": {
"type": "python",
"module": "os.path",
"func": "exists",
"arguments": {
"path": os.getcwd()
},
},
"pauses": {
"before": 1,
"after": 0.7
},
})
except InvalidActivity:
pytest.fail("pauses should support numbers")
try:
ensure_activity_is_valid({
"name": "can-use-numbers",
"type": "probe",
"provider": {
"type": "python",
"module": "os.path",
"func": "exists",
"arguments": {
"path": os.getcwd()
},
},
"pauses": {
"before": "${pause_before}",
"after": "${pause_after}",
},
})
except InvalidActivity:
pytest.fail("pauses should support substitution patterns")
with pytest.raises(InvalidActivity):
ensure_activity_is_valid({
"name": "can-use-numbers",
"type": "probe",
"provider": {
"type": "python",
"module": "os.path",
"func": "exists",
"arguments": {
"path": os.getcwd()
},
},
"pauses": {
"before": "hello",
"after": 0.9,
},
})
with pytest.raises(InvalidActivity):
ensure_activity_is_valid({
"name": "can-use-numbers",
"type": "probe",
"provider": {
"type": "python",
"module": "os.path",
"func": "exists",
"arguments": {
"path": os.getcwd()
},
},
"pauses": {
"before": 0.8,
"after": "world",
},
})
def test_empty_action_is_invalid():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(actions.EmptyAction)
assert "empty activity is no activity" in str(exc.value)
def test_python_probe_func_cannot_have_too_many_args():
with pytest.raises(InvalidActivity) as exc:
ensure_activity_is_valid(probes.TooManyFuncArgsProbe)
assert ("argument 'should_not_be_here' is not part of the "
"function signature" in str(exc.value))
