def test_https_with_verification():
with requests_mock.mock() as m:
m.get("https://example.com/experiment.yaml",
exc=requests.exceptions.SSLError)
with pytest.raises(requests.exceptions.SSLError):
load_experiment("https://example.com/experiment.yaml",
verify_tls=True)
def test_yaml_safe_load_from_http():
with requests_mock.mock() as m:
m.get('http://example.com/experiment.yaml',
status_code=200,
headers={"Content-Type": "application/x-yaml"},
text=experiments.UnsafeYamlExperiment)
with pytest.raises(InvalidSource):
load_experiment('http://example.com/experiment.yaml')
def test_http_loads_fails_when_known_type():
with requests_mock.mock() as m:
m.get('http://example.com/experiment.yaml',
status_code=200,
headers={"Content-Type": "text/css"},
text="body {}")
with pytest.raises(InvalidExperiment):
load_experiment('http://example.com/experiment.yaml')
def test_load_from_http_without_auth(generic_experiment: str):
with requests_mock.mock() as m:
m.get('http://example.com/experiment.json',
status_code=200,
headers={"Content-Type": "application/json"},
json=json.dumps(generic_experiment))
try:
load_experiment('http://example.com/experiment.json')
except InvalidSource as x:
pytest.fail(str(x))
def test_can_load_yaml_from_plain_text_http(generic_experiment: str):
with requests_mock.mock() as m:
m.get('http://example.com/experiment.json',
status_code=200,
headers={"Content-Type": "text/plain; charset=utf-8"},
text=json.dumps(generic_experiment))
try:
load_experiment('http://example.com/experiment.json')
except InvalidExperiment as x:
pytest.fail(str(x))
def test_can_load_json_from_plain_text_http():
with requests_mock.mock() as m:
m.get('http://example.com/experiment.yaml',
status_code=200,
headers={"Content-Type": "text/plain; charset=utf-8"},
text=experiments.YamlExperiment)
try:
load_experiment('http://example.com/experiment.yaml')
except InvalidExperiment as x:
pytest.fail(str(x))
def test_https_no_verification():
with requests_mock.mock() as m:
m.get(
"https://example.com/experiment.yaml",
status_code=200,
headers={"Content-Type": "text/css"},
text="body {}",
)
with pytest.raises(InvalidExperiment):
load_experiment("https://example.com/experiment.yaml", verify_tls=False)
def test_load_from_http_with_auth(settings: Settings, generic_experiment: str):
with requests_mock.mock() as m:
settings['auths'] = {'example.com': {'type': 'bearer', 'value': 'XYZ'}}
m.get('http://example.com/experiment.json',
status_code=200,
request_headers={
"Authorization": "bearer XYZ",
"Accept": "application/json, application/x-yaml"
},
headers={"Content-Type": "application/json"},
json=json.dumps(generic_experiment))
try:
load_experiment('http://example.com/experiment.json', settings)
except InvalidSource as x:
pytest.fail(str(x))
def run_chaos_engine(self, file, env_params: dict, report: str,
report_endpoint: str) -> bool:
"""
Runs chaos engine programmatically instead of using chaos binary
:param file:
:param env_params:
:param report:
:param report_endpoint:
:return:
"""
settings = ({}, os.environ.get("settings_path")
)[os.environ.get("settings_path") is not None]
has_deviated = False
has_failed = False
load_global_controls(settings)
jornal_file_suffix = file
try:
try:
with open(file, "r"):
logger.info("File exists in local")
except FileNotFoundError:
logger.info(
"File is not available in the current directory, looking inside site packages"
)
location = site.getsitepackages().__getitem__(0)
file_found = False
for root, dirs, files in os.walk(location):
if file in files:
file_found = True
file = os.path.join(root, file)
break
if not file_found:
logger.error("File " + file +
" not found in site packages too, quitting")
raise FileNotFoundError("Chaos file is not found")
experiment = load_experiment(click.format_filename(file), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
logger.info("chaos json file found, proceeding with test")
journal = run_experiment(experiment, settings=settings)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
json_file_name = "journal" + "-" + jornal_file_suffix
with open(json_file_name, "w") as r:
json.dump(journal,
r,
indent=2,
ensure_ascii=False,
default=encoder)
r.close()
if report == 'true':
self.create_report(os.environ, journal, report_endpoint)
if has_failed or has_deviated:
logger.error("Test Failed")
return has_failed and has_deviated
else:
logger.info("Test Passed")
return True
def validate(
ctx: click.Context, source: str, no_verify_tls: bool = False
) -> Experiment:
"""Validate the experiment at SOURCE."""
settings = load_settings(ctx.obj["settings_path"])
try:
experiment = load_experiment(source, settings, verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
try:
notify(settings, ValidateFlowEvent.ValidateStarted, experiment)
ensure_experiment_is_valid(experiment)
notify(settings, ValidateFlowEvent.ValidateCompleted, experiment)
logger.info("experiment syntax and semantic look valid")
except ChaosException as x:
notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x)
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
return experiment
def test_controls_on_loaded_experiment():
settings = {
"controls": {
"dummy": {
"provider": {
"type":
"python",
"module":
"fixtures.controls.dummy_retitle_experiment_on_loading"
}
}
}
}
load_global_controls(settings)
initialize_global_controls({}, {}, {}, settings)
with tempfile.NamedTemporaryFile(suffix=".json") as f:
try:
f.write(
json.dumps(experiments.ExperimentNoControls).encode('utf-8'))
f.seek(0)
experiment = load_experiment(f.name)
assert experiment["title"] == "BOOM I changed it"
finally:
cleanup_global_controls()
def run(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
no_exit: bool = False,
no_verify_tls: bool = False,
rollback_strategy: str = "default") -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource. SOURCE can be formatted as JSON or YAML."""
settings = load_settings(ctx.obj["settings_path"]) or {}
has_deviated = False
has_failed = False
load_global_controls(settings)
try:
experiment = load_experiment(source,
settings,
verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = dry
settings.setdefault("runtime",
{}).setdefault("rollbacks",
{}).setdefault("strategy",
rollback_strategy)
journal = run_experiment(experiment, settings=settings)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def test_load_from_http_with_auth(settings: Settings, generic_experiment: str):
with requests_mock.mock() as m:
settings["auths"] = {"example.com": {"type": "bearer", "value": "XYZ"}}
m.get(
"http://example.com/experiment.json",
status_code=200,
request_headers={
"Authorization": "bearer XYZ",
"Accept": "application/json, application/x-yaml",
},
headers={"Content-Type": "application/json"},
json=json.dumps(generic_experiment),
)
try:
load_experiment("http://example.com/experiment.json", settings)
except InvalidSource as x:
pytest.fail(str(x))
def test_controls_on_loading_experiment():
settings = {
"controls": {
"dummy": {
"provider": {
"type": "python",
"module": "fixtures.controls.dummy_fail_loading_experiment"
}
}
}
}
load_global_controls(settings)
initialize_global_controls({}, {}, {}, settings)
with tempfile.NamedTemporaryFile(suffix=".json") as f:
try:
with pytest.raises(InterruptExecution):
load_experiment(f.name)
finally:
cleanup_global_controls()
def run(ctx: click.Context, source: str, journal_path: str = "./journal.json",
dry: bool = False, no_validation: bool = False,
no_exit: bool = False) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"]) or {}
initialize_global_controls(settings)
has_deviated = False
has_failed = False
try:
try:
experiment = load_experiment(
click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
with io.open(journal_path, "w") as r:
json.dump(
journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
finally:
cleanup_global_controls()
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def run(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
fail_fast: bool = True) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"])
try:
experiment = load_experiment(click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
else:
notify(settings, RunFlowEvent.RunFailed, journal)
if journal.get("deviated", False):
notify(settings, RunFlowEvent.RunDeviated, journal)
# when set (default) we exit this command immediatly if the execution
# failed, was aborted or interrupted
# when unset, plugins can continue the processing. In that case, they
# are responsible to set the process exit code accordingly.
if fail_fast:
sys.exit(1)
return journal
def validate(ctx: click.Context, path: str) -> Experiment:
"""Validate the experiment at PATH."""
settings = load_settings(ctx.obj["settings_path"])
experiment = load_experiment(click.format_filename(path))
try:
notify(settings, ValidateFlowEvent.ValidateStarted, experiment)
ensure_experiment_is_valid(experiment)
notify(settings, ValidateFlowEvent.ValidateCompleted, experiment)
logger.info("experiment syntax and semantic look valid")
except ChaosException as x:
notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x)
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
return experiment
def verify(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
no_exit: bool = False,
no_verify_tls: bool = False):
"""Run the verification loaded from SOURCE, either a local file or a
HTTP resource. SOURCE can be formatted as JSON or YAML."""
settings = load_settings(ctx.obj["settings_path"]) or {}
load_global_controls(settings)
try:
if not switch_team_during_verification_run(source, settings):
ctx.exit(1)
verification = load_experiment(source,
settings,
verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
if not no_validation:
try:
ensure_verification_is_valid(verification)
except ChaosException as v:
logger.error(str(v))
logger.debug(v)
ctx.exit(1)
verification["dry"] = dry
journal = run_verification(verification,
settings=settings,
strategy=Strategy.CONTINOUS)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
return journal
def run(
ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: Optional[str] = None,
no_validation: bool = False,
no_exit: bool = False,
no_verify_tls: bool = False,
rollback_strategy: str = "default",
var: Dict[str, Any] = None,
var_file: List[str] = None,
hypothesis_strategy: str = "default",
hypothesis_frequency: float = 1.0,
fail_fast: bool = False,
) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource. SOURCE can be formatted as JSON or YAML."""
settings = load_settings(ctx.obj["settings_path"]) or {}
has_deviated = False
has_failed = False
experiment_vars = merge_vars(var, var_file)
load_global_controls(settings)
try:
experiment = load_experiment(source, settings, verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = Dry.from_string(dry)
settings.setdefault("runtime", {}).setdefault("rollbacks", {}).setdefault(
"strategy", rollback_strategy
)
hypothesis_strategy = check_hypothesis_strategy_spelling(hypothesis_strategy)
schedule = Schedule(
continuous_hypothesis_frequency=hypothesis_frequency, fail_fast=fail_fast
)
journal = run_experiment(
experiment,
settings=settings,
strategy=hypothesis_strategy,
schedule=schedule,
experiment_vars=experiment_vars,
)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
if "dry" in journal["experiment"]:
journal["experiment"]["dry"] = dry
with open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def run(self, path):
experiment = load_experiment(path)
journal = run_experiment(experiment)
return journal
def test_load_from_http_with_missing_auth(generic_experiment: str):
with requests_mock.mock() as m:
m.get("http://example.com/experiment.json", status_code=401)
with pytest.raises(InvalidSource):
load_experiment("http://example.com/experiment.json")
def test_load_invalid_filepath(generic_experiment: str):
with pytest.raises(InvalidSource) as x:
load_experiment("/tmp/xyuzye.txt")
assert 'Path "/tmp/xyuzye.txt" does not exist.' in str(x.value)
def run(self, path):
experiment = load_experiment(path)
return experiment
def test_load_from_file(generic_experiment: str):
try:
load_experiment(generic_experiment)
except InvalidSource as x:
pytest.fail(str(x))
