def test_notify_via_plugin(logger):
payload = {"msg": "hello", "ts": str(time.time())}
event_payload = {"event": str(RunFlowEvent.RunStarted), "payload": payload}
notify(
{"notifications": [{
"type": "plugin",
"module": "fixtures.notifier"
}]}, RunFlowEvent.RunStarted)
logger.debug.assert_called_with("boom")
def test_notify_via_plugin(logger):
notify(
{"notifications": [{
"type": "plugin",
"module": "fixtures.notifier"
}]},
RunFlowEvent.RunStarted,
)
logger.debug.assert_called_with("boom")
def test_notify_to_http_endpoint_requires_a_url(logger):
payload = {"msg": "hello", "ts": str(time.time())}
event_payload = {"event": str(RunFlowEvent.RunStarted), "payload": payload}
with requests_mock.mock() as m:
m.post("http://example.com", status_code=200, json=event_payload)
notify({"notifications": [{"type": "http"}]}, RunFlowEvent.RunStarted)
m = "missing url in notification channel"
logger.debug.assert_called_with(m)
def test_notify_via_plugin_with_non_default_func_name(logger):
notify(
{
"notifications": [{
"type": "plugin",
"module": "fixtures.notifier",
"func": "notify_other",
}]
},
RunFlowEvent.RunStarted,
)
logger.debug.assert_called_with("doh")
def test_notify_via_plugin_gracefully_handles_failure_to_import_plugin(
logger: MagicMock,
) -> None:
notify(
{"notifications": [{"type": "plugin", "module": "fixtures.notifier___"}]},
RunFlowEvent.RunStarted,
)
logger.debug.assert_called_with(
"could not find Python plugin '{mod}' for notification".format(
mod="fixtures.notifier___"
)
)
def test_notify_to_http_endpoint_may_fail(logger):
url = "http://example.com"
with requests_mock.mock() as m:
m.post("http://example.com", status_code=404, text="boom")
notify({"notifications": [{
"type": "http",
"url": url
}]}, RunFlowEvent.RunStarted)
m = "Notification sent to '{u}' failed with '{t}'".format(u=url,
t="boom")
logger.debug.assert_called_with(m)
def test_notify_to_http_endpoint():
payload = {"msg": "hello", "ts": str(time.time())}
event_payload = {"event": str(RunFlowEvent.RunStarted), "payload": payload}
with requests_mock.mock() as m:
m.post('http://example.com', status_code=200, json=event_payload)
notify(
{"notifications": [{
"type": "http",
"url": "http://example.com"
}]}, RunFlowEvent.RunStarted)
assert m.called
def run(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
no_exit: bool = False,
no_verify_tls: bool = False,
rollback_strategy: str = "default") -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource. SOURCE can be formatted as JSON or YAML."""
settings = load_settings(ctx.obj["settings_path"]) or {}
has_deviated = False
has_failed = False
load_global_controls(settings)
try:
experiment = load_experiment(source,
settings,
verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = dry
settings.setdefault("runtime",
{}).setdefault("rollbacks",
{}).setdefault("strategy",
rollback_strategy)
journal = run_experiment(experiment, settings=settings)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def test_notify_only_notifies_on_events_specified(
mock_notify_with_http: MagicMock,
) -> None:
channel = {
"type": "http",
"url": "http://example.com",
"events": ["discover-started"],
}
notify(
settings={"notifications": [channel]},
event=RunFlowEvent.RunFailed,
payload=None,
)
mock_notify_with_http.assert_not_called()
def test_notify_via_plugin_failed_to_import_plugin(logger):
payload = {"msg": "hello", "ts": str(time.time())}
event_payload = {"event": str(RunFlowEvent.RunStarted), "payload": payload}
notify(
{
"notifications": [{
"type": "plugin",
"module": "fixtures.notifier___"
}]
}, RunFlowEvent.RunStarted)
logger.debug.assert_called_with(
"could not find Python plugin '{mod}' for notification".format(
mod="fixtures.notifier___"))
def test_notify_to_http_endpoint_can_timeout(logger):
url = "http://example.com"
exc_mock = requests.exceptions.ConnectTimeout()
with requests_mock.mock() as m:
m.post(url, exc=exc_mock)
notify({"notifications": [{
"type": "http",
"url": url
}]}, RunFlowEvent.RunStarted)
m = "failed calling notification endpoint"
logger.debug.assert_called_with(m, exc_info=exc_mock)
def test_notify_to_http_endpoint_may_fail(logger):
url = "http://example.com"
payload = {"msg": "hello", "ts": str(time.time())}
event_payload = {"event": str(RunFlowEvent.RunStarted), "payload": payload}
with requests_mock.mock() as m:
m.post('http://example.com', status_code=404, text="boom")
notify({"notifications": [{
"type": "http",
"url": url
}]}, RunFlowEvent.RunStarted)
m = "Notification sent to '{u}' failed with '{t}'".format(u=url,
t="boom")
logger.debug.assert_called_with(m)
def test_notify_via_plugin_failed_to_import_func(logger):
notify(
{
"notifications": [{
"type": "plugin",
"module": "fixtures.notifier",
"func": "blah"
}]
},
RunFlowEvent.RunStarted,
)
logger.debug.assert_called_with(
"could not find function '{f}' in plugin '{mod}' "
"for notification".format(mod="fixtures.notifier", f="blah"))
def test_notify_via_plugin_correctly_invokes_notify_func_with_non_default_func_name(
logger: MagicMock,
) -> None:
notify(
{
"notifications": [
{
"type": "plugin",
"module": "fixtures.notifier",
"func": "notify_other",
}
]
},
RunFlowEvent.RunStarted,
)
logger.debug.assert_called_with("doh")
def run(ctx: click.Context, source: str, journal_path: str = "./journal.json",
dry: bool = False, no_validation: bool = False,
no_exit: bool = False) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"]) or {}
initialize_global_controls(settings)
has_deviated = False
has_failed = False
try:
try:
experiment = load_experiment(
click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
with io.open(journal_path, "w") as r:
json.dump(
journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
finally:
cleanup_global_controls()
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def test_notify_via_plugin_gracefully_handles_failure_in_invoked_func(
logger: MagicMock,
) -> None:
notify(
{
"notifications": [
{
"type": "plugin",
"module": "fixtures.notifier",
"func": "notify_broken",
}
]
},
RunFlowEvent.RunStarted,
)
logger.debug.assert_called_with(
"failed calling notification plugin", exc_info=callee.InstanceOf(Exception)
)
def test_notify_correctly_assigns_phase_from_event_class(
mock_notify_with_http: MagicMock,
) -> None:
channel = {"type": "http", "url": "http://example.com"}
for phase, event_class in [
("discovery", DiscoverFlowEvent.DiscoverStarted),
("init", InitFlowEvent.InitStarted),
("run", RunFlowEvent.RunStarted),
("validate", ValidateFlowEvent.ValidateStarted),
]:
mock_notify_with_http.reset_mock()
notify(settings={"notifications": [channel]}, event=event_class, payload=None)
mock_notify_with_http.assert_called_once_with(
channel,
{
"name": event_class.value,
"payload": None,
"phase": phase,
"ts": datetime.utcnow().replace(tzinfo=timezone.utc).timestamp(),
},
)
def test_notify_appends_error_to_event_payload_if_provided(
mock_notify_with_http: MagicMock,
) -> None:
channel = {"type": "http", "url": "http://example.com"}
exception = ChaosException("Something went wrong")
notify(
settings={"notifications": [channel]},
event=DiscoverFlowEvent.DiscoverStarted,
payload=None,
error=exception,
)
mock_notify_with_http.assert_called_once_with(
channel,
{
"name": DiscoverFlowEvent.DiscoverStarted.value,
"payload": None,
"phase": "discovery",
"ts": datetime.utcnow().replace(tzinfo=timezone.utc).timestamp(),
"error": exception,
},
)
def run(path: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False) -> Journal:
"""Run the experiment given at PATH."""
experiment = load_experiment(click.format_filename(path))
settings = load_settings()
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
else:
notify(settings, RunFlowEvent.RunFailed, journal)
return journal
def validate(
ctx: click.Context, source: str, no_verify_tls: bool = False
) -> Experiment:
"""Validate the experiment at SOURCE."""
settings = load_settings(ctx.obj["settings_path"])
try:
experiment = load_experiment(source, settings, verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
try:
notify(settings, ValidateFlowEvent.ValidateStarted, experiment)
ensure_experiment_is_valid(experiment)
notify(settings, ValidateFlowEvent.ValidateCompleted, experiment)
logger.info("experiment syntax and semantic look valid")
except ChaosException as x:
notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x)
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
return experiment
def discover(
ctx: click.Context,
package: str,
discovery_path: str = "./discovery.json",
no_system_info: bool = False,
no_install: bool = False,
) -> Discovery:
"""Discover capabilities and experiments."""
settings = load_settings(ctx.obj["settings_path"])
try:
notify(settings, DiscoverFlowEvent.DiscoverStarted, package)
discovery = disco(
package_name=package,
discover_system=not no_system_info,
download_and_install=not no_install,
)
except DiscoveryFailed as err:
notify(settings, DiscoverFlowEvent.DiscoverFailed, package, err)
logger.debug(f"Failed to discover {package}", exc_info=err)
logger.fatal(str(err))
return
with open(discovery_path, "w") as d:
d.write(json.dumps(discovery, indent=2, default=encoder))
logger.info(f"Discovery outcome saved in {discovery_path}")
notify(settings, DiscoverFlowEvent.DiscoverCompleted, discovery)
return discovery
def run(ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: bool = False,
no_validation: bool = False,
fail_fast: bool = True) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource."""
settings = load_settings(ctx.obj["settings_path"])
try:
experiment = load_experiment(click.format_filename(source), settings)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
sys.exit(1)
experiment["dry"] = dry
journal = run_experiment(experiment)
with io.open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
else:
notify(settings, RunFlowEvent.RunFailed, journal)
if journal.get("deviated", False):
notify(settings, RunFlowEvent.RunDeviated, journal)
# when set (default) we exit this command immediatly if the execution
# failed, was aborted or interrupted
# when unset, plugins can continue the processing. In that case, they
# are responsible to set the process exit code accordingly.
if fail_fast:
sys.exit(1)
return journal
def test_notify_calls_notify_via_plugin_when_type_is_plugin(
mock_notify_via_plugin: MagicMock,
) -> None:
now = datetime.utcnow().replace(tzinfo=timezone.utc).timestamp()
payload = {"test-key": "test-value", "test-dict": {"test-dict-key": "test"}}
channel = {"type": "plugin", "module": "fixtures.notifier"}
notify(
settings={"notifications": [channel]},
event=RunFlowEvent.RunStarted,
payload=payload,
)
mock_notify_via_plugin.assert_called_once_with(
channel,
{
"name": RunFlowEvent.RunStarted.value,
"payload": payload,
"phase": "run",
"ts": now,
},
)
def test_notify_calls_notify_with_http_when_type_is_http(
mock_notify_with_http: MagicMock,
) -> None:
now = datetime.utcnow().replace(tzinfo=timezone.utc).timestamp()
payload = {"test-key": "test-value", "test-dict": {"test-dict-key": "test"}}
channel = {"type": "http", "url": "http://example.com"}
notify(
settings={"notifications": [channel]},
event=RunFlowEvent.RunStarted,
payload=payload,
)
mock_notify_with_http.assert_called_once_with(
channel,
{
"name": RunFlowEvent.RunStarted.value,
"payload": payload,
"phase": "run",
"ts": now,
},
)
def validate(ctx: click.Context, path: str) -> Experiment:
"""Validate the experiment at PATH."""
settings = load_settings(ctx.obj["settings_path"])
experiment = load_experiment(click.format_filename(path))
try:
notify(settings, ValidateFlowEvent.ValidateStarted, experiment)
ensure_experiment_is_valid(experiment)
notify(settings, ValidateFlowEvent.ValidateCompleted, experiment)
logger.info("experiment syntax and semantic look valid")
except ChaosException as x:
notify(settings, ValidateFlowEvent.ValidateFailed, experiment, x)
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
return experiment
def discover(package: str, discovery_path: str="./discovery.json",
no_system_info: bool=False,
no_install: bool=False) -> Discovery:
"""Discover capabilities and experiments."""
settings = load_settings()
try:
notify(settings, DiscoverFlowEvent.DiscoverStarted, package)
discovery = disco(
package_name=package, discover_system=not no_system_info,
download_and_install=not no_install)
except DiscoveryFailed as err:
notify(settings, DiscoverFlowEvent.DiscoverFailed, package, err)
logger.debug("Failed to discover {}".format(package), exc_info=err)
logger.fatal(str(err))
return
with open(discovery_path, "w") as d:
d.write(json.dumps(discovery, indent=2))
logger.info("Discovery outcome saved in {p}".format(
p=discovery_path))
notify(settings, DiscoverFlowEvent.DiscoverCompleted, discovery)
return discovery
def run(
ctx: click.Context,
source: str,
journal_path: str = "./journal.json",
dry: Optional[str] = None,
no_validation: bool = False,
no_exit: bool = False,
no_verify_tls: bool = False,
rollback_strategy: str = "default",
var: Dict[str, Any] = None,
var_file: List[str] = None,
hypothesis_strategy: str = "default",
hypothesis_frequency: float = 1.0,
fail_fast: bool = False,
) -> Journal:
"""Run the experiment loaded from SOURCE, either a local file or a
HTTP resource. SOURCE can be formatted as JSON or YAML."""
settings = load_settings(ctx.obj["settings_path"]) or {}
has_deviated = False
has_failed = False
experiment_vars = merge_vars(var, var_file)
load_global_controls(settings)
try:
experiment = load_experiment(source, settings, verify_tls=not no_verify_tls)
except InvalidSource as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
notify(settings, RunFlowEvent.RunStarted, experiment)
if not no_validation:
try:
ensure_experiment_is_valid(experiment)
except ChaosException as x:
logger.error(str(x))
logger.debug(x)
ctx.exit(1)
experiment["dry"] = Dry.from_string(dry)
settings.setdefault("runtime", {}).setdefault("rollbacks", {}).setdefault(
"strategy", rollback_strategy
)
hypothesis_strategy = check_hypothesis_strategy_spelling(hypothesis_strategy)
schedule = Schedule(
continuous_hypothesis_frequency=hypothesis_frequency, fail_fast=fail_fast
)
journal = run_experiment(
experiment,
settings=settings,
strategy=hypothesis_strategy,
schedule=schedule,
experiment_vars=experiment_vars,
)
has_deviated = journal.get("deviated", False)
has_failed = journal["status"] != "completed"
if "dry" in journal["experiment"]:
journal["experiment"]["dry"] = dry
with open(journal_path, "w") as r:
json.dump(journal, r, indent=2, ensure_ascii=False, default=encoder)
if journal["status"] == "completed":
notify(settings, RunFlowEvent.RunCompleted, journal)
elif has_failed:
notify(settings, RunFlowEvent.RunFailed, journal)
if has_deviated:
notify(settings, RunFlowEvent.RunDeviated, journal)
if (has_failed or has_deviated) and not no_exit:
ctx.exit(1)
return journal
def test_no_settings_is_okay():
assert notify(None, DiscoverFlowEvent.DiscoverStarted) is None
def init(ctx: click.Context, discovery_path: str = "./discovery.json",
experiment_path: str = "./experiment.json") -> Experiment:
"""Initialize a new experiment from discovered capabilities."""
settings = load_settings(ctx.obj["settings_path"])
notify(settings, InitFlowEvent.InitStarted)
click.secho(
"You are about to create an experiment.\n"
"This wizard will walk you through each step so that you can build\n"
"the best experiment for your needs.\n"
"\n"
"An experiment is made up of three elements:\n"
"- a steady-state hypothesis [OPTIONAL]\n"
"- an experimental method\n"
"- a set of rollback activities [OPTIONAL]\n"
"\n"
"Only the method is required. Also your experiment will\n"
"not run unless you define at least one activity (probe or action)\n"
"within it",
fg="blue")
discovery = None
if discovery_path and os.path.exists(discovery_path):
with open(discovery_path) as d:
discovery = json.loads(d.read())
else:
click.echo("No discovery was found, let's create an empty experiment")
base_experiment = {
"version": "1.0.0",
"title": "",
"description": "N/A",
"tags": []
}
s = click.style
title = click.prompt(s("Experiment's title", fg='green'), type=str)
base_experiment["title"] = title
click.secho(
"\nA steady state hypothesis defines what 'normality' "
"looks like in your system\n"
"The steady state hypothesis is a collection of "
"conditions that are used,\n"
"at the beginning of an experiment, to decide if the "
"system is in a recognised\n"
"'normal' state. The steady state conditions are then "
"used again when your experiment\n"
" is complete to detect where your system may have "
"deviated in an interesting,\n"
"weakness-detecting way\n"
"\n"
"Initially you may not know what your steady state "
"hypothesis is\n"
"and so instead you might create an experiment "
"without one\n"
"This is why the stead state hypothesis is optional.", fg="blue")
m = s('Do you want to define a steady state hypothesis now?',
dim=True)
if click.confirm(m):
hypo = {}
title = click.prompt(s("Hypothesis's title", fg='green'), type=str)
hypo["title"] = title
hypo["probes"] = []
if discovery:
activities = []
for a in discovery["activities"]:
if a["type"] == "probe":
activities.append((a["name"], a))
click.secho(
"\nYou may now define probes that will determine\n"
"the steady-state of your system.",
fg="blue")
add_activities(activities, hypo["probes"], with_tolerance=True)
base_experiment["steady-state-hypothesis"] = hypo
if discovery:
base_experiment["method"] = []
click.secho(
"\nAn experiment's method contains actions "
"and probes. Actions\n"
"vary real-world events in your system to determine if your\n"
"steady-state hypothesis is maintained when those events occur.\n"
"\n"
"An experimental method can also contain probes to gather"
" additional\n"
"information about your system as your method is executed.",
fg="blue")
m = s('Do you want to define an experimental method?', dim=True)
if click.confirm(m):
activities = [(a["name"], a) for a in discovery["activities"]]
add_activities(activities, base_experiment["method"])
click.secho(
"\nAn experiment may optionally define a set of remedial"
" actions\nthat are used to rollback the system to a given"
" state.",
fg="blue")
m = s('Do you want to add some rollbacks now?', dim=True)
if click.confirm(m):
rollbacks = []
activities = []
for a in discovery["activities"]:
if a["type"] == "action":
activities.append((a["name"], a))
add_activities(activities, rollbacks)
base_experiment["rollbacks"] = rollbacks
if is_yaml(experiment_path):
output = yaml.dump(base_experiment,
indent=4,
default_flow_style=False,
sort_keys=False)
else:
output = json.dumps(base_experiment, indent=4, default=encoder)
with open(experiment_path, "w") as e:
e.write(output)
click.echo(
"\nExperiment created and saved in '{e}'".format(e=experiment_path))
notify(settings, InitFlowEvent.InitCompleted, base_experiment)
return base_experiment
def test_no_notifications_in_settings_is_okay():
assert notify({}, DiscoverFlowEvent.DiscoverStarted) is None
