def run_experiment(
experiment: Experiment,
settings: Settings = None,
experiment_vars: Dict[str, Any] = None,
strategy: Strategy = Strategy.DEFAULT,
schedule: Schedule = None,
event_handlers: List[RunEventHandler] = None,
) -> Journal:
"""
Run the given `experiment` method step by step, in the following sequence:
steady probe, action, close probe.
Activities can be executed in background when they have the
`"background"` property set to `true`. In that case, the activity is run in
a thread. By the end of runs, those threads block until they are all
complete.
If the experiment has the `"dry"` property set to `False`, the experiment
runs without actually executing the activities.
NOTE: Tricky to make a decision whether we should rollback when exiting
abnormally (Ctrl-C, SIGTERM...). Afterall, there is a chance we actually
cannot afford to rollback properly. Better bailing to a conservative
approach. This means we swallow :exc:`KeyboardInterrupt` and
:exc:`SystemExit` and do not bubble it back up to the caller. We when were
interrupted, we set the `interrupted` flag of the result accordingly to
notify the caller this was indeed not terminated properly.
"""
with Runner(strategy, schedule) as runner:
if event_handlers:
for h in event_handlers:
runner.register_event_handler(h)
return runner.run(experiment,
settings,
experiment_vars=experiment_vars)
def test_play_rollbacks_on_graceful_exit_with_python_action():
x = deepcopy(experiments.ExperimentGracefulExitLongPythonCall)
with Runner(Strategy.DEFAULT) as runner:
journal = runner.run(x, settings={
"runtime": {"rollbacks": {"strategy": "always"}}})
assert journal["status"] == "interrupted"
assert len(journal["rollbacks"]) == 1
def test_wait_for_background_activity_on_graceful_exit():
server = threading.Thread(target=run_http_server_in_background)
server.start()
x = deepcopy(experiments.ExperimentGracefulExitLongHTTPCall)
with Runner(Strategy.DEFAULT) as runner:
journal = runner.run(x)
assert journal["status"] == "interrupted"
assert 3.0 < journal["run"][0]["duration"] < 3.2
server.join()
def test_wait_for_background_activity_to_finish_on_graceful_exit():
def _exit_soon():
time.sleep(1.5)
exit_gracefully()
t = threading.Thread(target=_exit_soon)
x = deepcopy(experiments.SimpleExperimentWithBackgroundActivity)
with Runner(Strategy.DEFAULT) as runner:
t.start()
journal = runner.run(x)
assert journal["status"] == "interrupted"
assert journal["run"][0]["status"] == "succeeded"
def test_do_not_play_rollbacks_on_graceful_exit_with_python_action():
server = threading.Thread(target=run_http_server_in_background)
server.start()
x = deepcopy(experiments.ExperimentUngracefulExitLongHTTPCall)
with Runner(Strategy.DEFAULT) as runner:
journal = runner.run(x, settings={
"runtime": {"rollbacks": {"strategy": "always"}}})
assert journal["status"] == "interrupted"
assert len(journal["rollbacks"]) == 0
server.join()
def test_do_not_wait_for_background_activity_on_ungraceful_exit():
def _exit_soon():
time.sleep(1.5)
exit_ungracefully()
t = threading.Thread(target=_exit_soon)
x = deepcopy(experiments.SimpleExperimentWithBackgroundActivity)
with Runner(Strategy.DEFAULT) as runner:
t.start()
journal = runner.run(x)
assert journal["status"] == "interrupted"
assert journal["run"][0]["status"] == "failed"
assert "ExperimentExitedException" in journal["run"][0]["exception"][-1]
def run_verification(experiment: Experiment,
settings: Settings = None,
experiment_vars: Dict[str, Any] = None,
strategy: Strategy = CONTINUOUS,
schedule: Schedule = None) -> Journal:
with Runner(strategy, schedule) as runner:
runner.register_event_handler(RunEventHandler())
runner.register_event_handler(
VerificationRunEventHandler(experiment, settings))
journal = initialize_run_journal(experiment)
# TODO: remove with https://github.com/chaosiq/chaosiq-cloud/issues/70
journal["measurements"] = []
return runner.run(experiment,
settings,
journal=journal,
experiment_vars=experiment_vars)