def test_prefect_no_retries(evaluator_config,
function_ensemble_builder_factory, tmpdir):
"""Evaluator tries and fails once. Asserts if job and step fails"""
cloudpickle.register_pickle_by_value(sys.modules[__name__])
pickle_func = cloudpickle.dumps(function_that_fails_once)
cloudpickle.unregister_pickle_by_value(sys.modules[__name__])
ensemble = (function_ensemble_builder_factory(pickle_func).set_retry_delay(
1).set_max_retries(0).build())
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with tmpdir.as_cwd():
# Get events
event_list = []
with evaluator.run() as mon:
for event in mon.track():
event_list.append(event)
wait_until_done(mon, event)
# Find if job and step failed
step_failed = False
job_failed = False
for real in ensemble.snapshot.reals.values():
for step in real.steps.values():
for job in step.jobs.values():
if job.status == state.JOB_STATE_FAILURE:
job_failed = True
assert job.error == "This is an expected ERROR"
if step.status == state.STEP_STATE_FAILURE:
step_failed = True
assert ensemble.status == state.ENSEMBLE_STATE_FAILED
assert job_failed, f"Events: {event_list}"
assert step_failed, f"Events: {event_list}"
def test_prefect_retries(evaluator_config, function_ensemble_builder_factory,
tmpdir, ensemble_size):
"""Evaluator fails once through pickled-fail-function. Asserts fail and retries"""
cloudpickle.register_pickle_by_value(sys.modules[__name__])
pickle_func = cloudpickle.dumps(function_that_fails_once)
cloudpickle.unregister_pickle_by_value(sys.modules[__name__])
ensemble = function_ensemble_builder_factory(pickle_func).set_retry_delay(
2).build()
evaluator = EnsembleEvaluator(ensemble, evaluator_config, 0, ee_id="1")
with tmpdir.as_cwd():
error_event_reals: Set[str] = set()
with evaluator.run() as mon:
# close_events_in_ensemble_run(monitor=mon) # more strict as above
for event in mon.track():
# Capture the job error messages
if event.data is not None and "This is an expected ERROR" in str(
event.data):
error_event_reals.update(event.data["reals"].keys())
wait_until_done(mon, event)
assert evaluator._ensemble.status == state.ENSEMBLE_STATE_STOPPED
successful_realizations = evaluator._ensemble.get_successful_realizations(
)
assert successful_realizations == ensemble_size
# Check we get only one job error message per realization
assert len(error_event_reals) == ensemble_size
assert "0" in error_event_reals
assert "1" in error_event_reals
def pickle(fn):
for module in includes:
cloudpickle.register_pickle_by_value(module)
code = cloudpickle.dumps(fn)
for module in includes:
cloudpickle.unregister_pickle_by_value(module)
return code