def startSimulations(self, arg):
with open(arg["config_file"], "r") as f:
config = yaml.safe_load(f)
ensemble = PrefectEnsemble(config)
ee_config = arg["ee_config"]
ee_id = str(uuid.uuid1()).split("-")[0]
ee = EnsembleEvaluator(ensemble, ee_config, ee_id=ee_id)
ee.run_and_get_successful_realizations()
self._stop_time = time.time()
self._queue_running = False
ee.stop()
def test_exhaust_retries_in_run_and_get_successful_realizations(
make_ee_config, num_realizations, num_failing):
ee_config = make_ee_config(use_token=False,
generate_cert=False,
custom_host="localhost")
ensemble = AutorunTestEnsemble(iter=1,
reals=num_realizations,
steps=1,
jobs=2)
for i in range(num_failing):
ensemble.addFailJob(real=i, step=0, job=1)
ee = EnsembleEvaluator(ensemble, ee_config, 0, ee_id="0")
with patch.object(
_Monitor,
"track",
side_effect=[get_connection_closed_exception()] * 2 +
[ConnectionRefusedError("Connection error")] +
[dummy_iterator("DUMMY TRACKING ITERATOR")] +
[get_connection_closed_exception()] * 2 +
[ConnectionRefusedError("Connection error")] * 3 +
["DUMMY TRACKING ITERATOR2"
], # This should not be reached, hence we assert call_count == 9
) as mock:
num_successful = ee.run_and_get_successful_realizations()
assert mock.call_count == 9
assert num_successful == num_realizations - num_failing
def test_ens_eval_run_and_get_successful_realizations_connection_refused_no_recover(
make_ee_config, num_realizations, num_failing):
ee_config = make_ee_config(use_token=False, generate_cert=False)
ensemble = AutorunTestEnsemble(iter=1,
reals=num_realizations,
steps=1,
jobs=2)
for i in range(num_failing):
ensemble.addFailJob(real=i, step=0, job=1)
ee = EnsembleEvaluator(ensemble, ee_config, 0, ee_id="0")
with patch.object(
_Monitor,
"track",
side_effect=ConnectionRefusedError("Connection error")) as mock:
num_successful = ee.run_and_get_successful_realizations()
assert mock.call_count == 3
assert num_successful == num_realizations - num_failing
def test_ensemble_evaluator_run_and_get_successful_realizations_connection_closed_recover(
make_ee_config, num_realizations, num_failing):
ee_config = make_ee_config(use_token=False, generate_cert=False)
ensemble = AutorunTestEnsemble(iter=1,
reals=num_realizations,
steps=1,
jobs=2)
for i in range(num_failing):
ensemble.addFailJob(real=i, step=0, job=1)
ee = EnsembleEvaluator(ensemble, ee_config, 0, ee_id="0")
with patch.object(
_Monitor,
"track",
side_effect=[get_connection_closed_excpetion()] * 2 +
[ConnectionRefusedError("Connection error")] +
[dummy_iterator("DUMMY TRACKING ITERATOR")] +
[get_connection_closed_excpetion()] * 2 +
[ConnectionRefusedError("Connection error")] * 2 +
["DUMMY TRACKING ITERATOR2"],
) as mock:
num_successfull = ee.run_and_get_successful_realizations()
assert mock.call_count == 9
assert num_successfull == num_realizations - num_failing
