def startSimulations(self, arg): with open(arg["config_file"], "r") as f: config = yaml.safe_load(f) ensemble = PrefectEnsemble(config) ee_config = arg["ee_config"] ee_id = str(uuid.uuid1()).split("-")[0] ee = EnsembleEvaluator(ensemble, ee_config, ee_id=ee_id) ee.run_and_get_successful_realizations() self._stop_time = time.time() self._queue_running = False ee.stop()
def test_exhaust_retries_in_run_and_get_successful_realizations( make_ee_config, num_realizations, num_failing): ee_config = make_ee_config(use_token=False, generate_cert=False, custom_host="localhost") ensemble = AutorunTestEnsemble(iter=1, reals=num_realizations, steps=1, jobs=2) for i in range(num_failing): ensemble.addFailJob(real=i, step=0, job=1) ee = EnsembleEvaluator(ensemble, ee_config, 0, ee_id="0") with patch.object( _Monitor, "track", side_effect=[get_connection_closed_exception()] * 2 + [ConnectionRefusedError("Connection error")] + [dummy_iterator("DUMMY TRACKING ITERATOR")] + [get_connection_closed_exception()] * 2 + [ConnectionRefusedError("Connection error")] * 3 + ["DUMMY TRACKING ITERATOR2" ], # This should not be reached, hence we assert call_count == 9 ) as mock: num_successful = ee.run_and_get_successful_realizations() assert mock.call_count == 9 assert num_successful == num_realizations - num_failing
def test_ens_eval_run_and_get_successful_realizations_connection_refused_no_recover( make_ee_config, num_realizations, num_failing): ee_config = make_ee_config(use_token=False, generate_cert=False) ensemble = AutorunTestEnsemble(iter=1, reals=num_realizations, steps=1, jobs=2) for i in range(num_failing): ensemble.addFailJob(real=i, step=0, job=1) ee = EnsembleEvaluator(ensemble, ee_config, 0, ee_id="0") with patch.object( _Monitor, "track", side_effect=ConnectionRefusedError("Connection error")) as mock: num_successful = ee.run_and_get_successful_realizations() assert mock.call_count == 3 assert num_successful == num_realizations - num_failing
def test_ensemble_evaluator_run_and_get_successful_realizations_connection_closed_recover( make_ee_config, num_realizations, num_failing): ee_config = make_ee_config(use_token=False, generate_cert=False) ensemble = AutorunTestEnsemble(iter=1, reals=num_realizations, steps=1, jobs=2) for i in range(num_failing): ensemble.addFailJob(real=i, step=0, job=1) ee = EnsembleEvaluator(ensemble, ee_config, 0, ee_id="0") with patch.object( _Monitor, "track", side_effect=[get_connection_closed_excpetion()] * 2 + [ConnectionRefusedError("Connection error")] + [dummy_iterator("DUMMY TRACKING ITERATOR")] + [get_connection_closed_excpetion()] * 2 + [ConnectionRefusedError("Connection error")] * 2 + ["DUMMY TRACKING ITERATOR2"], ) as mock: num_successfull = ee.run_and_get_successful_realizations() assert mock.call_count == 9 assert num_successfull == num_realizations - num_failing