def startSimulation(self):
self._run_model.reset()
self._snapshot_model.reset()
self._tab_widget.clear()
def run():
asyncio.set_event_loop(asyncio.new_event_loop())
self._run_model.startSimulations(self._simulations_argments)
simulation_thread = Thread(name="ert_gui_simulation_thread")
simulation_thread.setDaemon(True)
simulation_thread.run = run
simulation_thread.start()
self._ticker.start(1000)
tracker = create_tracker(
self._run_model,
num_realizations=self._simulations_argments["active_realizations"].
count(),
ee_config=self._simulations_argments.get("ee_config", None),
)
worker = TrackerWorker(tracker)
worker_thread = QThread()
worker.done.connect(worker_thread.quit)
worker.consumed_event.connect(self._on_tracker_event)
worker.moveToThread(worker_thread)
self.simulation_done.connect(worker.stop)
self._worker = worker
self._worker_thread = worker_thread
worker_thread.started.connect(worker.consume_and_emit)
self._worker_thread.start()
def run_cli(args):
logging.basicConfig(level=logging.INFO, format="%(message)s")
res_config = ResConfig(args.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=args.verbose)
notifier = ErtCliNotifier(ert, args.config)
ERT.adapt(notifier)
if args.mode == WORKFLOW_MODE:
execute_workflow(args.name)
return
model, argument = create_model(args)
# Test run does not have a current_case
if "current_case" in args and args.current_case:
ERT.enkf_facade.select_or_create_new_case(args.current_case)
if (
args.mode
in [ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE]
and args.target_case == ERT.enkf_facade.get_current_case_name()
):
msg = (
"ERROR: Target file system and source file system can not be the same. "
"They were both: {}.".format(args.target_case)
)
_clear_and_exit(msg)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig()
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.startSimulations,
args=(argument,),
)
thread.start()
tracker = create_tracker(model, detailed_interval=0, ee_config=ee_config)
out = open(os.devnull, "w") if args.disable_monitoring else sys.stdout
monitor = Monitor(out=out, color_always=args.color_always)
try:
monitor.monitor(tracker)
except (SystemExit, KeyboardInterrupt):
print("\nKilling simulations...")
tracker.request_termination()
if args.disable_monitoring:
out.close()
thread.join()
if model.hasRunFailed():
_clear_and_exit(1) # the monitor has already reported the error message
def test_tracking(
experiment_folder,
cmd_line_arguments,
num_successful,
num_iters,
tmpdir,
source_root,
):
shutil.copytree(
os.path.join(source_root, "test-data", "local",
f"{experiment_folder}"),
os.path.join(str(tmpdir), f"{experiment_folder}"),
)
config_lines = [
"INSTALL_JOB poly_eval2 POLY_EVAL\n"
"SIMULATION_JOB poly_eval2\n"
]
with tmpdir.as_cwd():
with open(f"{experiment_folder}/poly.ert", "a") as fh:
fh.writelines(config_lines)
parser = ArgumentParser(prog="test_main")
parsed = ert_parser(
parser,
cmd_line_arguments,
)
FeatureToggling.update_from_args(parsed)
res_config = ResConfig(parsed.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose)
notifier = ErtCliNotifier(ert, parsed.config)
ERT.adapt(notifier)
model, argument = create_model(parsed)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig()
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.startSimulations,
args=(argument, ),
)
thread.start()
tracker = create_tracker(model,
general_interval=1,
detailed_interval=2,
ee_config=ee_config)
snapshots = {}
for event in tracker.track():
if isinstance(event, FullSnapshotEvent):
snapshots[event.iteration] = event.snapshot
if isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
snapshots[event.iteration].merge(
event.partial_snapshot.data())
if isinstance(event, EndEvent):
break
assert tracker._progress() == 1.0
assert len(snapshots) == num_iters
for iter_, snapshot in snapshots.items():
successful_reals = list(
filter(
lambda item: item[1].status == REALIZATION_STATE_FINISHED,
snapshot.get_reals().items(),
))
assert len(successful_reals) == num_successful
for real_id, real in successful_reals:
assert (real.status == REALIZATION_STATE_FINISHED
), f"iter:{iter_} real:{real_id} was not finished"
poly = real.steps["0"].jobs["0"]
poly2 = real.steps["0"].jobs["1"]
assert poly.name == "poly_eval"
assert (poly.status == JOB_STATE_FINISHED
), f"real {real_id}/{poly['name']} was not finished"
assert poly2.name == "poly_eval2"
assert (poly2.status == JOB_STATE_FINISHED
), f"real {real_id}/{poly['name']} was not finished"
def test_tracking(
experiment_folder,
cmd_line_arguments,
num_successful,
num_iters,
assert_present_in_snapshot,
tmpdir,
source_root,
):
shutil.copytree(
os.path.join(source_root, "test-data", "local",
f"{experiment_folder}"),
os.path.join(str(tmpdir), f"{experiment_folder}"),
)
config_lines = [
"INSTALL_JOB poly_eval2 POLY_EVAL\n"
"SIMULATION_JOB poly_eval2\n"
]
with tmpdir.as_cwd():
with open(f"{experiment_folder}/poly.ert", "a") as fh:
fh.writelines(config_lines)
parser = ArgumentParser(prog="test_main")
parsed = ert_parser(
parser,
cmd_line_arguments,
)
FeatureToggling.update_from_args(parsed)
res_config = ResConfig(parsed.config)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose)
notifier = ErtCliNotifier(ert, parsed.config)
ERT.adapt(notifier)
model, argument = create_model(parsed)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig()
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.start_simulations_thread,
args=(argument, ),
)
thread.start()
tracker = create_tracker(model,
general_interval=1,
detailed_interval=2,
ee_config=ee_config)
snapshots = {}
for event in tracker.track():
if isinstance(event, FullSnapshotEvent):
snapshots[event.iteration] = event.snapshot
if isinstance(event, SnapshotUpdateEvent):
if event.partial_snapshot is not None:
snapshots[event.iteration].merge(
event.partial_snapshot.data())
if isinstance(event, EndEvent):
pass
assert tracker._progress() == 1.0
assert len(snapshots) == num_iters
for iter_, snapshot in snapshots.items():
successful_reals = list(
filter(
lambda item: item[1].status == state.
REALIZATION_STATE_FINISHED,
snapshot.get_reals().items(),
))
assert len(successful_reals) == num_successful
for (
iter_expression,
snapshot_expression,
expected,
) in assert_present_in_snapshot:
for i, snapshot in snapshots.items():
if re.match(iter_expression, str(i)):
check_expression(
snapshot.to_dict(),
snapshot_expression,
expected,
f"Snapshot {i} did not match:\n",
)
thread.join()
def run_cli(args):
res_config = ResConfig(args.config)
# Create logger inside function to make sure all handlers have been added to the root-logger.
logger = logging.getLogger(__name__)
logger.info(
"Logging forward model jobs",
extra={
"workflow_jobs":
str(res_config.model_config.getForwardModel().joblist())
},
)
os.chdir(res_config.config_path)
ert = EnKFMain(res_config, strict=True, verbose=args.verbose)
notifier = ErtCliNotifier(ert, args.config)
with ERT.adapt(notifier):
if args.mode == WORKFLOW_MODE:
execute_workflow(args.name)
return
model, argument = create_model(args)
# Test run does not have a current_case
if "current_case" in args and args.current_case:
ERT.enkf_facade.select_or_create_new_case(args.current_case)
if (args.mode in [
ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE,
ES_MDA_MODE
] and args.target_case == ERT.enkf_facade.get_current_case_name()):
msg = (
"ERROR: Target file system and source file system can not be the same. "
"They were both: {}.".format(args.target_case))
raise ErtCliError(msg)
ee_config = None
if FeatureToggling.is_enabled("ensemble-evaluator"):
ee_config = EvaluatorServerConfig(
custom_port_range=args.port_range)
argument.update({"ee_config": ee_config})
thread = threading.Thread(
name="ert_cli_simulation_thread",
target=model.start_simulations_thread,
args=(argument, ),
)
thread.start()
tracker = create_tracker(model,
detailed_interval=0,
ee_config=ee_config)
out = open(os.devnull, "w") if args.disable_monitoring else sys.stderr
monitor = Monitor(out=out, color_always=args.color_always)
try:
monitor.monitor(tracker)
except (SystemExit, KeyboardInterrupt):
print("\nKilling simulations...")
tracker.request_termination()
if args.disable_monitoring:
out.close()
thread.join()
if model.hasRunFailed():
raise ErtCliError(model.getFailMessage())