def test_run_export_pathfile(monkeypatch): with ErtPluginContext(): run_path_file = Path("output/run_path_file/.some_new_name") config_file = "snake_oil.ert" with open(config_file, encoding="utf-8", mode="a") as fout: fout.write(f"RUNPATH_FILE {run_path_file}\n") rc = ResConfig(user_config_file=config_file) ert = EnKFMain(rc) notifier = ErtCliNotifier(ert, config_file) with ERT.adapt(notifier): run_mock = MagicMock() run_mock.hasFailed.return_value = False export_mock = MagicMock() export_mock.hasFailed.return_value = False monkeypatch.setattr( ERT.enkf_facade, "get_workflow_job", MagicMock(side_effect=[export_mock, run_mock]), ) ex = Exporter() parameters = { "output_file": "export.csv", "time_index": "raw", "column_keys": "FOPR", } ex.run_export(parameters) expected_call = call( arguments=[ f"{run_path_file.absolute()}", "export.csv", "raw", "FOPR" ], ert=ERT.ert, verbose=True, ) assert export_mock.run.call_args == expected_call
def run_cli(args): logging.basicConfig(level=logging.INFO, format="%(message)s") res_config = ResConfig(args.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=args.verbose) notifier = ErtCliNotifier(ert, args.config) ERT.adapt(notifier) if args.mode == WORKFLOW_MODE: execute_workflow(args.name) return model, argument = create_model(args) # Test run does not have a current_case if "current_case" in args and args.current_case: ERT.enkf_facade.select_or_create_new_case(args.current_case) if ( args.mode in [ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE] and args.target_case == ERT.enkf_facade.get_current_case_name() ): msg = ( "ERROR: Target file system and source file system can not be the same. " "They were both: {}.".format(args.target_case) ) _clear_and_exit(msg) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.startSimulations, args=(argument,), ) thread.start() tracker = create_tracker(model, detailed_interval=0, ee_config=ee_config) out = open(os.devnull, "w") if args.disable_monitoring else sys.stdout monitor = Monitor(out=out, color_always=args.color_always) try: monitor.monitor(tracker) except (SystemExit, KeyboardInterrupt): print("\nKilling simulations...") tracker.request_termination() if args.disable_monitoring: out.close() thread.join() if model.hasRunFailed(): _clear_and_exit(1) # the monitor has already reported the error message
def test_exporter_is_not_valid(): config_file = "snake_oil.ert" rc = ResConfig(user_config_file=config_file) rc.convertToCReference(None) ert = EnKFMain(rc) notifier = ErtCliNotifier(ert, config_file) with ERT.adapt(notifier): ex = Exporter() assert not ex.is_valid()
def test_default_target_case_name_format_mode(self): config_file = self.createTestPath('local/poly_example/poly.ert') with ErtTestContext('test_default_target_case_name_format_mode', config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) args = Namespace(target_case=None) res = model_factory._target_case_name(args, format_mode=True) self.assertEqual("default_%d", res)
def test_exporter_is_valid(): with ErtPluginContext(): config_file = "snake_oil.ert" rc = ResConfig(user_config_file=config_file) rc.convertToCReference(None) ert = EnKFMain(rc) notifier = ErtCliNotifier(ert, config_file) with ERT.adapt(notifier): ex = Exporter() assert ex.is_valid(), "Missing CSV_EXPORT2 or EXPORT_RUNPATH jobs"
def test_setup_ensemble_experiment(self): config_file = self.createTestPath('local/poly_example/poly.ert') with ErtTestContext('test_single_test_run', config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) model, argument = model_factory._setup_single_test_run() self.assertTrue(isinstance(model, EnsembleExperiment)) self.assertEqual(1, len(argument.keys())) self.assertTrue("active_realizations" in argument)
def test_custom_target_case_name(self): config_file = self.createTestPath('local/poly_example/poly.ert') with ErtTestContext('test_custom_target_case_name', config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) custom_name = "test" args = Namespace(target_case=custom_name) res = model_factory._target_case_name(args) self.assertEqual(custom_name, res)
def test_default_target_case_name(self): config_file = self.createTestPath("local/poly_example/poly.ert") with ErtTestContext("test_default_target_case_name", config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) args = Namespace(target_case=None) res = model_factory._target_case_name(args) self.assertEqual("default_smoother_update", res)
def test_setup_single_test_run(self): config_file = self.createTestPath("local/poly_example/poly.ert") with ErtTestContext("test_single_test_run", config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) model, argument = model_factory._setup_single_test_run() self.assertTrue(isinstance(model, SingleTestRun)) self.assertEqual(1, len(argument.keys())) self.assertTrue("active_realizations" in argument) model.create_context(argument)
def test_init_iteration_number(self): config_file = self.createTestPath('local/poly_example/poly.ert') with ErtTestContext('test_init_iteration_number', config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) args = Namespace(iter_num=10, realizations=None) model, argument = model_factory._setup_ensemble_experiment(args) run_context = model.create_context(argument) self.assertEqual(argument['iter_num'], 10) self.assertEqual(run_context.get_iter(), 10)
def test_custom_realizations(self): config_file = self.createTestPath('local/poly_example/poly.ert') with ErtTestContext('test_custom_realizations', config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) args = Namespace(realizations="0-4,7,8") res = model_factory._realizations(args) ensemble_size = ERT.ert.getEnsembleSize() mask = BoolVector(default_value=False, initial_size=ensemble_size) mask.updateActiveMask("0-4,7,8") self.assertEqual(mask, res)
def run_cli(args): logging.basicConfig(level=logging.INFO, format='%(message)s') res_config = ResConfig(args.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=args.verbose) notifier = ErtCliNotifier(ert, args.config) ERT.adapt(notifier) if args.mode == WORKFLOW_MODE: execute_workflow(args.name) return model, argument = create_model(args) if args.disable_monitoring: model.startSimulations(argument) if model.hasRunFailed(): _clear_and_exit(model.getFailMessage()) else: # Test run does not have a current_case if "current_case" in args and args.current_case: ERT.enkf_facade.select_or_create_new_case(args.current_case) if (args.mode in [ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE] and args.target_case == ERT.enkf_facade.get_current_case_name()): msg = ( "ERROR: Target file system and source file system can not be the same. " "They were both: {}.".format(args.target_case) ) _clear_and_exit(msg) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.startSimulations, args=(argument,) ) thread.start() tracker = create_tracker(model, tick_interval=0, detailed_interval=0) monitor = Monitor(color_always=args.color_always) try: monitor.monitor(tracker) except (SystemExit, KeyboardInterrupt): print("\nKilling simulations...") model.killAllSimulations() thread.join() if model.hasRunFailed(): _clear_and_exit(1) # the monitor has already reported the error message
def test_setup_ensemble_smoother(self): config_file = self.createTestPath('local/poly_example/poly.ert') with ErtTestContext('test_single_test_run', config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) args = Namespace(realizations="0-4,7,8", target_case="test_case") model, argument = model_factory._setup_ensemble_smoother(args) self.assertTrue(isinstance(model, EnsembleSmoother)) self.assertEqual(3, len(argument.keys())) self.assertTrue("active_realizations" in argument) self.assertTrue("target_case" in argument) self.assertTrue("analysis_module" in argument)
def test_setup_multiple_data_assimilation(self): config_file = self.createTestPath('local/poly_example/poly.ert') with ErtTestContext('test_single_test_run', config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) args = Namespace(realizations="0-4,7,8", weights="6,4,2", target_case="test_case") model, argument = model_factory._setup_multiple_data_assimilation(args) self.assertTrue(isinstance(model, MultipleDataAssimilation)) self.assertEqual(4, len(argument.keys())) self.assertTrue("active_realizations" in argument) self.assertTrue("target_case" in argument) self.assertTrue("analysis_module" in argument) self.assertTrue("weights" in argument)
def test_executing_workflow(self): with open('test_wf', 'w') as wf_file: wf_file.write('EXPORT_RUNPATH') config_file = 'poly.ert' with open(config_file, 'a') as file: file.write("LOAD_WORKFLOW test_wf") rc = ResConfig(user_config_file=config_file) rc.convertToCReference(None) ert = EnKFMain(rc) notifier = ErtCliNotifier(ert, config_file) ERT.adapt(notifier) args = Namespace(name="test_wf") execute_workflow(args.name) assert os.path.isfile(".ert_runpath_list")
def test_run_export(): with ErtPluginContext(): config_file = "snake_oil.ert" rc = ResConfig(user_config_file=config_file) rc.convertToCReference(None) ert = EnKFMain(rc) notifier = ErtCliNotifier(ert, config_file) with ERT.adapt(notifier): ex = Exporter() parameters = { "output_file": "export.csv", "time_index": "raw", "column_keys": "FOPR", } ex.run_export(parameters) shutil.rmtree("storage") with pytest.raises(UserWarning) as warn: ex.run_export(parameters) assert ex._export_job in str(warn)
def run_cli(args): logging.basicConfig(level=logging.INFO, format='%(message)s') res_config = ResConfig(args.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=args.verbose) notifier = ErtCliNotifier(ert, args.config) ERT.adapt(notifier) if args.mode == 'workflow': execute_workflow(args.name) return model, argument = create_model(args) if args.disable_monitoring: model.startSimulations(argument) if model.hasRunFailed(): sys.exit(model.getFailMessage()) else: thread = threading.Thread(name="ert_cli_simulation_thread", target=model.startSimulations, args=(argument, )) thread.start() tracker = create_tracker(model, tick_interval=0, detailed_interval=0) monitor = Monitor(color_always=args.color_always) try: monitor.monitor(tracker) except (SystemExit, KeyboardInterrupt): print("\nKilling simulations...") model.killAllSimulations() thread.join() if model.hasRunFailed(): sys.exit(1) # the monitor has already reported the error message
def test_setup_iterative_ensemble_smoother(self): config_file = self.createTestPath("local/poly_example/poly.ert") with ErtTestContext("_setup_iterative_ensemble_smoother", config_file) as work_area: ert = work_area.getErt() notifier = ErtCliNotifier(ert, config_file) with ERT.adapt(notifier): args = Namespace( realizations="0-4,7,8", target_case="test_case_%d", num_iterations="10", ) model, argument = model_factory._setup_iterative_ensemble_smoother( args) self.assertTrue(isinstance(model, IteratedEnsembleSmoother)) self.assertEqual(4, len(argument.keys())) self.assertTrue("active_realizations" in argument) self.assertTrue("target_case" in argument) self.assertTrue("analysis_module" in argument) self.assertTrue("num_iterations" in argument) self.assertTrue( ERT.enkf_facade.get_number_of_iterations() == 10) model.create_context(argument, 0)
def test_tracking( experiment_folder, cmd_line_arguments, num_successful, num_iters, assert_present_in_snapshot, tmpdir, source_root, ): shutil.copytree( os.path.join(source_root, "test-data", "local", f"{experiment_folder}"), os.path.join(str(tmpdir), f"{experiment_folder}"), ) config_lines = [ "INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n" ] with tmpdir.as_cwd(): with open(f"{experiment_folder}/poly.ert", "a") as fh: fh.writelines(config_lines) parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, cmd_line_arguments, ) FeatureToggling.update_from_args(parsed) res_config = ResConfig(parsed.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose) notifier = ErtCliNotifier(ert, parsed.config) ERT.adapt(notifier) model, argument = create_model(parsed) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.start_simulations_thread, args=(argument, ), ) thread.start() tracker = create_tracker(model, general_interval=1, detailed_interval=2, ee_config=ee_config) snapshots = {} for event in tracker.track(): if isinstance(event, FullSnapshotEvent): snapshots[event.iteration] = event.snapshot if isinstance(event, SnapshotUpdateEvent): if event.partial_snapshot is not None: snapshots[event.iteration].merge( event.partial_snapshot.data()) if isinstance(event, EndEvent): pass assert tracker._progress() == 1.0 assert len(snapshots) == num_iters for iter_, snapshot in snapshots.items(): successful_reals = list( filter( lambda item: item[1].status == state. REALIZATION_STATE_FINISHED, snapshot.get_reals().items(), )) assert len(successful_reals) == num_successful for ( iter_expression, snapshot_expression, expected, ) in assert_present_in_snapshot: for i, snapshot in snapshots.items(): if re.match(iter_expression, str(i)): check_expression( snapshot.to_dict(), snapshot_expression, expected, f"Snapshot {i} did not match:\n", ) thread.join()
def test_tracking( experiment_folder, cmd_line_arguments, num_successful, num_iters, tmpdir, source_root, ): shutil.copytree( os.path.join(source_root, "test-data", "local", f"{experiment_folder}"), os.path.join(str(tmpdir), f"{experiment_folder}"), ) config_lines = [ "INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n" ] with tmpdir.as_cwd(): with open(f"{experiment_folder}/poly.ert", "a") as fh: fh.writelines(config_lines) parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, cmd_line_arguments, ) FeatureToggling.update_from_args(parsed) res_config = ResConfig(parsed.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose) notifier = ErtCliNotifier(ert, parsed.config) ERT.adapt(notifier) model, argument = create_model(parsed) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.startSimulations, args=(argument, ), ) thread.start() tracker = create_tracker(model, general_interval=1, detailed_interval=2, ee_config=ee_config) snapshots = {} for event in tracker.track(): if isinstance(event, FullSnapshotEvent): snapshots[event.iteration] = event.snapshot if isinstance(event, SnapshotUpdateEvent): if event.partial_snapshot is not None: snapshots[event.iteration].merge( event.partial_snapshot.data()) if isinstance(event, EndEvent): break assert tracker._progress() == 1.0 assert len(snapshots) == num_iters for iter_, snapshot in snapshots.items(): successful_reals = list( filter( lambda item: item[1].status == REALIZATION_STATE_FINISHED, snapshot.get_reals().items(), )) assert len(successful_reals) == num_successful for real_id, real in successful_reals: assert (real.status == REALIZATION_STATE_FINISHED ), f"iter:{iter_} real:{real_id} was not finished" poly = real.steps["0"].jobs["0"] poly2 = real.steps["0"].jobs["1"] assert poly.name == "poly_eval" assert (poly.status == JOB_STATE_FINISHED ), f"real {real_id}/{poly['name']} was not finished" assert poly2.name == "poly_eval2" assert (poly2.status == JOB_STATE_FINISHED ), f"real {real_id}/{poly['name']} was not finished"
def run_cli(args): res_config = ResConfig(args.config) # Create logger inside function to make sure all handlers have been added to the root-logger. logger = logging.getLogger(__name__) logger.info( "Logging forward model jobs", extra={ "workflow_jobs": str(res_config.model_config.getForwardModel().joblist()) }, ) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=args.verbose) notifier = ErtCliNotifier(ert, args.config) with ERT.adapt(notifier): if args.mode == WORKFLOW_MODE: execute_workflow(args.name) return model, argument = create_model(args) # Test run does not have a current_case if "current_case" in args and args.current_case: ERT.enkf_facade.select_or_create_new_case(args.current_case) if (args.mode in [ ENSEMBLE_SMOOTHER_MODE, ITERATIVE_ENSEMBLE_SMOOTHER_MODE, ES_MDA_MODE ] and args.target_case == ERT.enkf_facade.get_current_case_name()): msg = ( "ERROR: Target file system and source file system can not be the same. " "They were both: {}.".format(args.target_case)) raise ErtCliError(msg) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig( custom_port_range=args.port_range) argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.start_simulations_thread, args=(argument, ), ) thread.start() tracker = create_tracker(model, detailed_interval=0, ee_config=ee_config) out = open(os.devnull, "w") if args.disable_monitoring else sys.stderr monitor = Monitor(out=out, color_always=args.color_always) try: monitor.monitor(tracker) except (SystemExit, KeyboardInterrupt): print("\nKilling simulations...") tracker.request_termination() if args.disable_monitoring: out.close() thread.join() if model.hasRunFailed(): raise ErtCliError(model.getFailMessage())