def test_ensemble_evaluator(tmpdir, source_root): shutil.copytree( os.path.join(source_root, "test-data", "local", "poly_example"), os.path.join(str(tmpdir), "poly_example"), ) with tmpdir.as_cwd(): parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, [ ENSEMBLE_SMOOTHER_MODE, "--target-case", "poly_runpath_file", "--realizations", "1,2,4,8,16,32,64", "--enable-ensemble-evaluator", "poly_example/poly.ert", ], ) FeatureToggling.update_from_args(parsed) assert FeatureToggling.is_enabled("ensemble-evaluator") is True run_cli(parsed) FeatureToggling.reset()
def test_es_mda(tmpdir, source_root): shutil.copytree( os.path.join(source_root, "test-data", "local", "poly_example"), os.path.join(str(tmpdir), "poly_example"), ) with tmpdir.as_cwd(): parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, [ ES_MDA_MODE, "--target-case", "iter-%d", "--realizations", "1,2,4,8,16", "poly_example/poly.ert", "--port-range", "1024-65535", "--weights", "1", ], ) FeatureToggling.update_from_args(parsed) run_cli(parsed) FeatureToggling.reset()
def main(): with open(LOGGING_CONFIG, encoding="utf-8") as conf_file: logging.config.dictConfig(yaml.safe_load(conf_file)) set_abort_handler(_log_util_abort) import locale locale.setlocale(locale.LC_NUMERIC, "C") args = ert_parser(None, sys.argv[1:]) logger = logging.getLogger(__name__) if args.verbose: logger.setLevel("DEBUG") FeatureToggling.update_from_args(args) try: with start_ert_server(args.mode), ErtPluginContext() as context: context.plugin_manager.add_logging_handle_to_root( logging.getLogger()) logger.info("Running ert with {}".format(str(args))) args.func(args) except ErtCliError as err: logger.exception(str(err)) sys.exit(str(err)) except BaseException as err: logger.exception(f'ERT crashed unexpectedly with "{err}"') logfiles = set() # Use set to avoid duplicates... for handler in logging.getLogger().handlers: if isinstance(handler, logging.FileHandler): logfiles.add(handler.baseFilename) msg = f'ERT crashed unexpectedly with "{err}".\nSee logfile(s) for details:' msg += "\n " + "\n ".join(logfiles) sys.exit(msg)
def main(): args = ert_parser(None, sys.argv[1:]) if args.verbose: logger = logging.getLogger() logger.setLevel("DEBUG") FeatureToggling.update_from_args(args) with ErtPluginContext(): args.func(args) clear_global_state()
def main(): import ert_logging # Only use ert logger config when running ERT args = ert_parser(None, sys.argv[1:]) if args.verbose: logger = logging.getLogger() logger.setLevel("DEBUG") FeatureToggling.update_from_args(args) with ErtPluginContext(): args.func(args) clear_global_state()
def main(): import ert_logging # Only use ert logger config when running ERT import locale locale.setlocale(locale.LC_NUMERIC, "C") args = ert_parser(None, sys.argv[1:]) if args.verbose: logger = logging.getLogger() logger.setLevel("DEBUG") FeatureToggling.update_from_args(args) initialize_databases() with start_ert_server(), ErtPluginContext(): args.func(args) clear_global_state()
def test_tracking( experiment_folder, cmd_line_arguments, num_successful, num_iters, tmpdir, source_root, ): shutil.copytree( os.path.join(source_root, "test-data", "local", f"{experiment_folder}"), os.path.join(str(tmpdir), f"{experiment_folder}"), ) config_lines = [ "INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n" ] with tmpdir.as_cwd(): with open(f"{experiment_folder}/poly.ert", "a") as fh: fh.writelines(config_lines) parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, cmd_line_arguments, ) FeatureToggling.update_from_args(parsed) res_config = ResConfig(parsed.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose) notifier = ErtCliNotifier(ert, parsed.config) ERT.adapt(notifier) model, argument = create_model(parsed) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.startSimulations, args=(argument, ), ) thread.start() tracker = create_tracker(model, general_interval=1, detailed_interval=2, ee_config=ee_config) snapshots = {} for event in tracker.track(): if isinstance(event, FullSnapshotEvent): snapshots[event.iteration] = event.snapshot if isinstance(event, SnapshotUpdateEvent): if event.partial_snapshot is not None: snapshots[event.iteration].merge( event.partial_snapshot.data()) if isinstance(event, EndEvent): break assert tracker._progress() == 1.0 assert len(snapshots) == num_iters for iter_, snapshot in snapshots.items(): successful_reals = list( filter( lambda item: item[1].status == REALIZATION_STATE_FINISHED, snapshot.get_reals().items(), )) assert len(successful_reals) == num_successful for real_id, real in successful_reals: assert (real.status == REALIZATION_STATE_FINISHED ), f"iter:{iter_} real:{real_id} was not finished" poly = real.steps["0"].jobs["0"] poly2 = real.steps["0"].jobs["1"] assert poly.name == "poly_eval" assert (poly.status == JOB_STATE_FINISHED ), f"real {real_id}/{poly['name']} was not finished" assert poly2.name == "poly_eval2" assert (poly2.status == JOB_STATE_FINISHED ), f"real {real_id}/{poly['name']} was not finished"
def test_tracking( extra_config, extra_poly_eval, cmd_line_arguments, num_successful, num_iters, progress, assert_present_in_snapshot, tmpdir, source_root, ): experiment_folder = "poly_example" shutil.copytree( os.path.join(source_root, "test-data", "local", f"{experiment_folder}"), os.path.join(str(tmpdir), f"{experiment_folder}"), ) config_lines = [ "INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n", extra_config, ] with tmpdir.as_cwd(): with open(f"{experiment_folder}/poly.ert", "a") as fh: fh.writelines(config_lines) with fileinput.input(f"{experiment_folder}/poly_eval.py", inplace=True) as fin: for line in fin: if line.strip().startswith("coeffs"): print(extra_poly_eval) print(line, end="") parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, cmd_line_arguments, ) FeatureToggling.update_from_args(parsed) res_config = ResConfig(parsed.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose) facade = LibresFacade(ert) model = create_model( ert, facade.get_ensemble_size(), facade.get_current_case_name(), parsed, ) evaluator_server_config = EvaluatorServerConfig( custom_port_range=range(1024, 65535), custom_host="127.0.0.1" ) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.start_simulations_thread, args=(evaluator_server_config,), ) thread.start() tracker = EvaluatorTracker( model, ee_con_info=evaluator_server_config.get_connection_info(), ) snapshots = {} for event in tracker.track(): if isinstance(event, FullSnapshotEvent): snapshots[event.iteration] = event.snapshot if isinstance(event, SnapshotUpdateEvent): if event.partial_snapshot is not None: snapshots[event.iteration].merge(event.partial_snapshot.data()) if isinstance(event, EndEvent): pass assert tracker._progress() == progress assert len(snapshots) == num_iters for iter_, snapshot in snapshots.items(): successful_reals = list( filter( lambda item: item[1].status == REALIZATION_STATE_FINISHED, snapshot.reals.items(), ) ) assert len(successful_reals) == num_successful for ( iter_expression, snapshot_expression, expected, ) in assert_present_in_snapshot: for i, snapshot in snapshots.items(): if re.match(iter_expression, str(i)): check_expression( snapshot.to_dict(), snapshot_expression, expected, f"Snapshot {i} did not match:\n", ) thread.join() FeatureToggling.reset()
def test_tracking( experiment_folder, cmd_line_arguments, num_successful, num_iters, assert_present_in_snapshot, tmpdir, source_root, ): shutil.copytree( os.path.join(source_root, "test-data", "local", f"{experiment_folder}"), os.path.join(str(tmpdir), f"{experiment_folder}"), ) config_lines = [ "INSTALL_JOB poly_eval2 POLY_EVAL\n" "SIMULATION_JOB poly_eval2\n" ] with tmpdir.as_cwd(): with open(f"{experiment_folder}/poly.ert", "a") as fh: fh.writelines(config_lines) parser = ArgumentParser(prog="test_main") parsed = ert_parser( parser, cmd_line_arguments, ) FeatureToggling.update_from_args(parsed) res_config = ResConfig(parsed.config) os.chdir(res_config.config_path) ert = EnKFMain(res_config, strict=True, verbose=parsed.verbose) notifier = ErtCliNotifier(ert, parsed.config) ERT.adapt(notifier) model, argument = create_model(parsed) ee_config = None if FeatureToggling.is_enabled("ensemble-evaluator"): ee_config = EvaluatorServerConfig() argument.update({"ee_config": ee_config}) thread = threading.Thread( name="ert_cli_simulation_thread", target=model.start_simulations_thread, args=(argument, ), ) thread.start() tracker = create_tracker(model, general_interval=1, detailed_interval=2, ee_config=ee_config) snapshots = {} for event in tracker.track(): if isinstance(event, FullSnapshotEvent): snapshots[event.iteration] = event.snapshot if isinstance(event, SnapshotUpdateEvent): if event.partial_snapshot is not None: snapshots[event.iteration].merge( event.partial_snapshot.data()) if isinstance(event, EndEvent): pass assert tracker._progress() == 1.0 assert len(snapshots) == num_iters for iter_, snapshot in snapshots.items(): successful_reals = list( filter( lambda item: item[1].status == state. REALIZATION_STATE_FINISHED, snapshot.get_reals().items(), )) assert len(successful_reals) == num_successful for ( iter_expression, snapshot_expression, expected, ) in assert_present_in_snapshot: for i, snapshot in snapshots.items(): if re.match(iter_expression, str(i)): check_expression( snapshot.to_dict(), snapshot_expression, expected, f"Snapshot {i} did not match:\n", ) thread.join()