class SummaryCollectorTest(ResTest):
def setUp(self):
self.monkeypatch = MonkeyPatch()
self.monkeypatch.setenv(
"TZ", "CET"
) # The ert_statoil case was generated in CET
self.config = self.createTestPath("local/snake_oil/snake_oil.ert")
def tearDown(self):
self.monkeypatch.undo()
def test_summary_collector(self):
with ErtTestContext(
"python/enkf/export/summary_collector", self.config
) as context:
ert = context.getErt()
data = SummaryCollector.loadAllSummaryData(ert, "default_0")
self.assertFloatEqual(data["WWCT:OP2"][0]["2010-01-10"], 0.385549)
self.assertFloatEqual(data["WWCT:OP2"][24]["2010-01-10"], 0.498331)
self.assertFloatEqual(data["FOPR"][0]["2010-01-10"], 0.118963)
self.assertFloatEqual(data["FOPR"][0]["2015-06-23"], 0.133601)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_60 = data.loc[60]
data = SummaryCollector.loadAllSummaryData(
ert, "default_0", ["WWCT:OP1", "WWCT:OP2"]
)
self.assertFloatEqual(data["WWCT:OP1"][0]["2010-01-10"], 0.352953)
self.assertFloatEqual(data["WWCT:OP2"][0]["2010-01-10"], 0.385549)
with self.assertRaises(KeyError):
data["FOPR"]
realization_index = 10
data = SummaryCollector.loadAllSummaryData(
ert,
"default_0",
["WWCT:OP1", "WWCT:OP2"],
realization_index=realization_index,
)
assert data.index.levels[0] == [realization_index]
assert len(data.index.levels[1]) == 200
assert list(data.columns) == ["WWCT:OP1", "WWCT:OP2"]
non_existing_realization_index = 150
with pytest.raises(IndexError):
data = SummaryCollector.loadAllSummaryData(
ert,
"default_0",
["WWCT:OP1", "WWCT:OP2"],
realization_index=non_existing_realization_index,
)
class SummaryObservationCollectorTest(ResTest):
def setUp(self):
self.monkeypatch = MonkeyPatch()
self.monkeypatch.setenv(
"TZ", "CET"
) # The ert_statoil case was generated in CET
self.config = self.createTestPath("local/snake_oil/snake_oil.ert")
def tearDown(self):
self.monkeypatch.undo()
def test_summary_observation_collector(self):
with ErtTestContext(
"python/enkf/export/summary_observation_collector", self.config
) as context:
ert = context.getErt()
self.assertTrue(
SummaryObservationCollector.summaryKeyHasObservations(ert, "FOPR")
)
self.assertFalse(
SummaryObservationCollector.summaryKeyHasObservations(ert, "FOPT")
)
keys = SummaryObservationCollector.getAllObservationKeys(ert)
self.assertTrue("FOPR" in keys)
self.assertTrue("WOPR:OP1" in keys)
self.assertFalse("WOPR:OP2" in keys)
data = SummaryObservationCollector.loadObservationData(ert, "default_0")
self.assertFloatEqual(data["FOPR"]["2010-01-10"], 0.001696887)
self.assertFloatEqual(data["STD_FOPR"]["2010-01-10"], 0.1)
self.assertFloatEqual(data["WOPR:OP1"]["2010-03-31"], 0.1)
self.assertFloatEqual(data["STD_WOPR:OP1"]["2010-03-31"], 0.05)
with self.assertRaises(KeyError):
fgir = data["FGIR"]
data = SummaryObservationCollector.loadObservationData(
ert, "default_0", ["WOPR:OP1"]
)
self.assertFloatEqual(data["WOPR:OP1"]["2010-03-31"], 0.1)
self.assertFloatEqual(data["STD_WOPR:OP1"]["2010-03-31"], 0.05)
with self.assertRaises(KeyError):
data["FOPR"]
def patch_module(
monkeypatch: MonkeyPatch = MonkeyPatch(),
) -> Generator[MonkeyPatch, None, None]:
async def mock_get_file_content(*args: Any, **kwargs: Any) -> bytes:
filename = kwargs["file"].name
if filename in {
"doctest.py",
"annotation.py",
"descriptive_name.py",
"return_annotation.py",
"no_errors.py",
}:
return get_source(filename)
else:
return b""
monkeypatch.setattr(utils, "get_file_content", mock_get_file_content)
yield monkeypatch
monkeypatch.undo()
class RMSConfigTest(ResTest):
def setUp(self):
self.monkeypatch = MonkeyPatch()
pass
def tearDown(self):
self.monkeypatch.undo()
def test_load(self):
self.monkeypatch.setenv("RMS_SITE_CONFIG", "file/does/not/exist")
with self.assertRaises(IOError):
conf = RMSConfig()
self.monkeypatch.setenv("RMS_SITE_CONFIG", RMSConfig.DEFAULT_CONFIG_FILE)
conf = RMSConfig()
with self.assertRaises(OSError):
conf.executable
with TestAreaContext("yaml"):
with open("file.yml", "w") as f:
f.write("this:\n -should\n-be\ninvalid:yaml?")
self.monkeypatch.setenv("RMS_SITE_CONFIG", "file.yml")
with self.assertRaises(ValueError):
conf = RMSConfig()
os.mkdir("bin")
with open("bin/rms", "w") as f:
f.write("This is an RMS executable ...")
os.chmod("bin/rms", stat.S_IEXEC)
with open("file.yml", "w") as f:
f.write("executable: bin/rms")
conf = RMSConfig()
self.assertEqual(conf.executable, "bin/rms")
self.assertIsNone(conf.threads)
with open("file.yml", "w") as f:
f.write("executable: bin/rms\n")
f.write("threads: 17")
conf = RMSConfig()
self.assertEqual(conf.threads, 17)
with open("file.yml", "w") as f:
f.write("executable: bin/rms\n")
f.write("wrapper: not-exisiting-exec")
conf = RMSConfig()
with self.assertRaises(OSError):
conf.wrapper
with open("file.yml", "w") as f:
f.write("executable: bin/rms\n")
f.write("wrapper: bash")
conf = RMSConfig()
self.assertEqual(conf.wrapper, "bash")
def test_load_env(self):
with TestAreaContext("yaml"):
self.monkeypatch.setenv("RMS_SITE_CONFIG", "file.yml")
with open("file.yml", "w") as f:
f.write(
"""\
executable: bin/rms\n
wrapper: bash
env:
10.1.3:
PATH_PREFIX: /some/path
PYTHONPATH: /some/pythonpath
"""
)
conf = RMSConfig()
self.assertEqual(conf.env("10.1.3")["PATH_PREFIX"], "/some/path")
self.assertEqual(conf.env("10.1.3")["PYTHONPATH"], "/some/pythonpath")
self.assertEqual(conf.env("non_existing"), {})
class SiteConfigTest(ResTest):
def setUp(self):
self.case_directory = self.createTestPath("local/simple_config/")
self.snake_case_directory = self.createTestPath("local/snake_oil/")
self.monkeypatch = MonkeyPatch()
def tearDown(self):
self.monkeypatch.undo()
def test_invalid_user_config(self):
with TestAreaContext("void land"):
with self.assertRaises(IOError):
SiteConfig("this/is/not/a/file")
def test_init(self):
with TestAreaContext("site_config_init_test") as work_area:
work_area.copy_directory(self.case_directory)
config_file = "simple_config/minimum_config"
site_config = SiteConfig(user_config_file=config_file)
self.assertIsNotNone(site_config)
def test_constructors(self):
with TestAreaContext("site_config_constructor_test") as work_area:
work_area.copy_directory(self.snake_case_directory)
config_file = "snake_oil/snake_oil.ert"
ERT_SITE_CONFIG = SiteConfig.getLocation()
ERT_SHARE_PATH = os.path.dirname(ERT_SITE_CONFIG)
snake_config_dict = {
ConfigKeys.INSTALL_JOB: [
{
ConfigKeys.NAME:
"SNAKE_OIL_SIMULATOR",
ConfigKeys.PATH:
os.getcwd() + "/snake_oil/jobs/SNAKE_OIL_SIMULATOR",
},
{
ConfigKeys.NAME:
"SNAKE_OIL_NPV",
ConfigKeys.PATH:
os.getcwd() + "/snake_oil/jobs/SNAKE_OIL_NPV",
},
{
ConfigKeys.NAME:
"SNAKE_OIL_DIFF",
ConfigKeys.PATH:
os.getcwd() + "/snake_oil/jobs/SNAKE_OIL_DIFF",
},
],
ConfigKeys.INSTALL_JOB_DIRECTORY: [
ERT_SHARE_PATH + "/forward-models/res",
ERT_SHARE_PATH + "/forward-models/shell",
ERT_SHARE_PATH + "/forward-models/templating",
ERT_SHARE_PATH + "/forward-models/old_style",
],
ConfigKeys.SETENV: [
{
ConfigKeys.NAME: "SILLY_VAR",
ConfigKeys.VALUE: "silly-value"
},
{
ConfigKeys.NAME: "OPTIONAL_VAR",
ConfigKeys.VALUE: "optional-value",
},
],
ConfigKeys.LICENSE_PATH:
"some/random/path",
ConfigKeys.UMASK:
18,
}
site_config_user_file = SiteConfig(user_config_file=config_file)
site_config_dict = SiteConfig(config_dict=snake_config_dict)
self.assertEqual(site_config_dict, site_config_user_file)
with self.assertRaises(ValueError):
site_config = SiteConfig(user_config_file=config_file,
config_dict=snake_config_dict)
@tmpdir()
def test_site_config_hook_workflow(self):
site_config_filename = "test_site_config"
test_config_filename = "test_config"
site_config_content = """
LOAD_WORKFLOW_JOB ECHO_WORKFLOW_JOB
LOAD_WORKFLOW ECHO_WORKFLOW
HOOK_WORKFLOW ECHO_WORKFLOW PRE_SIMULATION
"""
with open(site_config_filename, "w") as fh:
fh.write(site_config_content)
with open(test_config_filename, "w") as fh:
fh.write("NUM_REALIZATIONS 1\n")
with open("ECHO_WORKFLOW_JOB", "w") as fh:
fh.write("""INTERNAL False
EXECUTABLE echo
MIN_ARG 1
""")
with open("ECHO_WORKFLOW", "w") as fh:
fh.write("ECHO_WORKFLOW_JOB hello")
self.monkeypatch.setenv("ERT_SITE_CONFIG", site_config_filename)
res_config = ResConfig(user_config_file=test_config_filename)
self.assertTrue(len(res_config.hook_manager) == 1)
self.assertEqual(
res_config.hook_manager[0].getWorkflow().src_file,
os.path.join(os.getcwd(), "ECHO_WORKFLOW"),
)
class EclConfigTest(ResTest):
def setUp(self):
self.ecl_config_path = os.path.dirname(inspect.getsourcefile(Ecl100Config))
self.monkeypatch = MonkeyPatch()
def tearDown(self):
self.monkeypatch.undo()
def test_load(self):
self.monkeypatch.setenv("ECL100_SITE_CONFIG", "file/does/not/exist")
with self.assertRaises(IOError):
conf = Ecl100Config()
self.monkeypatch.setenv(
"ECL100_SITE_CONFIG",
os.path.join(self.ecl_config_path, "ecl100_config.yml"),
)
conf = Ecl100Config()
with TestAreaContext("yaml_invalid"):
with open("file.yml", "w") as f:
f.write("this:\n -should\n-be\ninvalid:yaml?")
self.monkeypatch.setenv("ECL100_SITE_CONFIG", "file.yml")
with self.assertRaises(ValueError):
conf = Ecl100Config()
scalar_exe = "bin/scalar_exe"
mpi_exe = "bin/mpi_exe"
mpi_run = "bin/mpi_run"
os.mkdir("bin")
for f in ["scalar_exe", "mpi_exe", "mpi_run"]:
fname = os.path.join("bin", f)
with open(fname, "w") as fh:
fh.write("This is an exectable ...")
os.chmod(fname, stat.S_IEXEC)
intel_path = "intel"
self.monkeypatch.setenv("ENV1", "A")
self.monkeypatch.setenv("ENV2", "C")
d = {
Keys.env: {"LICENSE_SERVER": "*****@*****.**"},
Keys.versions: {
"2015": {
Keys.scalar: {Keys.executable: scalar_exe},
Keys.mpi: {
Keys.executable: mpi_exe,
Keys.mpirun: mpi_run,
Keys.env: {
"I_MPI_ROOT": "$ENV1:B:$ENV2",
"TEST_VAR": "$ENV1.B.$ENV2 $UNKNOWN_VAR",
"P4_RSHCOMMAND": "",
"LD_LIBRARY_PATH": "{}:$LD_LIBRARY_PATH".format(
intel_path
),
"PATH": "{}/bin64:$PATH".format(intel_path),
},
},
},
"2016": {
Keys.scalar: {Keys.executable: "/does/not/exist"},
Keys.mpi: {
Keys.executable: "/does/not/exist",
Keys.mpirun: mpi_run,
},
},
"2017": {
Keys.mpi: {
Keys.executable: mpi_exe,
Keys.mpirun: "/does/not/exist",
}
},
},
}
with open("file.yml", "w") as f:
f.write(yaml.dump(d))
conf = Ecl100Config()
# Fails because there is no version 2020
with self.assertRaises(KeyError):
sim = conf.sim("2020")
# Fails because the 2016 version points to a not existing executable
with self.assertRaises(OSError):
sim = conf.sim("2016")
# Fails because the 2016 mpi version points to a non existing mpi
# executable
with self.assertRaises(OSError):
sim = conf.mpi_sim("2016")
# Fails because the 2017 mpi version mpirun points to a non existing
# mpi executable
with self.assertRaises(OSError):
sim = conf.mpi_sim("2017")
# Fails because the 2017 scalar version is not registered
with self.assertRaises(KeyError):
sim = conf.sim("2017")
sim = conf.sim("2015")
mpi_sim = conf.mpi_sim("2015")
# Check that global environment has been propagated down.
self.assertIn("LICENSE_SERVER", mpi_sim.env)
# Check replacement of $ENV_VAR in values.
self.assertEqual(mpi_sim.env["I_MPI_ROOT"], "A:B:C")
self.assertEqual(mpi_sim.env["TEST_VAR"], "A.B.C $UNKNOWN_VAR")
self.assertEqual(len(mpi_sim.env), 1 + 5)
sim = conf.sim("2015")
self.assertEqual(sim.executable, scalar_exe)
self.assertIsNone(sim.mpirun)
with self.assertRaises(Exception):
simulators = conf.simulators()
simulators = conf.simulators(strict=False)
self.assertEqual(len(simulators), 2)
def test_default(self):
with TestAreaContext("default"):
os.mkdir("bin")
scalar_exe = "bin/scalar_exe"
with open(scalar_exe, "w") as fh:
fh.write("This is an exectable ...")
os.chmod(scalar_exe, stat.S_IEXEC)
d0 = {
Keys.versions: {
"2015": {Keys.scalar: {Keys.executable: scalar_exe}},
"2016": {Keys.scalar: {Keys.executable: scalar_exe}},
}
}
d1 = {
Keys.default_version: "2015",
Keys.versions: {
"2015": {Keys.scalar: {Keys.executable: scalar_exe}},
"2016": {Keys.scalar: {Keys.executable: scalar_exe}},
},
}
self.monkeypatch.setenv("ECL100_SITE_CONFIG", os.path.join("file.yml"))
with open("file.yml", "w") as f:
f.write(yaml.dump(d1))
conf = Ecl100Config()
sim = conf.sim()
self.assertEqual(sim.version, "2015")
self.assertIn("2015", conf)
self.assertNotIn("xxxx", conf)
self.assertIn(Keys.default, conf)
self.assertIn(None, conf)
sim = conf.sim("default")
self.assertEqual(sim.version, "2015")
with open("file.yml", "w") as f:
f.write(yaml.dump(d0))
conf = Ecl100Config()
self.assertNotIn(Keys.default, conf)
self.assertIsNone(conf.default_version)
with self.assertRaises(Exception):
sim = conf.sim()
with self.assertRaises(Exception):
sim = conf.sim(Keys.default)
class ShellTest(ResTest):
def setUp(self):
self.monkeypatch = MonkeyPatch()
def tearDown(self):
self.monkeypatch.undo()
def test_symlink(self):
with self.assertRaises(IOError):
symlink("target/does/not/exist", "link")
with TestAreaContext("symlink/test"):
with open("target", "w") as fileH:
fileH.write("target ...")
symlink("target", "link")
self.assertTrue(os.path.islink("link"))
self.assertEqual(os.readlink("link"), "target")
with open("target2", "w") as fileH:
fileH.write("target ...")
with self.assertRaises(OSError):
symlink("target2", "target")
symlink("target2", "link")
self.assertTrue(os.path.islink("link"))
self.assertEqual(os.readlink("link"), "target2")
os.makedirs("root1/sub1/sub2")
os.makedirs("root2/sub1/sub2")
os.makedirs("run")
symlink("../target", "linkpath/link")
self.assertTrue(os.path.isdir("linkpath"))
self.assertTrue(os.path.islink("linkpath/link"))
symlink("../target", "linkpath/link")
self.assertTrue(os.path.isdir("linkpath"))
self.assertTrue(os.path.islink("linkpath/link"))
with TestAreaContext("symlink/test2"):
os.makedirs("path")
with open("path/target", "w") as f:
f.write("1234")
symlink("path/target", "link")
self.assertTrue(os.path.islink("link"))
self.assertTrue(os.path.isfile("path/target"))
symlink("path/target", "link")
self.assertTrue(os.path.islink("link"))
self.assertTrue(os.path.isfile("path/target"))
with open("link") as f:
s = f.read()
self.assertEqual(s, "1234")
def test_mkdir(self):
with TestAreaContext("shell/mkdir"):
with open("file", "w") as f:
f.write("Hei")
with self.assertRaises(OSError):
mkdir("file")
mkdir("path")
self.assertTrue(os.path.isdir("path"))
mkdir("path")
mkdir("path/subpath")
self.assertTrue(os.path.isdir("path/subpath"))
def test_move_file(self):
with TestAreaContext("shell/move_file"):
with open("file", "w") as f:
f.write("Hei")
move_file("file", "file2")
self.assertTrue(os.path.isfile("file2"))
self.assertFalse(os.path.isfile("file"))
with self.assertRaises(IOError):
move_file("file2", "path/file2")
mkdir("path")
move_file("file2", "path/file2")
self.assertTrue(os.path.isfile("path/file2"))
self.assertFalse(os.path.isfile("file2"))
with self.assertRaises(IOError):
move_file("path", "path2")
with self.assertRaises(IOError):
move_file("not_existing", "target")
with open("file2", "w") as f:
f.write("123")
move_file("file2", "path/file2")
self.assertTrue(os.path.isfile("path/file2"))
self.assertFalse(os.path.isfile("file2"))
mkdir("rms/ipl")
with open("global_variables.ipl", "w") as f:
f.write("123")
move_file("global_variables.ipl", "rms/ipl/global_variables.ipl")
def test_move_file_into_folder_file_exists(self):
with TestAreaContext("shell/test_move_file_into_folder_file_exists"):
mkdir("dst_folder")
with open("dst_folder/file", "w") as f:
f.write("old")
with open("file", "w") as f:
f.write("new")
with open("dst_folder/file", "r") as f:
content = f.read()
self.assertEqual(content, "old")
move_file("file", "dst_folder")
with open("dst_folder/file", "r") as f:
content = f.read()
self.assertEqual(content, "new")
self.assertFalse(os.path.exists("file"))
def test_move_pathfile_into_folder(self):
with TestAreaContext("shell/test_move_pathfile"):
mkdir("dst_folder")
mkdir("source1/source2/")
with open("source1/source2/file", "w") as f:
f.write("stuff")
move_file("source1/source2/file", "dst_folder")
with open("dst_folder/file", "r") as f:
content = f.read()
self.assertEqual(content, "stuff")
self.assertFalse(os.path.exists("source1/source2/file"))
def test_move_pathfile_into_folder_file_exists(self):
with TestAreaContext("shell/test_move_pathfile"):
mkdir("dst_folder")
mkdir("source1/source2/")
with open("source1/source2/file", "w") as f:
f.write("stuff")
with open("dst_folder/file", "w") as f:
f.write("garbage")
move_file("source1/source2/file", "dst_folder")
with open("dst_folder/file", "r") as f:
content = f.read()
self.assertEqual(content, "stuff")
self.assertFalse(os.path.exists("source1/source2/file"))
def test_delete_file(self):
with TestAreaContext("delete/file"):
mkdir("pathx")
with self.assertRaises(IOError):
delete_file("pathx")
# deleteFile which does not exist - is silently ignored
delete_file("does/not/exist")
with open("file", "w") as f:
f.write("hei")
symlink("file", "link")
self.assertTrue(os.path.islink("link"))
delete_file("file")
self.assertFalse(os.path.isfile("file"))
self.assertTrue(os.path.islink("link"))
delete_file("link")
self.assertFalse(os.path.islink("link"))
def test_delete_directory(self):
with TestAreaContext("delete/directory"):
# deleteDriecteory which does not exist - is silently ignored
delete_directory("does/not/exist")
with open("file", "w") as f:
f.write("hei")
with self.assertRaises(IOError):
delete_directory("file")
mkdir("link_target/subpath")
with open("link_target/link_file", "w") as f:
f.write("hei")
mkdir("path/subpath")
with open("path/file", "w") as f:
f.write("hei")
with open("path/subpath/file", "w") as f:
f.write("hei")
symlink("../link_target", "path/link")
delete_directory("path")
self.assertFalse(os.path.exists("path"))
self.assertTrue(os.path.exists("link_target/link_file"))
def test_copy_directory_error(self):
with self.assertRaises(IOError):
copy_directory("does/not/exist", "target")
with TestAreaContext("copy/directory"):
with open("file", "w") as f:
f.write("hei")
with self.assertRaises(IOError):
copy_directory("hei", "target")
def test_copy_file(self):
with TestAreaContext("copy/file"):
with self.assertRaises(IOError):
copy_file("does/not/exist", "target")
mkdir("path")
with self.assertRaises(IOError):
copy_file("path", "target")
with open("file1", "w") as f:
f.write("hei")
copy_file("file1", "file2")
self.assertTrue(os.path.isfile("file2"))
copy_file("file1", "path")
self.assertTrue(os.path.isfile("path/file1"))
copy_file("file1", "path2/file1")
self.assertTrue(os.path.isfile("path2/file1"))
with TestAreaContext("copy/file2"):
mkdir("root/sub/path")
with open("file", "w") as f:
f.write("Hei ...")
copy_file("file", "root/sub/path/file")
self.assertTrue(os.path.isfile("root/sub/path/file"))
with open("file2", "w") as f:
f.write("Hei ...")
with pushd("root/sub/path"):
copy_file("../../../file2")
self.assertTrue(os.path.isfile("file2"))
def test_copy_file2(self):
with TestAreaContext("copy/file2"):
mkdir("rms/output")
with open("file.txt", "w") as f:
f.write("Hei")
copy_file("file.txt", "rms/output/")
self.assertTrue(os.path.isfile("rms/output/file.txt"))
def test_careful_copy_file(self):
with TestAreaContext("careful/copy/file"):
with open("file1", "w") as f:
f.write("hei")
with open("file2", "w") as f:
f.write("hallo")
careful_copy_file("file1", "file2")
with open("file2", "r") as f:
self.assertEqual("hallo", f.readline())
careful_copy_file("file1", "file3")
self.assertTrue(os.path.isfile("file3"))
class EclRunTest(ResTest):
def setUp(self):
self.ecl_config_path = os.path.dirname(
inspect.getsourcefile(Ecl100Config))
self.monkeypatch = MonkeyPatch()
def tearDown(self):
self.monkeypatch.undo()
def init_ecl100_config(self):
conf = {
"env": {
"F_UFMTENDIAN": "big",
"LM_LICENSE_FILE": "*****@*****.**",
"ARCH": "x86_64",
},
"versions": {
"2014.2": {
"scalar": {
"executable":
"/prog/ecl/grid/2014.2/bin/linux_x86_64/eclipse.exe"
},
"mpi": {
"executable":
"/prog/ecl/grid/2014.2/bin/linux_x86_64/eclipse_ilmpi.exe",
"mpirun":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044/bin64/mpirun",
"env": {
"I_MPI_ROOT":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044/",
"P4_RSHCOMMAND":
"ssh",
"LD_LIBRARY_PATH":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044/lib64:$LD_LIBRARY_PATH",
"PATH":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044/bin64:$PATH",
},
},
},
"2019.3": {
"scalar": {
"executable":
"/prog/res/ecl/grid/2019.3/bin/linux_x86_64/eclipse.exe"
},
"mpi": {
"executable":
"/prog/res/ecl/grid/2019.3/bin/linux_x86_64/eclipse_ilmpi.exe",
"mpirun":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044/bin64/mpirun",
"env": {
"I_MPI_ROOT":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044",
"P4_RSHCOMMAND":
"ssh",
"LD_LIBRARY_PATH":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044/lib64:$LD_LIBRARY_PATH",
"PATH":
"/prog/ecl/grid/tools/linux_x86_64/intel/mpi/5.0.2.044/bin64:$PATH",
},
},
},
},
}
with open("ecl100_config.yml", "w") as f:
f.write(yaml.dump(conf))
self.monkeypatch.setenv("ECL100_SITE_CONFIG", "ecl100_config.yml")
def init_flow_config(self):
version = "2018.10"
p = Popen(["flow", "--version"], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, err = p.communicate()
rc = p.returncode
if rc == 0:
version = find_version(output)
path_to_exe = find_executable("flow")
conf = {
"default_version": version,
"versions": {
version: {
"scalar": {
"executable": path_to_exe
},
}
},
}
with open("flow_config.yml", "w") as f:
f.write(yaml.dump(conf))
self.monkeypatch.setenv("FLOW_SITE_CONFIG", "flow_config.yml")
def test_make_LSB_MCPU_machine_list(self):
self.assertListEqual(
[
"host1", "host1", "host1", "host1", "host2", "host2", "host2",
"host2"
],
ecl_run.make_LSB_MCPU_machine_list("host1 4 host2 4"),
)
@tmpdir()
def test_create(self):
# This test can make do with a mock simulator; - just something executable
conf = {
"versions": {
"2014.2": {
"scalar": {
"executable": "bin/scalar_exe"
},
"mpi": {
"executable": "bin/mpi_exe",
"mpirun": "bin/mpirun"
},
}
}
}
with open("ecl100_config.yml", "w") as f:
f.write(yaml.dump(conf))
os.mkdir("bin")
self.monkeypatch.setenv("ECL100_SITE_CONFIG", "ecl100_config.yml")
for f in ["scalar_exe", "mpi_exe", "mpirun"]:
fname = os.path.join("bin", f)
with open(fname, "w") as fh:
fh.write("This is an exectable ...")
os.chmod(fname, stat.S_IEXEC)
with open("ECLIPSE.DATA", "w") as f:
f.write("Mock eclipse data file")
ecl_config = Ecl100Config()
sim = ecl_config.sim("2014.2")
mpi_sim = ecl_config.mpi_sim("2014.2")
ecl_run = EclRun("ECLIPSE.DATA", sim)
self.assertEqual(ecl_run.runPath(), os.getcwd())
os.mkdir("path")
with open("path/ECLIPSE.DATA", "w") as f:
f.write("Mock eclipse data file")
ecl_run = EclRun("path/ECLIPSE.DATA", sim)
self.assertEqual(ecl_run.runPath(), os.path.join(os.getcwd(), "path"))
self.assertEqual(ecl_run.baseName(), "ECLIPSE")
self.assertEqual(1, ecl_run.numCpu())
# invalid number of CPU
with self.assertRaises(ValueError):
ecl_run = EclRun("path/ECLIPSE.DATA", sim, num_cpu="xxx")
ecl_run = EclRun("path/ECLIPSE.DATA", mpi_sim, num_cpu="10")
self.assertEqual(10, ecl_run.numCpu())
# Missing datafile
with self.assertRaises(IOError):
ecl_run = EclRun("DOES/NOT/EXIST", mpi_sim, num_cpu="10")
@pytest.mark.xfail(
reason="Finding a version on Komodo of flow that is not OPM-flow")
@flow_installed
@tmpdir()
def test_flow(self):
self.init_flow_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1_ERROR.DATA"),
"SPE1_ERROR.DATA",
)
flow_config = FlowConfig()
sim = flow_config.sim()
flow_run = EclRun("SPE1.DATA", sim)
flow_run.runEclipse()
run(flow_config, ["SPE1.DATA"])
flow_run = EclRun("SPE1_ERROR.DATA", sim)
with self.assertRaises(Exception):
flow_run.runEclipse()
run(flow_config, ["SPE1_ERROR.DATA", "--ignore-errors"])
# Invalid version
with self.assertRaises(Exception):
run(flow_config, ["SPE1.DATA", "--version=no/such/version"])
@tmpdir()
def test_running_flow_given_env_config_can_still_read_parent_env(self):
version = "1111.11"
# create a script that prints env vars ENV1 and ENV2 to a file
with open("flow", "w") as f:
f.write("#!/bin/bash\n")
f.write("echo $ENV1 > out.txt\n")
f.write("echo $ENV2 >> out.txt\n")
executable = os.path.join(os.getcwd(), "flow")
os.chmod(executable, 0o777)
# create a flow_config.yml with environment extension ENV2
conf = {
"default_version": version,
"versions": {
version: {
"scalar": {
"executable": executable,
"env": {
"ENV2": "VAL2"
}
},
}
},
}
with open("flow_config.yml", "w") as f:
f.write(yaml.dump(conf))
# set the environment variable ENV1
self.monkeypatch.setenv("ENV1", "VAL1")
self.monkeypatch.setenv("FLOW_SITE_CONFIG", "flow_config.yml")
with open("DUMMY.DATA", "w") as f:
f.write("dummy")
with open("DUMMY.PRT", "w") as f:
f.write("Errors 0\n")
f.write("Bugs 0\n")
# run the script
flow_config = FlowConfig()
sim = flow_config.sim()
flow_run = EclRun("DUMMY.DATA", sim)
flow_run.runEclipse()
# assert that the script was able to read both the variables correctly
with open("out.txt") as f:
lines = f.readlines()
self.assertEqual(len(lines), 2)
self.assertEqual(lines[0].strip(), "VAL1")
self.assertEqual(lines[1].strip(), "VAL2")
@tmpdir()
def test_running_flow_given_no_env_config_can_still_read_parent_env(self):
version = "1111.11"
# create a script that prints env vars ENV1 and ENV2 to a file
with open("flow", "w") as f:
f.write("#!/bin/bash\n")
f.write("echo $ENV1 > out.txt\n")
f.write("echo $ENV2 >> out.txt\n")
executable = os.path.join(os.getcwd(), "flow")
os.chmod(executable, 0o777)
# create a flow_config.yml with environment extension ENV2
conf = {
"default_version": version,
"versions": {
version: {
"scalar": {
"executable": executable
},
}
},
}
with open("flow_config.yml", "w") as f:
f.write(yaml.dump(conf))
# set the environment variable ENV1
self.monkeypatch.setenv("ENV1", "VAL1")
self.monkeypatch.setenv("ENV2", "VAL2")
self.monkeypatch.setenv("FLOW_SITE_CONFIG", "flow_config.yml")
with open("DUMMY.DATA", "w") as f:
f.write("dummy")
with open("DUMMY.PRT", "w") as f:
f.write("Errors 0\n")
f.write("Bugs 0\n")
# run the script
flow_config = FlowConfig()
sim = flow_config.sim()
flow_run = EclRun("DUMMY.DATA", sim)
flow_run.runEclipse()
# assert that the script was able to read both the variables correctly
with open("out.txt") as f:
lines = f.readlines()
self.assertEqual(len(lines), 2)
self.assertEqual(lines[0].strip(), "VAL1")
self.assertEqual(lines[1].strip(), "VAL2")
@tmpdir()
def test_running_flow_given_env_variables_with_same_name_as_parent_env_variables_will_overwrite(
self, ):
version = "1111.11"
# create a script that prints env vars ENV1 and ENV2 to a file
with open("flow", "w") as f:
f.write("#!/bin/bash\n")
f.write("echo $ENV1 > out.txt\n")
f.write("echo $ENV2 >> out.txt\n")
executable = os.path.join(os.getcwd(), "flow")
os.chmod(executable, 0o777)
# create a flow_config.yml with environment extension ENV2
conf = {
"default_version": version,
"versions": {
version: {
"scalar": {
"executable": executable,
"env": {
"ENV1": "OVERWRITTEN1",
"ENV2": "OVERWRITTEN2"
},
},
}
},
}
with open("flow_config.yml", "w") as f:
f.write(yaml.dump(conf))
# set the environment variable ENV1
self.monkeypatch.setenv("ENV1", "VAL1")
self.monkeypatch.setenv("ENV2", "VAL2")
self.monkeypatch.setenv("FLOW_SITE_CONFIG", "flow_config.yml")
with open("DUMMY.DATA", "w") as f:
f.write("dummy")
with open("DUMMY.PRT", "w") as f:
f.write("Errors 0\n")
f.write("Bugs 0\n")
# run the script
flow_config = FlowConfig()
sim = flow_config.sim()
flow_run = EclRun("DUMMY.DATA", sim)
flow_run.runEclipse()
# assert that the script was able to read both the variables correctly
with open("out.txt") as f:
lines = f.readlines()
self.assertEqual(len(lines), 2)
self.assertEqual(lines[0].strip(), "OVERWRITTEN1")
self.assertEqual(lines[1].strip(), "OVERWRITTEN2")
@tmpdir()
@pytest.mark.equinor_test
def test_run(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
sim = ecl_config.sim("2014.2")
ecl_run = EclRun("SPE1.DATA", sim)
ecl_run.runEclipse()
ok_path = os.path.join(ecl_run.runPath(),
"{}.OK".format(ecl_run.baseName()))
log_path = os.path.join(ecl_run.runPath(),
"{}.LOG".format(ecl_run.baseName()))
self.assertTrue(os.path.isfile(ok_path))
self.assertTrue(os.path.isfile(log_path))
self.assertTrue(os.path.getsize(log_path) > 0)
errors = ecl_run.parseErrors()
self.assertEqual(0, len(errors))
# Monkey patching the ecl_run to use an executable which
# will fail with exit(1); don't think Eclipse actually
# fails with exit(1) - but let us at least be prepared
# when/if it does.
ecl_run.sim.executable = os.path.join(
self.SOURCE_ROOT, "tests/libres_tests/res/fm/ecl_run_fail")
with self.assertRaises(Exception):
ecl_run.runEclipse()
@tmpdir()
@pytest.mark.equinor_test
def test_run_new_log_file(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
sim = ecl_config.sim("2019.3")
ecl_run = EclRun("SPE1.DATA", sim)
ecl_run.runEclipse()
ok_path = os.path.join(ecl_run.runPath(),
"{}.OK".format(ecl_run.baseName()))
log_path = os.path.join(ecl_run.runPath(),
"{}.OUT".format(ecl_run.baseName()))
self.assertTrue(os.path.isfile(ok_path))
self.assertTrue(os.path.isfile(log_path))
self.assertTrue(os.path.getsize(log_path) > 0)
errors = ecl_run.parseErrors()
self.assertEqual(0, len(errors))
# Monkey patching the ecl_run to use an executable which
# will fail with exit(1); don't think Eclipse actually
# fails with exit(1) - but let us at least be prepared
# when/if it does.
ecl_run.sim.executable = os.path.join(
self.SOURCE_ROOT, "tests/libres_tests/res/fm/ecl_run_fail")
with self.assertRaises(Exception):
ecl_run.runEclipse()
@pytest.mark.equinor_test
@tmpdir()
def test_run_api(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
run(ecl_config, ["SPE1.DATA", "--version=2014.2"])
self.assertTrue(os.path.isfile("SPE1.DATA"))
@pytest.mark.equinor_test
@tmpdir()
def test_failed_run(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1_ERROR.DATA"),
"SPE1_ERROR.DATA",
)
ecl_config = Ecl100Config()
sim = ecl_config.sim("2014.2")
ecl_run = EclRun("SPE1_ERROR", sim)
with self.assertRaises(Exception):
ecl_run.runEclipse()
try:
ecl_run.runEclipse()
except Exception as e:
self.assertTrue("ERROR" in str(e))
@pytest.mark.equinor_test
@tmpdir()
def test_failed_run_OK(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1_ERROR.DATA"),
"SPE1_ERROR.DATA",
)
ecl_config = Ecl100Config()
run(ecl_config, ["SPE1_ERROR", "--version=2014.2", "--ignore-errors"])
# Monkey patching the ecl_run to use an executable which will fail with exit(1),
# in the nocheck mode that should also be OK.
sim = ecl_config.sim("2014.2")
ecl_run = EclRun("SPE1_ERROR", sim, check_status=False)
ecl_run.sim.executable = os.path.join(
self.SOURCE_ROOT, "tests/libres_tests/res/fm/ecl_run_fail")
ecl_run.runEclipse()
@pytest.mark.equinor_test
@tmpdir()
def test_mpi_run(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT,
"local/eclipse/SPE1_PARALLELL.DATA"),
"SPE1_PARALLELL.DATA",
)
ecl_config = Ecl100Config()
run(ecl_config,
["SPE1_PARALLELL.DATA", "--version=2014.2", "--num-cpu=2"])
self.assertTrue(os.path.isfile("SPE1_PARALLELL.LOG"))
self.assertTrue(os.path.getsize("SPE1_PARALLELL.LOG") > 0)
@pytest.mark.equinor_test
@tmpdir()
def test_summary_block(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
sim = ecl_config.sim("2014.2")
ecl_run = EclRun("SPE1.DATA", sim)
ret_value = ecl_run.summary_block()
self.assertTrue(ret_value is None)
ecl_run.runEclipse()
ecl_sum = ecl_run.summary_block()
self.assertTrue(isinstance(ecl_sum, EclSum))
@pytest.mark.equinor_test
@tmpdir()
def test_check(self):
full_case = os.path.join(self.TESTDATA_ROOT,
"Equinor/ECLIPSE/Gurbat/ECLIPSE")
short_case = os.path.join(self.TESTDATA_ROOT,
"Equinor/ECLIPSE/ShortSummary/ECLIPSE")
failed_case = os.path.join(
self.SOURCE_ROOT,
"test-data/Equinor/ECLIPSE/SummaryFail/NOR-2013A_R002_1208-0",
)
with self.assertRaises(IOError):
self.assertTrue(EclRun.checkCase(full_case, failed_case))
with self.assertRaises(IOError):
self.assertTrue(EclRun.checkCase(full_case, "DOES-NOT-EXIST"))
with self.assertRaises(IOError):
self.assertTrue(EclRun.checkCase("DOES-NOT-EXIST", full_case))
with self.assertRaises(ValueError):
EclRun.checkCase(full_case, short_case)
self.assertTrue(not os.path.isfile("CHECK_ECLIPSE_RUN.OK"))
self.assertTrue(EclRun.checkCase(full_case, full_case))
self.assertTrue(os.path.isfile("CHECK_ECLIPSE_RUN.OK"))
os.remove("CHECK_ECLIPSE_RUN.OK")
self.assertTrue(EclRun.checkCase(
short_case, full_case)) # Simulation is longer than refcase - OK
self.assertTrue(os.path.isfile("CHECK_ECLIPSE_RUN.OK"))
@pytest.mark.equinor_test
@tmpdir()
def test_error_parse(self):
self.init_ecl100_config()
shutil.copy(
os.path.join(self.TESTDATA_ROOT, "local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
prt_file = os.path.join(self.TESTDATA_ROOT,
"local/eclipse/parse/ERROR.PRT")
shutil.copy(prt_file, "SPE1.PRT")
ecl_config = Ecl100Config()
sim = ecl_config.sim("2014.2")
ecl_run = EclRun("SPE1.DATA", sim)
error_list = ecl_run.parseErrors()
self.assertEqual(len(error_list), 2)
# NB: The ugly white space in the error0 literal is actually part of
# the string we are matching; i.e. it must be retained.
error0 = """ @-- ERROR AT TIME 0.0 DAYS ( 1-JAN-0):
@ UNABLE TO OPEN INCLUDED FILE
@ /private/joaho/ERT/git/Gurbat/XXexample_grid_sim.GRDECL
@ SYSTEM ERROR CODE IS 29 """
error1 = """ @-- ERROR AT TIME 0.0 DAYS ( 1-JAN-0):
@ INCLUDE FILES MISSING. """
self.assertEqual(error_list[0], error0)
self.assertEqual(error_list[1], error1)
def test_slurm_env_parsing(self):
host_list = make_SLURM_machine_list("ws", "2")
self.assertEqual(host_list, ["ws", "ws"])
host_list = make_SLURM_machine_list("ws1,ws2", "2,3")
self.assertEqual(host_list, ["ws1", "ws1", "ws2", "ws2", "ws2"])
host_list = make_SLURM_machine_list("ws[1-3]", "1,2,3")
self.assertEqual(host_list, ["ws1", "ws2", "ws2", "ws3", "ws3", "ws3"])
host_list = make_SLURM_machine_list("ws[1,3]", "1,3")
self.assertEqual(host_list, ["ws1", "ws3", "ws3", "ws3"])
host_list = make_SLURM_machine_list("ws[1-3,6-8]", "1,2,3,1,2,3")
self.assertEqual(
host_list,
[
"ws1",
"ws2",
"ws2",
"ws3",
"ws3",
"ws3",
"ws6",
"ws7",
"ws7",
"ws8",
"ws8",
"ws8",
],
)
host_list = make_SLURM_machine_list("ws[1-3,6-8]", "2(x2),3,1,2(x2)")
self.assertEqual(
host_list,
[
"ws1",
"ws1",
"ws2",
"ws2",
"ws3",
"ws3",
"ws3",
"ws6",
"ws7",
"ws7",
"ws8",
"ws8",
],
)
host_list = make_SLURM_machine_list("ws[1-3,6],ws[7-8]",
"2(x2),3,1,2(x2)")
self.assertEqual(
host_list,
[
"ws1",
"ws1",
"ws2",
"ws2",
"ws3",
"ws3",
"ws3",
"ws6",
"ws7",
"ws7",
"ws8",
"ws8",
],
)
class ExportJoinTest(ResTest):
def setUp(self):
self.monkeypatch = MonkeyPatch()
self.monkeypatch.setenv(
"TZ", "CET"
) # The ert_statoil case was generated in CET
self.config = self.createTestPath("local/snake_oil/snake_oil.ert")
def tearDown(self):
self.monkeypatch.undo()
def test_join(self):
with ErtTestContext("python/enkf/export/export_join", self.config) as context:
dumpDesignMatrix("DesignMatrix.txt")
ert = context.getErt()
summary_data = SummaryCollector.loadAllSummaryData(ert, "default_1")
gen_kw_data = GenKwCollector.loadAllGenKwData(ert, "default_1")
misfit = MisfitCollector.loadAllMisfitData(ert, "default_1")
dm = DesignMatrixReader.loadDesignMatrix("DesignMatrix.txt")
result = summary_data.join(gen_kw_data, how="inner")
result = result.join(misfit, how="inner")
result = result.join(dm, how="inner")
first_date = "2010-01-10"
last_date = "2015-06-23"
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][0][first_date], 3.947766
)
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][first_date], 4.206698
)
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][last_date], 4.206698
)
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][first_date], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][first_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][first_date], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][last_date], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][last_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][last_date], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][1][last_date], 0.07)
self.assertEqual(result["EXTRA_INT_COLUMN"][1][last_date], 225)
self.assertEqual(result["EXTRA_STRING_COLUMN"][1][last_date], "OFF")
self.assertFloatEqual(result["MISFIT:FOPR"][0][last_date], 457.491003)
self.assertFloatEqual(result["MISFIT:FOPR"][24][last_date], 1630.774198)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][first_date], 468.469969)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][last_date], 468.469969)
self.assertFloatEqual(result["MISFIT:TOTAL"][24][last_date], 1714.662370)
with self.assertRaises(KeyError):
realization_13 = result.loc[60]
column_count = len(result.columns)
self.assertEqual(result.dtypes[0], numpy.float64)
self.assertEqual(result.dtypes[column_count - 1], numpy.object)
self.assertEqual(result.dtypes[column_count - 2], numpy.int64)
class EclRunTest(ResTest):
def setUp(self):
self.ecl_config_path = os.path.dirname(
inspect.getsourcefile(Ecl100Config))
self.monkeypatch = MonkeyPatch()
def tearDown(self):
self.monkeypatch.undo()
@staticmethod
def _eclrun_conf():
return {
"eclrun_env": {
"SLBSLS_LICENSE_FILE": "*****@*****.**",
"ECLPATH": "/prog/res/ecl/grid",
"PATH": "/prog/res/ecl/grid/macros",
"F_UFMTENDIAN": "big",
"LSB_JOB_ID": None,
}
}
def init_eclrun_config(self):
conf = EclRunTest._eclrun_conf()
with open("ecl100_config.yml", "w") as f:
f.write(yaml.dump(conf))
self.monkeypatch.setenv("ECL100_SITE_CONFIG", "ecl100_config.yml")
@tmpdir()
@mock.patch.dict(os.environ, {"LSB_JOBID": "some-id"})
def test_env(self):
self.init_eclrun_config()
with open("eclrun", "w") as f, open("DUMMY.DATA", "w"):
f.write("""#!/usr/bin/env python
import os
import json
with open("env.json", "w") as f:
json.dump(dict(os.environ), f)
""")
os.chmod("eclrun", os.stat("eclrun").st_mode | stat.S_IEXEC)
ecl_config = Ecl100Config()
eclrun_config = EclrunConfig(ecl_config, "2019.3")
ecl_run = EclRun("DUMMY", None, check_status=False)
with mock.patch.object(ecl_run, "_get_run_command",
mock.MagicMock(return_value="./eclrun")):
ecl_run.runEclipse(eclrun_config=eclrun_config)
with open("env.json") as f:
run_env = json.load(f)
eclrun_env = self._eclrun_conf()["eclrun_env"]
for k, v in eclrun_env.items():
if v is None:
assert k not in run_env
continue
if k == "PATH":
assert run_env[k].startswith(v)
else:
assert v == run_env[k]
@tmpdir()
@pytest.mark.equinor_test
def test_run(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
ecl_run = EclRun("SPE1.DATA", None)
ecl_run.runEclipse(eclrun_config=EclrunConfig(ecl_config, "2019.3"))
ok_path = os.path.join(ecl_run.runPath(),
"{}.OK".format(ecl_run.baseName()))
log_path = os.path.join(ecl_run.runPath(),
"{}.LOG".format(ecl_run.baseName()))
self.assertTrue(os.path.isfile(ok_path))
self.assertTrue(os.path.isfile(log_path))
self.assertTrue(os.path.getsize(log_path) > 0)
errors = ecl_run.parseErrors()
self.assertEqual(0, len(errors))
@pytest.mark.equinor_test
@tmpdir()
def test_run_api(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
run(ecl_config, ["SPE1.DATA", "--version=2019.3"])
self.assertTrue(os.path.isfile("SPE1.DATA"))
@pytest.mark.equinor_test
@tmpdir()
def test_failed_run(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1_ERROR.DATA"),
"SPE1_ERROR.DATA",
)
ecl_config = Ecl100Config()
eclrun_config = EclrunConfig(ecl_config, "2019.3")
ecl_run = EclRun("SPE1_ERROR", None)
with self.assertRaises(Exception) as error_context:
ecl_run.runEclipse(eclrun_config=eclrun_config)
self.assertIn("ERROR", str(error_context.exception))
@pytest.mark.equinor_test
@tmpdir()
def test_failed_run_OK(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1_ERROR.DATA"),
"SPE1_ERROR.DATA",
)
ecl_config = Ecl100Config()
run(ecl_config, ["SPE1_ERROR", "--version=2019.3", "--ignore-errors"])
@pytest.mark.equinor_test
@tmpdir()
def test_no_hdf5_output_by_default_with_ecl100(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
# check that by default .h5 file IS NOT produced
run(ecl_config, ["SPE1.DATA", "--version=2019.3"])
self.assertFalse(os.path.exists("SPE1.h5"))
@pytest.mark.equinor_test
@tmpdir()
def test_flag_to_produce_hdf5_output_with_ecl100(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
# check that with flag .h5 file IS produced
run(ecl_config,
["SPE1.DATA", "--version=2019.3", "--summary-conversion"])
self.assertTrue(os.path.exists("SPE1.h5"))
@pytest.mark.equinor_test
@tmpdir()
def test_mpi_run(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1_PARALLELL.DATA"),
"SPE1_PARALLELL.DATA",
)
ecl_config = Ecl100Config()
run(ecl_config,
["SPE1_PARALLELL.DATA", "--version=2019.3", "--num-cpu=2"])
self.assertTrue(os.path.isfile("SPE1_PARALLELL.LOG"))
self.assertTrue(os.path.getsize("SPE1_PARALLELL.LOG") > 0)
@pytest.mark.equinor_test
@tmpdir()
def test_summary_block(self):
self.init_eclrun_config()
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"test-data/local/eclipse/SPE1.DATA"),
"SPE1.DATA",
)
ecl_config = Ecl100Config()
ecl_run = EclRun("SPE1.DATA", None)
ret_value = ecl_run.summary_block()
self.assertTrue(ret_value is None)
ecl_run.runEclipse(eclrun_config=EclrunConfig(ecl_config, "2019.3"))
ecl_sum = ecl_run.summary_block()
self.assertTrue(isinstance(ecl_sum, EclSum))
@pytest.mark.equinor_test
@tmpdir()
def test_check(self):
full_case = os.path.join(self.SOURCE_ROOT,
"test-data/Equinor/ECLIPSE/Gurbat/ECLIPSE")
short_case = os.path.join(
self.SOURCE_ROOT, "test-data/Equinor/ECLIPSE/ShortSummary/ECLIPSE")
failed_case = os.path.join(
self.SOURCE_ROOT,
"test-data/Equinor/ECLIPSE/SummaryFail/NOR-2013A_R002_1208-0",
)
with self.assertRaises(IOError):
self.assertTrue(EclRun.checkCase(full_case, failed_case))
with self.assertRaises(IOError):
self.assertTrue(EclRun.checkCase(full_case, "DOES-NOT-EXIST"))
with self.assertRaises(IOError):
self.assertTrue(EclRun.checkCase("DOES-NOT-EXIST", full_case))
with self.assertRaises(ValueError):
EclRun.checkCase(full_case, short_case)
self.assertTrue(not os.path.isfile("CHECK_ECLIPSE_RUN.OK"))
self.assertTrue(EclRun.checkCase(full_case, full_case))
self.assertTrue(os.path.isfile("CHECK_ECLIPSE_RUN.OK"))
os.remove("CHECK_ECLIPSE_RUN.OK")
self.assertTrue(EclRun.checkCase(
short_case, full_case)) # Simulation is longer than refcase - OK
self.assertTrue(os.path.isfile("CHECK_ECLIPSE_RUN.OK"))
class RMSRunTest(ResTest):
def setUp(self):
self.monkeypatch = MonkeyPatch()
pass
def tearDown(self):
self.monkeypatch.undo()
def test_create(self):
with self.assertRaises(OSError):
r = RMSRun(0, "/project/does/not/exist", "workflow")
with TestAreaContext("test_create"):
os.mkdir("rms")
r = RMSRun(0, "rms", "workflow")
def test_run_class(self):
with TestAreaContext("test_run"):
with open("rms_config.yml", "w") as f:
f.write("executable: {}/bin/rms".format(os.getcwd()))
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
r = RMSRun(0,
"project",
"workflow",
run_path="run_path",
allow_no_env=True)
r.run()
# -----------------------------------------------------------------
action = {"exit_status": 1}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
r = RMSRun(0,
"project",
"workflow",
run_path="run_path",
allow_no_env=True)
with self.assertRaises(RMSRunException):
r.run()
# -----------------------------------------------------------------
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
r = RMSRun(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
allow_no_env=True,
)
with self.assertRaises(RMSRunException):
r.run()
# -----------------------------------------------------------------
action = {
"exit_status": 0,
"target_file": os.path.join(os.getcwd(), "some_file"),
}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
r = RMSRun(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
allow_no_env=True,
)
r.run()
def test_run(self):
with TestAreaContext("test_run"):
with open("rms_config.yml", "w") as f:
f.write("executable: {}/bin/rms".format(os.getcwd()))
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
res.fm.rms.run(0,
"project",
"workflow",
run_path="run_path",
allow_no_env=True)
# -----------------------------------------------------------------
action = {"exit_status": 1}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
with self.assertRaises(RMSRunException):
res.fm.rms.run(0,
"project",
"workflow",
run_path="run_path",
allow_no_env=True)
# -----------------------------------------------------------------
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
with self.assertRaises(RMSRunException):
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
allow_no_env=True,
)
# -----------------------------------------------------------------
action = {
"exit_status": 0,
"target_file": os.path.join(os.getcwd(), "some_file"),
}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
allow_no_env=True,
)
def test_rms_load_env(self):
test_bed = [
(" ", False),
("", False),
(None, False),
("SOME_VAL", True),
]
for val, carry_over in test_bed:
with TestAreaContext("test_drop_path"):
# Setup RMS project
with open("rms_config.yml", "w") as f:
json.dump(
{
"executable": os.path.realpath("bin/rms"),
},
f,
)
with open("rms_exec_env.json", "w") as f:
json.dump(
{
"RMS_TEST_VAR": val,
},
f,
)
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"),
"bin",
)
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
rms_exec = pkg_resources.resource_filename(
"ert_shared", "share/ert/forward-models/res/script/rms")
subprocess.check_call([
rms_exec,
"--run-path",
"run_path",
"0",
"--version",
"10.4",
"project",
"--import-path",
"./",
"--export-path",
"./",
"workflow",
"-a",
])
with open("run_path/env.json") as f:
env = json.load(f)
if carry_over:
self.assertIn("RMS_TEST_VAR", env)
else:
self.assertNotIn("RMS_TEST_VAR", env)
def test_rms_drop_env(self):
test_bed = [
(" ", False),
("", False),
(None, False),
("SOME_VAL", True),
]
for val, carry_over in test_bed:
with TestAreaContext("test_drop_path"):
# Setup RMS project
with open("rms_config.yml", "w") as f:
json.dump(
{
"executable": os.path.realpath("bin/rms"),
},
f,
)
with open("rms_exec_env.json", "w") as f:
json.dump(
{
"RMS_TEST_VAR": val,
},
f,
)
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"),
"bin",
)
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
rms_exec = pkg_resources.resource_filename(
"ert_shared", "share/ert/forward-models/res/script/rms")
subprocess.check_call([
rms_exec,
"--run-path",
"run_path",
"0",
"--version",
"10.4",
"project",
"--import-path",
"./",
"--export-path",
"./",
"workflow",
"-a",
])
with open("run_path/env.json") as f:
env = json.load(f)
if carry_over:
self.assertIn("RMS_TEST_VAR", env)
else:
self.assertNotIn("RMS_TEST_VAR", env)
def test_run_class_with_existing_target_file(self):
with TestAreaContext("test_run_existing_target"):
with open("rms_config.yml", "w") as f:
f.write("executable: {}/bin/rms".format(os.getcwd()))
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
target_file = os.path.join(os.getcwd(), "rms_target_file")
action = {
"exit_status": 0,
"target_file": target_file,
}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
with open(target_file, "w") as f:
f.write("This is a dummy target file")
r = RMSRun(
0,
"project",
"workflow",
run_path="run_path",
target_file=target_file,
allow_no_env=True,
)
r.run()
def test_run_wrapper(self):
with TestAreaContext("test_run"):
wrapper_file_name = f"{os.getcwd()}/bin/rms_wrapper"
with open("rms_config.yml", "w") as f:
f.write("executable: {}/bin/rms\n".format(os.getcwd()))
f.write(f"wrapper: {wrapper_file_name}")
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
with open(wrapper_file_name, "w") as f:
f.write("#!/bin/bash\n")
f.write("exec ${@:1}")
st = os.stat(wrapper_file_name)
os.chmod(wrapper_file_name, st.st_mode | stat.S_IEXEC)
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
self.monkeypatch.setenv("PATH",
f"{os.getcwd()}/bin:{os.environ['PATH']}")
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
res.fm.rms.run(0,
"project",
"workflow",
run_path="run_path",
allow_no_env=True)
# -----------------------------------------------------------------
action = {"exit_status": 1}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
with self.assertRaises(RMSRunException):
res.fm.rms.run(0,
"project",
"workflow",
run_path="run_path",
allow_no_env=True)
# -----------------------------------------------------------------
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
with self.assertRaises(RMSRunException):
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
allow_no_env=True,
)
# -----------------------------------------------------------------
action = {
"exit_status": 0,
"target_file": os.path.join(os.getcwd(), "some_file"),
}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
allow_no_env=True,
)
def test_run_version_env(self):
with TestAreaContext("test_run"):
wrapper_file_name = f"{os.getcwd()}/bin/rms_wrapper"
with open("rms_config.yml", "w") as f:
f.write(f"""\
executable: {os.getcwd()}/bin/rms
wrapper: {wrapper_file_name}
env:
10.1.3:
PATH_PREFIX: /some/path
PYTHONPATH: /some/pythonpath
""")
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
with open(wrapper_file_name, "w") as f:
f.write(
TEST_ENV_WRAPPER.format(
expected_path_prefix="/some/path",
expected_pythonpath="/some/other/pythonpath",
))
st = os.stat(wrapper_file_name)
os.chmod(wrapper_file_name, st.st_mode | stat.S_IEXEC)
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
self.monkeypatch.setenv("PATH",
f"{os.getcwd()}/bin:{os.environ['PATH']}")
self.monkeypatch.setenv("PYTHONPATH", "/some/other/pythonpath")
action = {
"exit_status": 0,
"target_file": os.path.join(os.getcwd(), "some_file"),
}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
version="10.1.3",
)
def test_run_version_env_with_user_env(self):
with TestAreaContext("test_run"):
wrapper_file_name = f"{os.getcwd()}/bin/rms_wrapper"
with open("rms_config.yml", "w") as f:
f.write(f"""\
executable: {os.getcwd()}/bin/rms
wrapper: {wrapper_file_name}
env:
10.1.3:
PATH_PREFIX: /some/path
PYTHONPATH: /some/pythonpath
""")
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
with open(wrapper_file_name, "w") as f:
f.write(
TEST_ENV_WRAPPER.format(
expected_path_prefix="/some/other/path:/some/path",
expected_pythonpath="/some/other/pythonpath",
))
with open("rms_exec_env.json", "w") as f:
f.write("""\
{
"PATH_PREFIX" : "/some/other/path",
"PYTHONPATH" : "/some/other/pythonpath"
}
""")
st = os.stat(wrapper_file_name)
os.chmod(wrapper_file_name, st.st_mode | stat.S_IEXEC)
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
self.monkeypatch.setenv("PATH",
f"{os.getcwd()}/bin:{os.environ['PATH']}")
self.monkeypatch.setenv("PYTHONPATH", "/some/other/pythonpath")
with patch.object(sys, "argv", ["rms"]):
action = {
"exit_status": 0,
"target_file": os.path.join(os.getcwd(), "some_file"),
}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
version="10.1.3",
)
def test_run_allow_no_env(self):
with TestAreaContext("test_run"):
wrapper_file_name = f"{os.getcwd()}/bin/rms_wrapper"
with open("rms_config.yml", "w") as f:
f.write(f"""\
executable: {os.getcwd()}/bin/rms
env:
10.1.3:
PATH_PREFIX: /some/path
PYTHONPATH: /some/pythonpath
""")
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
self.monkeypatch.setenv("PATH",
f"{os.getcwd()}/bin:{os.environ['PATH']}")
action = {
"exit_status": 0,
"target_file": os.path.join(os.getcwd(), "some_file"),
}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
with self.assertRaises(RMSRunException) as e:
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
version="non-existing",
)
assert "non-existing" in str(e)
res.fm.rms.run(
0,
"project",
"workflow",
run_path="run_path",
target_file="some_file",
version="non-existing",
allow_no_env=True,
)
def test_rms_job_script_parser(self):
with TestAreaContext("test_run"):
# Setup RMS project
with open("rms_config.yml", "w") as f:
json.dump(
{
"executable": os.path.realpath("bin/rms"),
"env": {
"10.1.3": {
"PATH": ""
}
},
},
f,
)
self.monkeypatch.setenv("RMS_TEST_VAR", "fdsgfdgfdsgfds")
os.mkdir("run_path")
os.mkdir("bin")
os.mkdir("project")
shutil.copy(
os.path.join(self.SOURCE_ROOT,
"tests/libres_tests/res/fm/rms"), "bin")
self.monkeypatch.setenv("RMS_SITE_CONFIG", "rms_config.yml")
action = {"exit_status": 0}
with open("run_path/action.json", "w") as f:
f.write(json.dumps(action))
rms_exec = pkg_resources.resource_filename(
"ert_shared", "share/ert/forward-models/res/script/rms")
subprocess.check_call([
rms_exec,
"--run-path",
"run_path",
"0",
"--version",
"10.1.3",
"project",
"--import-path",
"./",
"--export-path",
"./",
"workflow",
"",
])
subprocess.check_call([
rms_exec,
"--run-path",
"run_path",
"0",
"--version",
"10.1.3",
"project",
"workflow",
"-a",
])
class PluginContextTest(unittest.TestCase):
def setUp(self):
self.monkeypatch = MonkeyPatch()
def tearDown(self):
self.monkeypatch.undo()
def test_no_plugins(self):
self.monkeypatch.delenv("ERT_SITE_CONFIG", raising=False)
with ErtPluginContext(plugins=[]) as c:
with self.assertRaises(KeyError):
os.environ["ECL100_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["ECL300_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["FLOW_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["RMS_SITE_CONFIG"]
self.assertTrue(os.path.isfile(os.environ["ERT_SITE_CONFIG"]))
with open(os.environ["ERT_SITE_CONFIG"]) as f:
self.assertEqual(f.read(),
c.plugin_manager.get_site_config_content())
path = os.environ["ERT_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["ERT_SITE_CONFIG"]
self.assertFalse(os.path.isfile(path))
def test_with_plugins(self):
self.monkeypatch.delenv("ERT_SITE_CONFIG", raising=False)
# We are comparing two function calls, both of which generate a tmpdir,
# this makes sure that the same tmpdir is called on both occasions.
self.monkeypatch.setattr(tempfile, "mkdtemp",
Mock(return_value=tempfile.mkdtemp()))
with ErtPluginContext(plugins=[dummy_plugins]) as c:
with self.assertRaises(KeyError):
os.environ["ECL100_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["ECL300_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["FLOW_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["RMS_SITE_CONFIG"]
self.assertTrue(os.path.isfile(os.environ["ERT_SITE_CONFIG"]))
with open(os.environ["ERT_SITE_CONFIG"]) as f:
self.assertEqual(f.read(),
c.plugin_manager.get_site_config_content())
path = os.environ["ERT_SITE_CONFIG"]
with self.assertRaises(KeyError):
os.environ["ERT_SITE_CONFIG"]
self.assertFalse(os.path.isfile(path))
def test_already_set(self):
for var in env_vars:
self.monkeypatch.setenv(var, "TEST")
with ErtPluginContext(plugins=[dummy_plugins]):
for var in env_vars:
self.assertEqual("TEST", os.environ[var])
for var in env_vars:
self.assertEqual("TEST", os.environ[var])
