def test_realizationcomb_virt_meta():
"""Test metadata aggregation of combinations
of virtualized realizations"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
real0dir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real0 = ensemble.ScratchRealization(real0dir)
real0.load_smry(time_index="yearly", column_keys=["F*"])
real1dir = os.path.join(testdir, "data/testensemble-reek001",
"realization-1/iter-0")
real1 = ensemble.ScratchRealization(real1dir)
real1.load_smry(time_index="yearly", column_keys=["FOPT", "WOPT*"])
# Virtualized based on the loades summary vectors, which
# differ between the two realizations.
vreal0 = real0.to_virtual()
vreal1 = real1.to_virtual()
assert "WOPT" not in vreal0.get_smry_meta(column_keys="*")
assert "FOPT" in vreal0.get_smry_meta(column_keys="*")
assert "WOPT:OP_3" in vreal1.get_smry_meta(column_keys="*")
assert "WOPT:OP_3" not in vreal0.get_smry_meta(column_keys="*")
assert "FOPT" in vreal1.get_smry_meta(column_keys="*")
def test_realizationcombination_basic():
"""Basic testing of combination of two realizations
to a RealizationCombination"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
real0dir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real0 = ensemble.ScratchRealization(real0dir)
real0.load_smry(time_index="yearly", column_keys=["F*"])
real0.load_scalar("npv.txt")
real1dir = os.path.join(testdir, "data/testensemble-reek001",
"realization-1/iter-0")
real1 = ensemble.ScratchRealization(real1dir)
real1.load_smry(time_index="yearly", column_keys=["F*"])
real1.load_scalar("npv.txt")
realdiff = real0 - real1
real0.data["a_string"] = "foo_in_real_0"
real1.data["a_string"] = "foo_in_real_1"
assert "FWPR" in realdiff["unsmry--yearly"]
assert "FWL" in realdiff["parameters"]
assert realdiff["npv.txt"] == real0["npv.txt"] - real1["npv.txt"]
assert realdiff["a_string"] is None
# Combination of the same when virtualized:
vreal0 = real0.to_virtual()
vreal1 = real1.to_virtual()
scaled_vreal0 = 3 * vreal0
assert "FWPR" in scaled_vreal0["unsmry--yearly"]
assert "FWL" in scaled_vreal0["parameters"]
assert "FWL" in scaled_vreal0.parameters
assert scaled_vreal0.parameters["FWL"] == real0.parameters["FWL"] * 3
vdiff = vreal1 - vreal0
assert "FWPR" in vdiff["unsmry--yearly"]
assert "FWL" in vdiff["parameters"]
assert vdiff["npv.txt"] == real1["npv.txt"] - real0["npv.txt"]
vdiff_filtered = vdiff.to_virtual(keyfilter="parameters")
assert "parameters.txt" in vdiff_filtered.keys()
with pytest.raises((KeyError, ValueError)):
vdiff_filtered.get_df("unsmry--yearly")
vdiff_filtered2 = vdiff.to_virtual(keyfilter="unsmry--yearly")
assert "parameters.txt" not in vdiff_filtered2.keys()
assert "FWPR" in vdiff_filtered2.get_df("unsmry--yearly")
smrymeta = realdiff.get_smry_meta(["FO*"])
assert "FOPT" in smrymeta
smry_params = realdiff.get_df("unsmry--yearly", merge="parameters.txt")
assert "SORG1" in smry_params
assert "FWCT" in smry_params
def test_independent_realization(tmp="TMP"):
"""Test what we are able to load a single Eclipse run
that might have nothing to do with FMU"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
datadir = os.path.join(testdir, "data")
tmpdir = os.path.join(datadir, tmp)
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
os.mkdir(tmpdir)
# Let the directory contain only the UNSMRY and SMSPEC file
shutil.copyfile(
os.path.join(
datadir,
"testensemble-reek001/realization-2/iter-0/eclipse/" +
"model/2_R001_REEK-2.UNSMRY",
),
os.path.join(tmpdir, "2_R001_REEK-2.UNSMRY"),
)
shutil.copyfile(
os.path.join(
datadir,
"testensemble-reek001/realization-2/iter-0/eclipse/" +
"model/2_R001_REEK-2.SMSPEC",
),
os.path.join(tmpdir, "2_R001_REEK-2.SMSPEC"),
)
# This should not fail, but with a nice constructive warning to the user hinting
# about the solution
empty = ensemble.ScratchRealization(tmpdir)
assert not empty.index # The index is None in such realizations.
# This is how it must be done:
real = ensemble.ScratchRealization(tmpdir, index="999")
assert real.index == 999
# No auto-discovery here:
assert real.get_smry().empty
# Explicit discovery:
real.find_files("*UNSMRY")
assert not real.get_smry().empty
# However, we can do something with an undefined index:
noindex = ensemble.ScratchRealization(tmpdir)
noindex.find_files("*UNSMRY")
assert not real.get_smry().empty
shutil.rmtree(tmpdir)
def test_virtual_realization():
"""Test making av virtual realization from
a fresh ScratchRealization, and veryfing that the
internalized data was conserved"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
# Check deepcopy(), first prove a bad situation
vreal = real.to_virtual(deepcopy=False)
assert "parameters.txt" in real.keys()
del vreal["parameters.txt"]
# This is a bad situation:
assert "parameters.txt" not in real.keys()
# Now confirm that we can fix the bad
# situation with the default to_virtual()
real = ensemble.ScratchRealization(realdir)
vreal = real.to_virtual()
del vreal["parameters.txt"]
assert "parameters.txt" in real.keys()
real = ensemble.ScratchRealization(realdir)
vreal = real.to_virtual()
assert real.keys() == vreal.keys()
# Test appending a random dictionary betteroutput
vreal.append("betteroutput", {"NPV": 200000000, "BREAKEVEN": 8.4})
assert vreal.get_df("betteroutput")["NPV"] > 0
# Appending to a key that exists should not help
vreal.append("betteroutput", {"NPV": -300, "BREAKEVEN": 300})
assert vreal.get_df("betteroutput")["NPV"] > 0
# Unless we overwrite explicitly:
vreal.append("betteroutput", {
"NPV": -300,
"BREAKEVEN": 300
},
overwrite=True)
assert vreal.get_df("betteroutput")["NPV"] < 0
with pytest.raises(ValueError):
vreal.get_df("bogusdataname")
def test_drop():
"""Test the drop functionality, where can delete
parts of internalized data"""
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
parametercount = len(real.parameters)
real.drop("parameters", key="RMS_SEED")
assert len(real.parameters) == parametercount - 1
real.drop("parameters", keys=["L_1GO", "E_1GO"])
assert len(real.parameters) == parametercount - 3
real.drop("parameters", key="notexistingkey")
# This will go unnoticed
assert len(real.parameters) == parametercount - 3
real.load_smry(column_keys="FOPT", time_index="monthly")
datecount = len(real.get_df("unsmry--monthly"))
real.drop("unsmry--monthly", rowcontains="2000-01-01")
assert len(real.get_df("unsmry--monthly")) == datecount - 1
real.drop("parameters")
assert "parameters.txt" not in real.keys()
def test_find_files_yml():
"""Test the more exotic features of find_files
Meta-data in yaml files.
"""
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
# Setup example files with some yaml data:
findable_files = ["grid1.gri", "grid2.gri"]
for filename in findable_files:
with open(os.path.join(realdir, filename), "w") as fileh:
fileh.write("baah")
yamlfile = "." + filename + ".yml"
with open(os.path.join(realdir, yamlfile), "w") as fileh:
fileh.write(yaml.dump(dict(a=dict(x=1, y=2), b="bar")))
# Now find the gri files, and add metadata:
files_df = real.find_files("*.gri", metayaml=True)
assert "a--x" in files_df
assert "a--y" in files_df
assert "b" in files_df
assert files_df["b"].unique()[0] == "bar"
assert files_df["a--x"].astype(int).unique()[0] == 1
assert files_df["a--y"].astype(int).unique()[0] == 2
# Cleanup
for filename in findable_files:
if os.path.exists(os.path.join(realdir, filename)):
os.unlink(os.path.join(realdir, filename))
yamlfile = "." + filename + ".yml"
if os.path.exists(os.path.join(realdir, yamlfile)):
os.unlink(os.path.join(realdir, yamlfile))
def test_get_smry_dates():
"""Test date grid functionality from a virtual realization.
Already internalized summary data is needed for this"""
# First test with no data:
empty_vreal = ensemble.VirtualRealization()
with pytest.raises(ValueError):
empty_vreal.get_smry_dates()
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
real.load_smry(time_index="yearly", column_keys=["F*", "W*"])
vreal = real.to_virtual()
assert len(vreal.get_smry_dates(freq="monthly")) == 49
assert len(vreal.get_smry_dates(freq="daily")) == 1462
assert len(vreal.get_smry_dates(freq="yearly")) == 5
with pytest.raises(ValueError):
assert vreal.get_smry_dates(freq="foobar")
def test_glob_smry_keys():
"""Test the globbing function for virtual realization"""
empty_vreal = ensemble.VirtualRealization()
with pytest.raises(ValueError):
empty_vreal._glob_smry_keys("FOP*")
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
real.load_smry(time_index="yearly", column_keys=["F*", "W*"])
vreal = real.to_virtual()
assert len(vreal._glob_smry_keys("FOP*")) == 9
assert len(vreal._glob_smry_keys("FOP?")) == 3
assert len(vreal._glob_smry_keys(["FOP*"])) == 9
assert len(vreal._glob_smry_keys("WOPT:*")) == 8
assert all([x.startswith("WOPT:") for x in vreal._glob_smry_keys("WOPT:*")])
assert not vreal._glob_smry_keys("FOOBAR")
def test_virtual_fromdisk(tmp="TMP"):
"""Test retrieval of a virtualrealization that
has been dumped to disk"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
# First make a ScratchRealization to virtualize and dump:
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
# Internalize some data that we can test for afterwards
real.load_smry(time_index="yearly", column_keys=["F*"])
real.load_scalar("npv.txt")
if not os.path.exists(tmp):
os.mkdir(tmp)
# Virtualize and dump to disk:
real.to_virtual().to_disk(os.path.join(tmp, "virtreal2"), delete=True)
# Reload the virtualized realization back from disk:
vreal = ensemble.VirtualRealization("foo")
vreal.load_disk(os.path.join(tmp, "virtreal2"))
for key in vreal.keys():
if isinstance(real.get_df(key), (pd.DataFrame, dict)):
assert len(real.get_df(key)) == len(vreal.get_df(key))
else: # Scalars:
assert real.get_df(key) == vreal.get_df(key)
assert real.get_df("parameters")["FWL"] == vreal.get_df(
"parameters")["FWL"]
assert (real.get_df("unsmry--yearly").iloc[-1]["FGIP"] == vreal.get_df(
"unsmry--yearly").iloc[-1]["FGIP"])
assert real.get_df("npv.txt") == 3444
def test_virtual_todisk(tmpdir):
"""Test writing a virtual realization to disk (as a directory with files)"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
real.load_smry(time_index="yearly", column_keys=["F*"])
real.load_scalar("npv.txt")
vreal = real.to_virtual()
assert "npv.txt" in vreal.keys()
tmpdir.chdir()
with pytest.raises(IOError):
vreal.to_disk("/")
print("virtreal1")
vreal.to_disk("virtreal1", delete=True)
assert os.path.exists("virtreal1/parameters.txt")
assert os.path.exists("virtreal1/STATUS")
assert os.path.exists("virtreal1/share/results/tables/unsmry--yearly.csv")
assert os.path.exists("virtreal1/npv.txt")
def test_datenormalization():
"""Test normalization of dates, where
dates can be ensured to be on dategrid boundaries"""
# pylint: disable=import-outside-toplevel
from fmu.ensemble.realization import normalize_dates
from datetime import date
start = date(1997, 11, 5)
end = date(2020, 3, 2)
assert normalize_dates(start, end, "monthly") == (
date(1997, 11, 1),
date(2020, 4, 1),
)
assert normalize_dates(start, end, "yearly") == (date(1997, 1, 1), date(2021, 1, 1))
# Check it does not touch already aligned dates
assert normalize_dates(date(1997, 11, 1), date(2020, 4, 1), "monthly") == (
date(1997, 11, 1),
date(2020, 4, 1),
)
assert normalize_dates(date(1997, 1, 1), date(2021, 1, 1), "yearly") == (
date(1997, 1, 1),
date(2021, 1, 1),
)
# Check that we normalize correctly with get_smry():
# realization-0 here has its last summary date at 2003-01-02
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
raw = real.get_smry(column_keys="FOPT", time_index="raw")
assert str(raw.index[-1]) == "2003-01-02 00:00:00"
daily = real.get_smry(column_keys="FOPT", time_index="daily")
assert str(daily.index[-1]) == "2003-01-02"
monthly = real.get_smry(column_keys="FOPT", time_index="monthly")
assert str(monthly.index[-1]) == "2003-02-01"
yearly = real.get_smry(column_keys="FOPT", time_index="yearly")
assert str(yearly.index[-1]) == "2004-01-01"
# Check that time_index=None and time_index="raw" behaves like default
raw = real.load_smry(column_keys="FOPT", time_index="raw")
assert list(real.load_smry(column_keys="FOPT")["FOPT"].values) == list(
raw["FOPT"].values
)
assert list(
real.load_smry(column_keys="FOPT", time_index=None)["FOPT"].values
) == list(raw["FOPT"].values)
# Check that we get the same correct normalization
# with load_smry()
real.load_smry(column_keys="FOPT", time_index="raw")
assert str(real.get_df("unsmry--raw")["DATE"].iloc[-1]) == "2003-01-02 00:00:00"
real.load_smry(column_keys="FOPT", time_index="daily")
assert str(real.get_df("unsmry--daily")["DATE"].iloc[-1]) == "2003-01-02"
real.load_smry(column_keys="FOPT", time_index="monthly")
assert str(real.get_df("unsmry--monthly")["DATE"].iloc[-1]) == "2003-02-01"
real.load_smry(column_keys="FOPT", time_index="yearly")
assert str(real.get_df("unsmry--yearly")["DATE"].iloc[-1]) == "2004-01-01"
def test_get_smry():
"""Check that we can to get_smry() on virtual realizations"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
real.load_smry(time_index="yearly", column_keys=["F*"])
vreal = real.to_virtual()
monthly = vreal.get_smry(column_keys=["FOPT", "FOPR", "FGPR", "FWCT"],
time_index="monthly")
assert "FOPT" in monthly.columns
assert len(monthly) > 20
assert "FOPR" in monthly.columns
assert len(monthly) == len(monthly.dropna())
vfopt = vreal.get_smry(column_keys="FOPT", time_index="yearly")
fopt = real.get_smry(column_keys="FOPT", time_index="yearly")
assert all(vfopt == fopt)
# But note that the dtype of the index in each dataframe differs
# vfopt.index.dtype == datetime, while fopt.index.dtype == object
assert len(fopt.columns) == 1 # DATE is index (unlabeled)
dvfopt = vreal.get_smry(column_keys="FOPT", time_index="daily")
assert all(dvfopt.diff() >= 0)
# Linear interpolation should give many unique values:
assert len(dvfopt["FOPT"].unique()) == 1462
# Length is here 1462 while daily smry for the scratchrealization
# would give 1098 (one year less) - this is correct here
# since we only have yearly dates to interpolate from.
dvfopr = vreal.get_smry(column_keys="FOPR", time_index="daily")
# FOPR is bfill'ed and should not have many unique values:
assert len(dvfopr["FOPR"].unique()) == 4
# Try with custom datetimes
long_time_ago = [datetime.date(1978, 5, 6), datetime.date(1988, 5, 6)]
assert all(
vreal.get_smry(column_keys=["FOPR", "FOPT"], time_index=long_time_ago)
== 0)
before_and_after = [datetime.date(1900, 1, 1), datetime.date(2100, 1, 1)]
assert all(
vreal.get_smry(column_keys=["FOPR", "FOPT"],
time_index=before_and_after).sort_index(axis=1) ==
real.get_smry(column_keys=["FOPR", "FOPT"],
time_index=before_and_after).sort_index(axis=1))
# If you supply repeating timeindices, you get duplicates out
# (only duplicates between existing and supplied timesteps are
# removed)
repeating = [datetime.date(2002, 2, 1), datetime.date(2002, 2, 1)]
assert len(vreal.get_smry(column_keys="FOPR",
time_index=repeating)) == len(repeating)
def test_realizationcombination_basic():
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
real0dir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real0 = ensemble.ScratchRealization(real0dir)
real0.load_smry(time_index="yearly", column_keys=["F*"])
real1dir = os.path.join(testdir, "data/testensemble-reek001",
"realization-1/iter-0")
real1 = ensemble.ScratchRealization(real1dir)
real1.load_smry(time_index="yearly", column_keys=["F*"])
assert "FWPR" in ((real0 - real1)["unsmry--yearly"]).columns
assert "FWL" in ((real0 - real1)["parameters"])
def test_get_df_merge():
"""Test the merge support in get_df. Could be tricky for virtualrealizations
since not everything is dataframes"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
unsmry = real.load_smry(column_keys="*", time_index="yearly")
real.load_csv("share/results/volumes/simulator_volume_fipnum.csv")
real.load_txt("outputs.txt")
real.load_scalar("npv.txt")
vreal = real.to_virtual()
assert len(vreal.get_df("unsmry--yearly", merge="parameters").columns) == len(
unsmry.columns
) + len(real.parameters)
smryoutput = vreal.get_df("unsmry--yearly", merge="outputs")
assert "top_structure" in smryoutput.columns
paramoutput = vreal.get_df("parameters", merge="outputs")
assert "SORG1" in paramoutput
assert "top_structure" in paramoutput
output_scalar = vreal.get_df("outputs", merge="npv.txt")
assert "npv.txt" in output_scalar
assert "top_structure" in output_scalar
output_scalar = vreal.get_df("npv.txt", merge="outputs")
assert "npv.txt" in output_scalar
assert "top_structure" in output_scalar
# Try merging dataframes:
real.load_csv("share/results/volumes/simulator_volume_fipnum.csv")
# Inject a mocked dataframe to the realization:
real.data["fipnum2zone"] = pd.DataFrame(
columns=["FIPNUM", "ZONE"],
data=[
[1, "UpperReek"],
[2, "MidReek"],
[3, "LowerReek"],
[4, "UpperReek"],
[5, "MidReek"],
[6, "LowerReek"],
],
)
volframe = real.get_df("simulator_volume_fipnum", merge="fipnum2zone")
assert "ZONE" in volframe
assert "FIPNUM" in volframe
assert "STOIIP_OIL" in volframe
assert len(volframe["ZONE"].unique()) == 3
def test_datenormalization():
"""Test normalization of dates, where
dates can be ensured to be on dategrid boundaries"""
# fmu.ensemble.util.normalized_dates is also tested in test_util.py
# Check that we normalize correctly with get_smry():
# realization-0 here has its last summary date at 2003-01-02
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
raw = real.get_smry(column_keys="FOPT", time_index="raw")
assert str(raw.index[-1]) == "2003-01-02 00:00:00"
daily = real.get_smry(column_keys="FOPT", time_index="daily")
assert str(daily.index[-1]) == "2003-01-02"
monthly = real.get_smry(column_keys="FOPT", time_index="monthly")
assert str(monthly.index[-1]) == "2003-02-01"
yearly = real.get_smry(column_keys="FOPT", time_index="yearly")
assert str(yearly.index[-1]) == "2004-01-01"
weekly = real.get_smry(column_keys="FOPT", time_index="weekly")
assert str(weekly.index[-1]) == "2003-01-06" # First Monday after 2003-01-02
weekly = real.get_smry(column_keys="FOPT", time_index="W-MON")
assert str(weekly.index[-1]) == "2003-01-06" # First Monday after 2003-01-02
weekly = real.get_smry(column_keys="FOPT", time_index="W-TUE")
assert str(weekly.index[-1]) == "2003-01-07" # First Tuesday after 2003-01-02
weekly = real.get_smry(column_keys="FOPT", time_index="W-THU")
assert str(weekly.index[-1]) == "2003-01-02" # First Thursday after 2003-01-02
# Check that time_index=None and time_index="raw" behaves like default
raw = real.load_smry(column_keys="FOPT", time_index="raw")
assert list(real.load_smry(column_keys="FOPT")["FOPT"].values) == list(
raw["FOPT"].values
)
assert list(
real.load_smry(column_keys="FOPT", time_index=None)["FOPT"].values
) == list(raw["FOPT"].values)
# Check that we get the same correct normalization
# with load_smry()
real.load_smry(column_keys="FOPT", time_index="raw")
assert str(real.get_df("unsmry--raw")["DATE"].iloc[-1]) == "2003-01-02 00:00:00"
real.load_smry(column_keys="FOPT", time_index="daily")
assert str(real.get_df("unsmry--daily")["DATE"].iloc[-1]) == "2003-01-02"
real.load_smry(column_keys="FOPT", time_index="monthly")
assert str(real.get_df("unsmry--monthly")["DATE"].iloc[-1]) == "2003-02-01"
real.load_smry(column_keys="FOPT", time_index="yearly")
assert str(real.get_df("unsmry--yearly")["DATE"].iloc[-1]) == "2004-01-01"
real.load_smry(column_keys="FOPT", time_index="weekly")
assert str(real.get_df("unsmry--weekly")["DATE"].iloc[-1]) == "2003-01-06"
def test_find_files_comps():
"""Test the more exotic features of find_files
Components extracted from filenames.
"""
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
# Make some filenames we can later "find", including some problematic ones.
findable_files = [
"foo--bar--com.gri",
"foo-bar--com.gri",
"foo---bar--com.gri",
"--bar--.gri",
]
for filename in findable_files:
with open(os.path.join(realdir, filename), "w") as fileh:
fileh.write("baah")
real.find_files("*.gri")
files_df = real.files.set_index("BASENAME")
assert "COMP0" not in real.files # We are 1-based, not zero.
assert "COMP1" in real.files
assert "COMP2" in real.files
assert "COMP3" in real.files
assert "COMP4" not in real.files
assert files_df.loc["foo--bar--com.gri"]["COMP1"] == "foo"
assert files_df.loc["foo--bar--com.gri"]["COMP2"] == "bar"
assert files_df.loc["foo--bar--com.gri"]["COMP3"] == "com"
assert files_df.loc["foo-bar--com.gri"]["COMP1"] == "foo-bar"
assert files_df.loc["foo-bar--com.gri"]["COMP2"] == "com"
assert files_df.loc["foo---bar--com.gri"]["COMP1"] == "foo"
assert files_df.loc["foo---bar--com.gri"]["COMP2"] == "-bar"
assert files_df.loc["foo---bar--com.gri"]["COMP3"] == "com"
assert files_df.loc["--bar--.gri"]["COMP1"] == ""
assert files_df.loc["--bar--.gri"]["COMP2"] == "bar"
assert files_df.loc["--bar--.gri"]["COMP3"] == ""
# Cleanup
for filename in findable_files:
if os.path.exists(os.path.join(realdir, filename)):
os.unlink(os.path.join(realdir, filename))
def test_get_smry2():
"""More tests for get_smry, with more choices in
what is internalized"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
real.load_smry(time_index="yearly", column_keys=["F*"])
real.load_smry(time_index="monthly", column_keys=["F*"])
daily = real.load_smry(time_index="daily", column_keys=["F*"])
real.load_smry(time_index="raw", column_keys=["F*"])
real.load_smry(time_index=None, column_keys=["F*"])
vreal = real.to_virtual()
assert len(vreal.get_smry(column_keys="FOPR", time_index="daily")["FOPR"]) == len(
daily
)
assert len(vreal.get_smry(column_keys="FOPT", time_index="daily")["FOPT"]) == len(
daily
)
# Check that time_index=None and time_index='raw' are equal
pd.testing.assert_series_equal(
vreal.get_smry(time_index="raw")["FOPT"].reset_index(drop=True),
vreal.get_smry(time_index=None)["FOPT"].reset_index(drop=True),
)
daily_dt = vreal.get_smry_dates("daily")
# If we now ask for daily, we probably pick from 'raw' as it is
# internalized.
daily2 = vreal.get_smry(column_keys=["FOPR", "FOPT"], time_index=daily_dt)
assert len(daily2["FOPR"].unique()) == len(daily["FOPR"].unique())
assert len(daily2["FOPT"].unique()) == len(daily["FOPT"].unique())
# Check defaults handling:
monthly_length = len(vreal.get_smry(column_keys="FOPR", time_index="monthly"))
assert len(vreal.get_smry(column_keys="FOPR")) == monthly_length
alldefaults = vreal.get_smry()
assert len(alldefaults) == monthly_length
assert len(alldefaults.columns) == 49
def test_get_smry_meta():
"""Test that summary meta information is preserved through
virtualization
"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
real.load_smry(column_keys="*", time_index="yearly")
vreal = real.to_virtual()
meta = vreal.get_smry_meta()
assert "FOPT" in meta
assert "WOPR:OP_1" in meta
assert meta["FOPT"]["wgname"] is None
def test_batch():
"""Test batch processing at time of object initialization"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(
realdir,
batch=[
{"load_scalar": {"localpath": "npv.txt"}},
{"load_smry": {"column_keys": "FOPT", "time_index": "yearly"}},
{"load_smry": {"column_keys": "*", "time_index": "daily"}},
{"illegal-ignoreme": {}},
],
)
assert real.get_df("npv.txt") == 3444
assert len(real.get_df("unsmry--daily")["FOPR"]) > 2
assert len(real.get_df("unsmry--yearly")["FOPT"]) > 2
def test_get_smry_meta():
"""
Test getting eclsum metadata for single realization.
"""
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
meta = real.get_smry_meta(column_keys=["*"])
assert isinstance(meta, dict)
assert "FOPT" in meta
assert "FOPTH" in meta
assert meta["FOPT"]["unit"] == "SM3"
assert meta["FOPR"]["unit"] == "SM3/DAY"
assert meta["FOPT"]["is_total"]
assert not meta["FOPR"]["is_total"]
assert not meta["FOPT"]["is_rate"]
assert meta["FOPR"]["is_rate"]
assert not meta["FOPT"]["is_historical"]
assert meta["FOPTH"]["is_historical"]
assert meta["WOPR:OP_1"]["wgname"] == "OP_1"
assert meta["WOPR:OP_1"]["keyword"] == "WOPR"
if "wgname" in meta["FOPT"]:
# Not enforced yet to have None fields actually included
assert meta["FOPT"]["wgname"] is None
# Can create dataframes like this:
meta_df = pd.DataFrame.from_dict(meta, orient="index")
hist_keys = meta_df[meta_df["is_historical"]].index
assert all([key.split(":")[0].endswith("H") for key in hist_keys])
# When virtualizing a realization, smry data must be loaded
# for smry metadata to be conserved
real.load_smry()
vreal = real.to_virtual()
vmeta = vreal.get_smry_meta()
assert "FOPT" in vmeta
assert vmeta["FOPR"]["unit"] == "SM3/DAY"
def test_volumetric_rates():
"""Test computation of volumetric rates from cumulative vectors
This function is primarily tested in test_realization.py. Here
we only check that the wrapper in VirtualRealization is actually
working
"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
fopt = real.load_smry(column_keys="FOPT", time_index="monthly")
vreal = real.to_virtual()
fopr = vreal.get_volumetric_rates(column_keys="FOPT", time_index="monthly")
assert fopt["FOPT"].iloc[-1] == pytest.approx(fopr["FOPR"].sum())
fopr = vreal.get_volumetric_rates(
column_keys="FOPT", time_index="yearly", time_unit="months"
)
assert all(np.isfinite(fopr["FOPR"]))
def test_singlereal_ecl(tmp="TMP"):
"""Test Eclipse specific functionality for realizations"""
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
# Eclipse summary files:
assert isinstance(real.get_eclsum(), ecl.summary.EclSum)
if not os.path.exists(tmp):
os.mkdir(tmp)
real.load_smry().to_csv(os.path.join(tmp, "real0smry.csv"), index=False)
assert real.load_smry().shape == (378, 474)
# 378 dates, 470 columns + DATE column
assert real.load_smry(column_keys=["FOP*"])["FOPT"].max() > 6000000
assert real.get_smryvalues("FOPT")["FOPT"].max() > 6000000
# get_smry() should be analogue to load_smry(), but it should
# not modify the internalized dataframes!
internalized_df = real["unsmry--raw"]
fresh_df = real.get_smry(column_keys=["G*"])
assert "GGIR:OP" in fresh_df.columns
assert "GGIR:OP" not in internalized_df.columns
# Test that the internalized was not touched:
assert "GGIR:OP" not in real["unsmry--raw"].columns
assert "FOPT" in real.get_smry(column_keys=["F*"], time_index="monthly")
assert "FOPT" in real.get_smry(column_keys="F*", time_index="yearly")
assert "FOPT" in real.get_smry(column_keys="FOPT", time_index="daily")
assert "FOPT" in real.get_smry(column_keys="FOPT", time_index="raw")
# Test date functionality
assert isinstance(real.get_smry_dates(), list)
assert isinstance(real.get_smry_dates(freq="last"), list)
assert isinstance(real.get_smry_dates(freq="last")[0], datetime.date)
assert len(real.get_smry_dates()) == len(
real.get_smry_dates(freq="monthly"))
monthly = real.get_smry_dates(freq="monthly")
assert monthly[-1] > monthly[0] # end date is later than start
assert len(real.get_smry_dates(freq="yearly")) == 5
assert len(monthly) == 38
assert len(real.get_smry_dates(freq="daily")) == 1098
# start and end should be included:
assert (len(
real.get_smry_dates(start_date="2000-06-05",
end_date="2000-06-07",
freq="daily")) == 3)
# No month boundary between start and end, but we
# should have the starts and ends included
assert (len(
real.get_smry_dates(start_date="2000-06-05",
end_date="2000-06-07",
freq="monthly")) == 2)
# Date normalization should be overriden here:
assert (len(
real.get_smry_dates(
start_date="2000-06-05",
end_date="2000-06-07",
freq="monthly",
normalize=True,
)) == 2)
# Start_date and end_date at the same date should work
assert len(
real.get_smry_dates(start_date="2000-01-01",
end_date="2000-01-01")) == 1
assert (len(
real.get_smry_dates(start_date="2000-01-01",
end_date="2000-01-01",
normalize=True)) == 1)
# Check that we can go way outside the smry daterange:
assert (len(
real.get_smry_dates(start_date="1978-01-01",
end_date="2030-01-01",
freq="yearly")) == 53)
assert (len(
real.get_smry_dates(
start_date="1978-01-01",
end_date="2030-01-01",
freq="yearly",
normalize=True,
)) == 53)
assert (len(
real.get_smry_dates(
start_date="2000-06-05",
end_date="2000-06-07",
freq="raw",
normalize=True,
)) == 2)
assert (len(
real.get_smry_dates(
start_date="2000-06-05",
end_date="2000-06-07",
freq="raw",
normalize=False,
)) == 2)
# Test caching/internalization of summary files
# This should be false, since only the full localpath is in keys():
assert "unsmry--raw.csv" not in real.keys()
assert "share/results/tables/unsmry--raw.csv" in real.keys()
assert "FOPT" in real["unsmry--raw"]
with pytest.raises(ValueError):
# This does not exist before we have asked for it
"FOPT" in real["unsmry--yearly"]
def test_get_df_merge():
"""Test that we can merge on the fly using get_df()"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001", "realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
onlysmry = real.load_smry(column_keys="F*", time_index="monthly")
assert "parameters.txt" in real.data
paramcount = len(real.parameters)
smrycount = len(onlysmry.columns)
smry = real.get_df("unsmry--monthly", merge="parameters.txt")
assert len(smry.columns) == paramcount + smrycount
assert len(smry["SORG1"].unique()) == 1
# Merge with list should also work:
smry = real.get_df("unsmry--monthly", merge=["parameters.txt"])
assert len(smry.columns) == paramcount + smrycount
# Merge with empty list:
smry = real.get_df("unsmry--monthly", merge=[])
assert len(smry.columns) == smrycount
# Merge with multiple output sets:
outputs = real.load_txt("outputs.txt")
smry = real.get_df("unsmry--monthly", merge=["parameters", "outputs"])
assert len(smry.columns) == paramcount + smrycount + len(outputs)
# Merge with scalar data, and combination of scalar and dict data:
real.load_scalar("npv.txt")
smry = real.get_df("unsmry--monthly", merge="npv.txt")
assert len(smry.columns) == smrycount + 1
smry = real.get_df("unsmry--monthly", merge=["parameters.txt", "npv.txt"])
assert len(smry.columns) == smrycount + paramcount + 1
# Try merging dataframes:
real.load_csv("share/results/volumes/simulator_volume_fipnum.csv")
# Inject a mocked dataframe to the realization:
real.data["fipnum2zone"] = pd.DataFrame(
columns=["FIPNUM", "ZONE"],
data=[
[1, "UpperReek"],
[2, "MidReek"],
[3, "LowerReek"],
[4, "UpperReek"],
[5, "MidReek"],
[6, "LowerReek"],
],
)
volframe = real.get_df("simulator_volume_fipnum", merge="fipnum2zone")
assert "ZONE" in volframe
assert "FIPNUM" in volframe
assert "STOIIP_OIL" in volframe
assert len(volframe["ZONE"].unique()) == 3
scalar_dict = real.get_df("npv.txt", merge="outputs")
assert "npv.txt" in scalar_dict
assert "top_structure" in scalar_dict
# Inject a random dict and merge with:
real.data["foodict"] = dict(BAR="COM")
dframe = real.get_df("parameters", merge="foodict")
assert "BAR" in dframe
assert "SORG1" in dframe
# Merge something that is not mergeable
real.data["randtable"] = pd.DataFrame(
columns=["BARF", "ARBF"], data=[[1, 3], [2, 4]]
)
with pytest.raises(TypeError):
# pylint: disable=pointless-statement
real.get_df("parameters", merge="randtable")
with pytest.raises(pd.errors.MergeError):
# pylint: disable=pointless-statement
real.get_df("unsmry--monthly", merge="randtable")
def test_status_load(tmpdir):
"""Test loading of STATUS file with different errors in them
These files are custom text files, and can have stray error
messages in them. Robustness (i.e. no crash) is more
important than best-effort parsing. Better parsing
is left for issue #12
"""
# Mock a realization:
tmpdir.join("realization-0").mkdir()
# Test with some selected STATUS files:
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
assert real.get_df("STATUS").empty
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("\n")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
assert real.get_df("STATUS").empty
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("foo bar bogus com\n")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
assert real.get_df("STATUS").empty
# Two sucessfull jobs
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("first line always ignored\n")
status_fh.write("INCLUDE_PC : 12:40:55 .... 12:40:55 \n")
status_fh.write("ECLIPSE100_2014.2 : 12:40:55 .... 12:43:16\n")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
status = real.get_df("STATUS")
assert len(status) == 2
assert "FORWARD_MODEL" in status
assert "STARTTIME" in status
assert "ENDTIME" in status
assert "DURATION" in status
assert (status["DURATION"].values == [0, 141]).all() # in seconds
# Two sucessfull jobs, but with error string
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("first line always ignored\n")
status_fh.write("INCLUDE_PC : 12:40:55 .... 12:40:55 \n")
status_fh.write(
"ECLIPSE100_2014.2 : 12:40:55 "
".... 12:43:16 SOMEERRORSTRING_FOO\n"
)
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
status = real.get_df("STATUS")
assert len(status) == 2
assert "FORWARD_MODEL" in status
assert (status["DURATION"].values == [0, 141]).all() # in seconds
assert status["errorstring"].values[1] == "SOMEERRORSTRING_FOO"
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("first line always ignored\n")
status_fh.write("INCLUDE_PC : 12:40:55 .... 12:40:55 \n")
status_fh.write("ECLIPSE100_2014.2 : 12:40:55 ....")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
status = real.get_df("STATUS")
assert len(status) == 2
assert "FORWARD_MODEL" in status
assert status["DURATION"].values[0] == 0 # in seconds
assert np.isnan(status["DURATION"].values[1])
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("first line always ignored\n")
status_fh.write("INCLUDE_PC : 12:40:55 .... 12:40:55 \n")
status_fh.write("error message: something went really wrong")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
status = real.get_df("STATUS")
assert len(status) == 2
assert "FORWARD_MODEL" in status
assert status["DURATION"].values[0] == 0 # in seconds
# NB: The current code is not able to pick this error string
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("first line always ignored\n")
status_fh.write("INCLUDE_PC : 12:XX:55 .... 12:40:55 \n")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
status = real.get_df("STATUS")
assert len(status) == 1
assert "FORWARD_MODEL" in status
assert np.isnan(status["DURATION"].values[0])
with open(str(tmpdir.join("realization-0/STATUS")), "w") as status_fh:
status_fh.write("first line always ignored\n")
status_fh.write("INCLUDE_PC : 12:40.... 12:40:55 \n")
real = ensemble.ScratchRealization(str(tmpdir.join("realization-0")))
status = real.get_df("STATUS")
assert len(status) == 1
assert "FORWARD_MODEL" in status
assert np.isnan(status["DURATION"].values[0]) # Unsupported time syntax
def test_apply():
"""
Test the callback functionality
"""
testdir = os.path.dirname(os.path.abspath(__file__))
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
def ex_func1():
"""Example constant function"""
return pd.DataFrame(index=["1", "2"],
columns=["foo", "bar"],
data=[[1, 2], [3, 4]])
result = real.apply(ex_func1)
assert isinstance(result, pd.DataFrame)
# Apply and store the result:
real.apply(ex_func1, localpath="df-1234")
internalized_result = real.get_df("df-1234")
assert isinstance(internalized_result, pd.DataFrame)
assert (result == internalized_result).all().all()
# Check that the submitted function can utilize data from **kwargs
def ex_func2(kwargs):
"""Example function using kwargs"""
arg = kwargs["foo"]
return pd.DataFrame(index=["1", "2"],
columns=["foo", "bar"],
data=[[arg, arg], [arg, arg]])
result2 = real.apply(ex_func2, foo="bar")
assert result2.iloc[0, 0] == "bar"
# We require applied function to return only DataFrames.
def scalar_func():
"""Dummy scalar function"""
return 1
with pytest.raises(ValueError):
real.apply(scalar_func)
# The applied function should have access to the realization object:
def real_func(kwargs):
"""Example function that accesses the realization object"""
return pd.DataFrame(index=[0],
columns=["path"],
data=kwargs["realization"].runpath())
origpath = real.apply(real_func)
assert os.path.exists(origpath.iloc[0, 0])
# Do not allow supplying the realization object to apply:
with pytest.raises(ValueError):
real.apply(real_func, realization="foo")
if SKIP_FMU_TOOLS:
return
# Test if we can wrap the volumetrics-parser in fmu.tools:
# It cannot be applied directly, as we need to combine the
# realization's root directory with the relative path coming in:
def rms_vol2df(kwargs):
return volumetrics.rmsvolumetrics_txt2df(
os.path.join(kwargs["realization"].runpath(), kwargs["filename"]))
rmsvol_df = real.apply(rms_vol2df,
filename="share/results/volumes/" +
"geogrid_vol_oil_1.txt")
assert rmsvol_df["STOIIP_OIL"].sum() > 0
# Also try internalization:
real.apply(
rms_vol2df,
filename="share/results/volumes/" + "geogrid_vol_oil_1.txt",
localpath="share/results/volumes/geogrid--oil.csv",
)
assert real.get_df("geogrid--oil")["STOIIP_OIL"].sum() > 0
def test_filesystem_changes():
"""Test loading of sparse realization (random data missing)
Performed by filesystem manipulations from the original realizations.
Clean up from previous runs are attempted, and also done when it finishes.
(after a failed test run, filesystem is tainted)
"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
datadir = testdir + "/data"
tmpensname = ".deleteme_ens"
# Clean up earlier failed runs:
if os.path.exists(datadir + "/" + tmpensname):
shutil.rmtree(datadir + "/" + tmpensname, ignore_errors=True)
os.mkdir(datadir + "/" + tmpensname)
shutil.copytree(
datadir + "/testensemble-reek001/realization-0",
datadir + "/" + tmpensname + "/realization-0",
)
realdir = datadir + "/" + tmpensname + "/realization-0/iter-0"
# Load untainted realization, nothing bad should happen
real = ensemble.ScratchRealization(realdir)
# Remove SMSPEC file and reload:
shutil.move(
realdir + "/eclipse/model/2_R001_REEK-0.SMSPEC",
realdir + "/eclipse/model/2_R001_REEK-0.SMSPEC-FOOO",
)
real = ensemble.ScratchRealization(realdir) # this should go fine
# This should just return None. Logging info is okay.
assert real.get_eclsum() is None
# This should return None
assert real.get_smry_dates() is None
# This should return empty dataframe:
assert isinstance(real.load_smry(), pd.DataFrame)
assert real.load_smry().empty
assert isinstance(real.get_smry(), pd.DataFrame)
assert real.get_smry().empty
# Also move away UNSMRY and redo:
shutil.move(
realdir + "/eclipse/model/2_R001_REEK-0.UNSMRY",
realdir + "/eclipse/model/2_R001_REEK-0.UNSMRY-FOOO",
)
real = ensemble.ScratchRealization(realdir) # this should go fine
# This should just return None
assert real.get_eclsum() is None
# This should return None
assert real.get_smry_dates() is None
# This should return empty dataframe:
assert isinstance(real.load_smry(), pd.DataFrame)
assert real.load_smry().empty
# Reinstate summary data:
shutil.move(
realdir + "/eclipse/model/2_R001_REEK-0.UNSMRY-FOOO",
realdir + "/eclipse/model/2_R001_REEK-0.UNSMRY",
)
shutil.move(
realdir + "/eclipse/model/2_R001_REEK-0.SMSPEC-FOOO",
realdir + "/eclipse/model/2_R001_REEK-0.SMSPEC",
)
# Remove jobs.json, this file should not be critical
# but the status dataframe should have less information
statuscolumnswithjson = len(real.get_df("STATUS").columns)
os.remove(realdir + "/jobs.json")
real = ensemble.ScratchRealization(realdir) # this should go fine
statuscolumnswithoutjson = len(real.get_df("STATUS").columns)
assert statuscolumnswithoutjson > 0
# Check that some STATUS info is missing.
assert statuscolumnswithoutjson < statuscolumnswithjson
# Remove parameters.txt
shutil.move(realdir + "/parameters.txt", realdir + "/parameters.text")
real = ensemble.ScratchRealization(realdir)
# Should not fail
# Move it back so the realization is valid again
shutil.move(realdir + "/parameters.text", realdir + "/parameters.txt")
# Remove STATUS altogether:
shutil.move(realdir + "/STATUS", realdir + "/MOVEDSTATUS")
real = ensemble.ScratchRealization(realdir)
# Should not fail
# Try with an empty STATUS file:
fhandle = open(realdir + "/STATUS", "w")
fhandle.close()
real = ensemble.ScratchRealization(realdir)
assert real.get_df("STATUS").empty
# This demonstrates we can fool the Realization object, and
# should perhaps leads to relaxation of the requirement..
# Try with a STATUS file with error message on first job
# the situation where there is one successful job.
fhandle = open(realdir + "/STATUS", "w")
fhandle.write(
"""Current host : st-rst16-02-03/x86_64 file-server:10.14.10.238
LSF JOBID: not running LSF
COPY_FILE : 20:58:57 .... 20:59:00 EXIT: 1/Executable: /project/res/komodo/2018.02/root/etc/ERT/Config/jobs/util/script/copy_file.py failed with exit code: 1
""") # noqa
fhandle.close()
real = ensemble.ScratchRealization(realdir)
# When issue 37 is resolved, update this to 1 and check the
# error message is picked up.
assert len(real.get_df('STATUS')) == 1
fhandle = open(realdir + '/STATUS', 'w')
fhandle.write(
"""Current host : st-rst16-02-03/x86_64 file-server:10.14.10.238
LSF JOBID: not running LSF
COPY_FILE : 20:58:55 .... 20:58:57
COPY_FILE : 20:58:57 .... 20:59:00 EXIT: 1/Executable: /project/res/komodo/2018.02/root/etc/ERT/Config/jobs/util/script/copy_file.py failed with exit code: 1
""") # noqa
fhandle.close()
real = ensemble.ScratchRealization(realdir)
assert len(real.get_df("STATUS")) == 2
# Check that we have the error string picked up:
assert (
real.get_df("STATUS")["errorstring"].dropna().values[0] ==
"EXIT: 1/Executable: /project/res/komodo/2018.02/root/etc/ERT/Config/jobs/util/script/copy_file.py failed with exit code: 1"
) # noqa
# Check that we can move the Eclipse files to another place
# in the realization dir and still load summary data:
shutil.move(realdir + "/eclipse", realdir + "/eclipsedir")
real = ensemble.ScratchRealization(realdir)
# load_smry() is now the same as no UNSMRY file found,
# an empty dataframe (and there would be some logging)
assert real.load_smry().empty
# Now discover the UNSMRY file explicitly, then load_smry()
# should work.
unsmry_file = real.find_files("eclipsedir/model/*.UNSMRY")
# Non-empty dataframe:
assert not real.load_smry().empty
assert len(unsmry_file) == 1
assert isinstance(unsmry_file, pd.DataFrame)
# Test having values with spaces in parameters.txt
# Current "wanted" behaviour is to ignore the anything after
# <key><whitespace><value><whitespace><ignoreremainder>
# which is similar to ERT CSV_EXPORT1
param_file = open(realdir + "/parameters.txt", "a")
param_file.write("FOOBAR 1 2 3 4 5 6")
param_file.close()
real = ensemble.ScratchRealization(realdir)
assert real.parameters["FOOBAR"] == 1
# Clean up when finished. This often fails on network drives..
shutil.rmtree(datadir + "/" + tmpensname, ignore_errors=True)
def test_volumetric_rates():
"""Test computation of volumetric rates from cumulative vectors"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
# Should work without prior internalization:
cum_df = real.get_smry(column_keys=["F*T", "W*T*"], time_index="yearly")
vol_rate_df = real.get_volumetric_rates(column_keys=["F*T", "W*T*"],
time_index="yearly")
assert vol_rate_df.index.name == "DATE"
assert "FWCR" not in vol_rate_df # We should not compute FWCT..
assert "FOPR" in vol_rate_df
assert "FWPR" in vol_rate_df
# Also check the static method that is inside here directly
assert not real._cum_smrycol2rate("FOPR")
assert real._cum_smrycol2rate("FOPT") == "FOPR"
assert not real._cum_smrycol2rate("FWCT")
assert not real._cum_smrycol2rate("WOPR:A-H")
assert not real._cum_smrycol2rate("FOPT:FOPT:FOPT")
assert real._cum_smrycol2rate("WOPT:A-1H") == "WOPR:A-1H"
assert real._cum_smrycol2rate("WOPTH:A-2H") == "WOPRH:A-2H"
# Test that computed rates can be summed up to cumulative at end:
assert vol_rate_df["FOPR"].sum() == cum_df["FOPT"].iloc[-1]
assert vol_rate_df["FGPR"].sum() == cum_df["FGPT"].iloc[-1]
assert vol_rate_df["FWPR"].sum() == cum_df["FWPT"].iloc[-1]
# Since rates are valid forwards in time, the last
# row should have a zero, since that is the final simulated
# date for the cumulative vector
assert vol_rate_df["FOPR"].iloc[-1] == 0
# Check that we allow cumulative allocated vectors:
cumvecs = real.get_volumetric_rates(column_keys=["F*TH", "W*TH*"])
assert not cumvecs.empty
assert "FOPRH" in cumvecs
assert "WOPRH:OP_1" in cumvecs
assert real.get_volumetric_rates(column_keys="FOOBAR").empty
assert real.get_volumetric_rates(column_keys=["FOOBAR"]).empty
assert real.get_volumetric_rates(column_keys={}).empty
with pytest.raises(ValueError):
real.get_volumetric_rates(column_keys="FOPT", time_index="bogus")
mcum = real.get_smry(column_keys="FOPT", time_index="monthly")
dmcum = real.get_volumetric_rates(column_keys="FOPT", time_index="monthly")
assert dmcum["FOPR"].sum() == mcum["FOPT"].iloc[-1]
# Pick 10 **random** dates to get the volumetric rates between:
daily_dates = real.get_smry_dates(freq="daily", normalize=False)
subset_dates = np.random.choice(daily_dates, size=10, replace=False)
subset_dates.sort()
dcum = real.get_smry(column_keys="FOPT", time_index=subset_dates)
ddcum = real.get_volumetric_rates(column_keys="FOPT",
time_index=subset_dates)
assert ddcum["FOPR"].iloc[-1] == 0
# We are probably neither at the start or at the end of the production
# interval.
cumulative_error = ddcum["FOPR"].sum() - (
dcum["FOPT"].loc[subset_dates[-1]] - dcum["FOPT"].loc[subset_dates[0]])
# Give some slack, we might have done a lot of interpolation
# here.
assert cumulative_error / ddcum["FOPR"].sum() < 0.000001
# Test the time_unit feature.
vol_rate_days = real.get_volumetric_rates(column_keys=["F*T"],
time_index="yearly",
time_unit="days")
vol_rate_months = real.get_volumetric_rates(column_keys=["F*T"],
time_index="yearly",
time_unit="months")
vol_rate_years = real.get_volumetric_rates(column_keys=["F*T"],
time_index="yearly",
time_unit="years")
# Sample test on correctness.
# Fine-accuracy (wrt leap days) is not tested here:
assert vol_rate_days["FWIR"].iloc[0] * 27.9 < vol_rate_months["FWIR"].iloc[
0]
assert vol_rate_days["FWIR"].iloc[0] * 31.1 > vol_rate_months["FWIR"].iloc[
0]
assert vol_rate_months["FWIR"].iloc[0] * 12 == pytest.approx(
vol_rate_years["FWIR"].iloc[0])
assert vol_rate_days["FWIR"].iloc[0] * 364.9 < vol_rate_years["FWIR"].iloc[
0]
assert vol_rate_days["FWIR"].iloc[0] * 366.1 > vol_rate_years["FWIR"].iloc[
0]
with pytest.raises(ValueError):
real.get_volumetric_rates(column_keys=["F*T"],
time_index="yearly",
time_unit="bogus")
# Try with the random dates
dayscum = real.get_volumetric_rates(column_keys="FOPT",
time_index=subset_dates,
time_unit="days")
assert all(np.isfinite(dayscum["FOPR"]))
diffdays = pd.DataFrame(pd.to_datetime(dayscum.index)).diff().shift(-1)
dayscum["DIFFDAYS"] = [x.days for x in diffdays["DATE"]]
# Calculate cumulative production from the computed volumetric daily rates:
dayscum["FOPRcum"] = dayscum["FOPR"] * dayscum["DIFFDAYS"]
# Check that this sum is equal to FOPT between first and last date:
assert dayscum["FOPRcum"].sum() == pytest.approx(dcum["FOPT"][-1] -
dcum["FOPT"][0])
# (here we could catch an error in case we don't support leap days)
# Monthly rates between the random dates:
monthlyrates = real.get_volumetric_rates(column_keys="FOPT",
time_index=subset_dates,
time_unit="months")
assert all(np.isfinite(monthlyrates["FOPR"]))
# Total number of months in production period
delta = relativedelta(vol_rate_days.index[-1], vol_rate_days.index[0])
months = delta.years * 12 + delta.months
tworows = real.get_volumetric_rates(
column_keys="FOPT",
time_index=[vol_rate_days.index[0], vol_rate_days.index[-1]],
time_unit="months",
)
assert tworows["FOPR"].iloc[0] * months == pytest.approx(
cum_df["FOPT"].iloc[-1])
# Check for defaults and error handling:
assert not real.get_smry(column_keys=None).empty
assert not real.get_smry(column_keys=[None]).empty
assert not real.get_smry(column_keys=[None, 'WOPT:BOGUS']).empty
assert not real.get_smry(column_keys=['WOPT:BOGUS', None]).empty
column_count = len(real.get_smry())
# Columns repeatedly asked for should not be added:
assert len(real.get_smry(column_keys=[None, 'FOPT'])) == column_count
assert len(real.get_smry(column_keys=[None, 'FOPT', 'FOPT'])) \
== column_count
assert real.get_smry(column_keys=['WOPT:BOGUS']).empty
assert 'FOPT' in real.get_smry(column_keys=['WOPT:BOGUS', 'FOPT'])
def test_single_realization():
"""Test internalization of properties pertaining
to single realizations"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
realdir = os.path.join(testdir, "data/testensemble-reek001",
"realization-0/iter-0")
real = ensemble.ScratchRealization(realdir)
assert os.path.isabs(real.runpath())
assert os.path.exists(real.runpath())
assert len(real.files) == 4
assert "parameters.txt" in real.data
assert isinstance(real.parameters["RMS_SEED"], int)
assert real.parameters["RMS_SEED"] == 422851785
assert isinstance(real.parameters["MULTFLT_F1"], float)
assert isinstance(
real.load_txt("parameters.txt",
convert_numeric=False,
force_reread=True)["RMS_SEED"],
str,
)
# We have rerun load_txt on parameters, but file count
# should not increase:
assert len(real.files) == 4
with pytest.raises(IOError):
real.load_txt("nonexistingfile.txt")
# Load more data from text files:
assert "NPV" in real.load_txt("outputs.txt")
assert len(real.files) == 5
assert "outputs.txt" in real.data
assert "top_structure" in real.data["outputs.txt"]
# STATUS file
status = real.get_df("STATUS")
assert not status.empty
assert isinstance(status, pd.DataFrame)
assert "ECLIPSE" in status.loc[49, "FORWARD_MODEL"]
assert int(status.loc[49, "DURATION"]) == 141
# CSV file loading
vol_df = real.load_csv("share/results/volumes/simulator_volume_fipnum.csv")
assert len(real.files) == 6
assert isinstance(vol_df, pd.DataFrame)
assert vol_df["STOIIP_TOTAL"].sum() > 0
# Test later retrieval of cached data:
vol_df2 = real.get_df("share/results/volumes/simulator_volume_fipnum.csv")
assert vol_df2["STOIIP_TOTAL"].sum() > 0
# Test scalar import
assert "OK" in real.keys() # Imported in __init__
assert real["OK"] == "All jobs complete 22:47:54"
# NB: Trailing whitespace from the OK-file is removed.
assert isinstance(real["OK"], str)
# Check that we can "reimport" the OK file
real.load_scalar("OK", force_reread=True)
assert "OK" in real.keys() # Imported in __init__
assert real["OK"] == "All jobs complete 22:47:54"
assert isinstance(real["OK"], str)
assert len(real.files[real.files.LOCALPATH == "OK"]) == 1
real.load_scalar("npv.txt")
assert real.get_df("npv.txt") == 3444
assert real["npv.txt"] == 3444
assert isinstance(real.data["npv.txt"], (int, np.integer))
assert "npv.txt" in real.files.LOCALPATH.values
assert real.files[real.files.LOCALPATH ==
"npv.txt"]["FILETYPE"].values[0] == "txt"
real.load_scalar("emptyscalarfile")
# Activate this test when filter() is merged:
# assert real.contains('emptyfile')
assert "emptyscalarfile" in real.data
assert isinstance(real["emptyscalarfile"], str)
assert "emptyscalarfile" in real.files["LOCALPATH"].values
# Check that FULLPATH always has absolute paths
assert all([os.path.isabs(x) for x in real.files["FULLPATH"]])
with pytest.raises(IOError):
real.load_scalar("notexisting.txt")
# Test internal storage:
localpath = "share/results/volumes/simulator_volume_fipnum.csv"
assert localpath in real.data
assert isinstance(real.get_df(localpath), pd.DataFrame)
assert isinstance(real.get_df("parameters.txt"), dict)
assert isinstance(real.get_df("outputs.txt"), dict)
# Test shortcuts to the internal datastore
assert isinstance(real.get_df("simulator_volume_fipnum.csv"), pd.DataFrame)
# test without extension:
assert isinstance(
real.get_df("share/results/volumes/" + "simulator_volume_fipnum"),
pd.DataFrame)
assert isinstance(real.get_df("parameters"), dict)
# test basename and no extension:
assert isinstance(real.get_df("simulator_volume_fipnum"), pd.DataFrame)
with pytest.raises(ValueError):
real.get_df("notexisting.csv")
# Test __delitem__()
keycount = len(real.keys())
del real["parameters.txt"]
assert len(real.keys()) == keycount - 1
# At realization level, wrong filenames should throw exceptions,
# at ensemble level it is fine.
with pytest.raises(IOError):
real.load_csv("bogus.csv")