def test_roundtrip_yaml(filename, readonly_testdata_dir):
"""Test converting all test data sets in testdir into yaml and back again.
Due to yaml supporting a subset of features in the internal dataframe format
some exceptions must be hardcoded in this test function.
Also pay attention to the way the yaml parser creates LABEL data.
"""
dframe = autoparse_file(filename)[1]
# Reduce to the subset supported by yaml:
dframe = dframe[(dframe["CLASS"] == "SUMMARY_OBSERVATION")
| (dframe["CLASS"] == "BLOCK_OBSERVATION")].dropna(
axis="columns", how="all")
# Convert to YAML (really dict) format and back again:
obsdict = df2obsdict(dframe)
yaml_roundtrip_dframe = obsdict2df(obsdict)
yaml_roundtrip_dframe.set_index("CLASS", inplace=True)
dframe.set_index("CLASS", inplace=True)
if "WELL" in yaml_roundtrip_dframe:
# WELL as used in yaml is not preservable in roundtrips
del yaml_roundtrip_dframe["WELL"]
if "WELL" in dframe:
del dframe["WELL"]
# print(yaml_roundtrip_dframe)
# print(dframe)
pd.testing.assert_frame_equal(
yaml_roundtrip_dframe.sort_index(axis="columns").sort_values("LABEL"),
dframe.sort_index(axis="columns").sort_values("LABEL"),
check_like=True,
)
def test_ertobs2df(string, expected):
"""Test converting all the way from ERT observation format to a Pandas
Dataframe works as expected (this includes many of the other functions
that are also tested individually)"""
dframe = ertobs2df(string)
pd.testing.assert_frame_equal(dframe.sort_index(axis=1),
expected.sort_index(axis=1),
check_dtype=False)
pd.testing.assert_frame_equal(
ertobs2df(df2ertobs(dframe)).sort_index(axis=1),
dframe.sort_index(axis=1))
# Round-trip test via yaml:
if "DATE" not in expected:
return
round_trip_yaml_dframe = obsdict2df(df2obsdict(dframe))
pd.testing.assert_frame_equal(round_trip_yaml_dframe.sort_index(axis=1),
dframe.sort_index(axis=1))
def dump_results(
dframe, csvfile=None, yamlfile=None, resinsightfile=None, ertfile=None
):
"""Dump dataframe with ERT observations to CSV and/or YML
format to disk. Writes to stdout if filenames are "-". Skips
export if filenames are empty or None.
Args:
dframe (pd.DataFrame)
csvfile (str): Filename
yamlfile (str): Filename
resinsightfile (str): Filename
ertfile (str): Filename
"""
if not (csvfile or yamlfile or resinsightfile or ertfile):
logger.warning("No output filenames provided")
if csvfile:
if csvfile != __MAGIC_STDOUT__:
logger.info("Writing observations as CSV to %s", csvfile)
dframe.to_csv(csvfile, index=False)
else:
# Ignore pipe errors when writing to stdout:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
dframe.to_csv(sys.stdout, index=False)
if yamlfile and yamlfile:
obs_dict_for_yaml = df2obsdict(dframe)
if not obs_dict_for_yaml and not dframe.empty:
logger.error("None of your observations are supported in YAML")
yaml_str = yaml.safe_dump(obs_dict_for_yaml)
if yamlfile != __MAGIC_STDOUT__:
logger.info(
"Writing observations in YAML (webviz) format to file: %s",
resinsightfile,
)
with open(yamlfile, "w") as f_handle:
f_handle.write(yaml_str)
else:
print(yaml_str)
if resinsightfile:
ri_dframe = df2resinsight_df(dframe)
if resinsightfile != __MAGIC_STDOUT__:
logger.info(
"Writing observations in ResInsight format to CSV-file: %s",
resinsightfile,
)
ri_dframe.to_csv(resinsightfile, index=False, sep=";")
else:
# Ignore pipe errors when writing to stdout:
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
ri_dframe.to_csv(sys.stdout, index=False, sep=";")
if ertfile:
ertobs_str = df2ertobs(dframe)
if ertfile != __MAGIC_STDOUT__:
with open(ertfile, "w") as f_handle:
logger.info("Writing ERT observation format to %s", ertfile)
f_handle.write(ertobs_str)
else:
print(ertobs_str)
def test_df2obsdict(obs_df, expected_dict):
"""Test converting from dataframe representation to the dictionary
representation designed for yaml output"""
assert df2obsdict(obs_df) == expected_dict