def test_summary_obs_runtime(monkeypatch, copy_snake_oil):
"""
This is mostly a regression test, as reading SUMMARY_OBS was very slow when using
SUMMARY_OBSERVATION and not HISTORY_OBSERVATION where multiple observations
were pointing to the same response. To simulate that we load the same observations
though individual points, and also in one go. To avoid this test being flaky the
we assert on the difference in runtime. The difference in runtime we assert on is
set to 10x though it should be around 2x
"""
obs_file = pathlib.Path.cwd() / "observations" / "observations.txt"
with obs_file.open(mode="a") as fin:
fin.write(create_summary_observation())
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
facade = LibresFacade(ert)
start_time = time.time()
foprh = MeasuredData(facade,
[f"FOPR_{restart}" for restart in range(1, 201)])
summary_obs_time = time.time() - start_time
start_time = time.time()
fopr = MeasuredData(facade, ["FOPR"])
history_obs_time = time.time() - start_time
assert (fopr.data.columns.get_level_values("data_index").values.tolist() ==
foprh.data.columns.get_level_values("data_index").values.tolist())
result = foprh.get_simulated_data().values == fopr.get_simulated_data(
).values
assert np.logical_and.reduce(result).all()
assert summary_obs_time < 10 * history_obs_time
def test_get_simulated_data(input_dataframe, expected_result, monkeypatch,
facade, measured_data_setup):
measured_data_setup(input_dataframe, monkeypatch)
md = MeasuredData(facade, ["test_key"])
expected_result.columns = _set_multiindex(expected_result)
result = md.get_simulated_data()
assert result.equals(
pd.concat({"test_key": expected_result.astype(float)}, axis=1))
def _spearman_correlation(facade, obs_keys, threshold, dry_run):
"""
Collects data, performs scaling and applies scaling, assumes validated input.
"""
measured_data = MeasuredData(facade, obs_keys)
measured_data.remove_failed_realizations()
measured_data.remove_inactive_observations()
measured_data.filter_ensemble_std(1.0e-6)
simulated_data = measured_data.get_simulated_data()
correlation_matrix = _calculate_correlation_matrix(simulated_data)
clusters = _cluster_analysis(correlation_matrix, threshold)
columns = correlation_matrix.columns
# Here the clusters are joined with the key and data index
# to group the observations, the column level values are the column
# headers, where key_index is the observation key and data_index
# is a range.
data = list(
zip(
clusters,
columns.get_level_values(0),
columns.get_level_values("data_index"),
)
)
clustered_data = _cluster_data(data)
job_configs = _config_creation(clustered_data)
_output_clusters(clustered_data)
if not dry_run:
_run_scaling(facade, job_configs)