def test_main_entry_point_gen_data(monkeypatch):
run_mock = Mock()
scal_job = Mock(return_value=Mock(run=run_mock))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scal_job)
test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
sc.SpearmanCorrelationJob(ert).run(*["-t", "1.0"])
# call_args represents the clusters, we expect the snake_oil
# observations to generate this amount of them
# call_args is a call object, which itself is a tuple of args and kwargs.
# In this case, we want args, and the first element of the arguments, which
# again is a tuple containing the configuration which is a list of configs.
assert len(run_mock.call_args[0][0]) == 47, "wrong number of clusters"
cor_matrix_file = ("storage/snake_oil/ensemble/reports/"
"SpearmanCorrelationJob/correlation_matrix.csv")
pd.read_csv(cor_matrix_file, index_col=[0, 1], header=[0, 1])
clusters_file = (
"storage/snake_oil/ensemble/reports/SpearmanCorrelationJob/clusters.json"
)
with open(clusters_file) as f:
clusters = json.load(f)
assert len(clusters.keys()) == 47
def test_skip_clusters_yielding_empty_data_matrixes(monkeypatch):
def raising_scaling_job(data):
if data == {
"CALCULATE_KEYS": {
"keys": [{
"index": [88, 89],
"key": "FOPR"
}]
}
}:
raise EmptyDatasetException("foo")
scaling_mock = Mock(return_value=Mock(
**{"run.side_effect": raising_scaling_job}))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scaling_mock)
test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
job = sc.SpearmanCorrelationJob(ert)
try:
job.run(*["-t", "1.0"])
except EmptyDatasetException:
pytest.fail("EmptyDatasetException was not handled by SC job")
def test_scaling():
test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
sc.SpearmanCorrelationJob(ert).run(*["-t", "1.0"])
# assert that this arbitrarily chosen cluster gets scaled as expected
obs = ert.getObservations()["FOPR"]
for index in [13, 14, 15, 16, 17, 18, 19, 20]:
assert obs.getNode(index).getStdScaling() == 2.8284271247461903
def test_main_entry_point_syn_data(monkeypatch, facade, measured_data):
run_mock = Mock()
scal_job = Mock(return_value=Mock(run=run_mock))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scal_job)
facade_mock = Mock()
facade_mock.return_value = facade
md_mock = Mock()
md_mock.return_value = measured_data
monkeypatch.setattr(sc, "LibresFacade", facade_mock)
monkeypatch.setattr(sc, "MeasuredData", md_mock)
rc_mock = Mock()
rc_mock.model_config.getEnspath.return_value = "."
ert_mock = Mock()
ert_mock.resConfig.return_value = rc_mock
sc.SpearmanCorrelationJob(ert_mock).run(*["-t", "1.0"])
cor_matrix_file = "reports/SpearmanCorrelationJob/correlation_matrix.csv"
r1 = [1, 5, 3]
r2 = [2, 90, 2]
r3 = [3, 2, 1]
expected_corr_matrix = []
for a in zip(r1, r2, r3):
row = []
for b in zip(r1, r2, r3):
expected_corr = stats.spearmanr(a, b)
row.append(expected_corr[0])
expected_corr_matrix.append(row)
corr_matrix = pd.read_csv(cor_matrix_file, index_col=[0, 1], header=[0, 1])
assert (expected_corr_matrix == corr_matrix.values).all()
clusters_file = "reports/SpearmanCorrelationJob/clusters.json"
with open(clusters_file) as f:
cluster_reports = json.load(f)
assert len(cluster_reports) == 1
clusters = cluster_reports[0]
assert len(clusters.keys()) == 1
def test_main_entry_point_gen_data(monkeypatch):
run_mock = Mock()
scal_job = Mock(return_value=Mock(run=run_mock))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scal_job)
test_data_dir = os.path.join(TEST_DATA_DIR, "local", "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
sc.SpearmanCorrelationJob(ert).run(*["-t", "1.0"])
# call_args represents the clusters, we expect the snake_oil
# observations to generate this amount of them
# call_args is a call object, which itself is a tuple of args and kwargs.
# In this case, we want args, and the first element of the arguments, which
# again is a tuple containing the configuration which is a list of configs.
assert len(run_mock.call_args[0][0]) == 47, "wrong number of clusters"