def test_skip_clusters_yielding_empty_data_matrixes(monkeypatch,
test_data_root):
def raising_scaling_job(data):
if data == {
"CALCULATE_KEYS": {
"keys": [{
"index": [88, 89],
"key": "FOPR"
}]
}
}:
raise EmptyDatasetException("foo")
scaling_mock = Mock(return_value=Mock(
**{"run.side_effect": raising_scaling_job}))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scaling_mock)
test_data_dir = os.path.join(test_data_root, "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
job = sc.SpearmanCorrelationJob(ert)
try:
job.run(*["-t", "1.0"])
except EmptyDatasetException:
pytest.fail("EmptyDatasetException was not handled by SC job")
def test_main_entry_point_gen_data(monkeypatch, test_data_root):
run_mock = Mock()
scal_job = Mock(return_value=Mock(run=run_mock))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scal_job)
test_data_dir = os.path.join(test_data_root, "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
sc.SpearmanCorrelationJob(ert).run(*["-t", "1.0"])
# call_args represents the clusters, we expect the snake_oil
# observations to generate this amount of them
# call_args is a call object, which itself is a tuple of args and kwargs.
# In this case, we want args, and the first element of the arguments, which
# again is a tuple containing the configuration which is a list of configs.
assert len(list(
run_mock.call_args)[0][0]) == 47, "wrong number of clusters"
cor_matrix_file = "reports/snake_oil/SpearmanCorrelationJob/correlation_matrix.csv"
pd.read_csv(cor_matrix_file, index_col=[0, 1], header=[0, 1])
clusters_file = "reports/snake_oil/SpearmanCorrelationJob/clusters.json"
with open(clusters_file) as f:
cluster_reports = json.load(f)
assert len(cluster_reports) == 1
clusters = cluster_reports[0]
assert len(clusters.keys()) == 47
def test_main_entry_point_syn_data(monkeypatch, facade, measured_data):
run_mock = Mock()
scal_job = Mock(return_value=Mock(run=run_mock))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scal_job)
facade_mock = Mock()
facade_mock.return_value = facade
md_mock = Mock()
md_mock.return_value = measured_data
monkeypatch.setattr(sc, "LibresFacade", facade_mock)
monkeypatch.setattr(sc, "MeasuredData", md_mock)
resconfig_mock = Mock()
resconfig_mock.model_config.getEnspath.return_value = os.path.join(
"config_path", "storage_path")
ert_mock = Mock()
fs_mock = Mock()
fs_mock.return_value.getCaseName.return_value = "user_case_name"
ert_mock.getEnkfFsManager.return_value.getCurrentFileSystem = fs_mock
ert_mock.resConfig.return_value = resconfig_mock
sc.SpearmanCorrelationJob(ert_mock).run(*["-t", "1.0"])
cor_matrix_file = os.path.join(
"config_path",
"reports",
"user_case_name",
"SpearmanCorrelationJob",
"correlation_matrix.csv",
)
r1 = [1, 5, 3]
r2 = [2, 90, 2]
r3 = [3, 2, 1]
expected_corr_matrix = []
for a in zip(r1, r2, r3):
row = []
for b in zip(r1, r2, r3):
expected_corr = stats.spearmanr(a, b)
row.append(expected_corr[0])
expected_corr_matrix.append(row)
corr_matrix = pd.read_csv(cor_matrix_file, index_col=[0, 1], header=[0, 1])
assert (expected_corr_matrix == corr_matrix.values).all()
clusters_file = os.path.join(
"config_path",
"reports",
"user_case_name",
"SpearmanCorrelationJob",
"clusters.json",
)
with open(clusters_file) as f:
cluster_reports = json.load(f)
assert len(cluster_reports) == 1
clusters = cluster_reports[0]
assert len(clusters.keys()) == 1
def test_scaling(test_data_root):
test_data_dir = os.path.join(test_data_root, "snake_oil")
shutil.copytree(test_data_dir, "test_data")
os.chdir(os.path.join("test_data"))
res_config = ResConfig("snake_oil.ert")
ert = EnKFMain(res_config)
sc.SpearmanCorrelationJob(ert).run(*["-t", "1.0"])
# assert that this arbitrarily chosen cluster gets scaled as expected
obs = ert.getObservations()["FOPR"]
for index in [13, 14, 15, 16, 17, 18, 19, 20]:
assert obs.getNode(index).getStdScaling() == 2.8284271247461903
def test_main_entry_point_syn_data(monkeypatch, facade, measured_data):
run_mock = Mock()
scal_job = Mock(return_value=Mock(run=run_mock))
monkeypatch.setattr(sc, "CorrelatedObservationsScalingJob", scal_job)
facade_mock = Mock()
facade_mock.return_value = facade
md_mock = Mock()
md_mock.return_value = measured_data
monkeypatch.setattr(sc, "LibresFacade", facade_mock)
monkeypatch.setattr(sc, "MeasuredData", md_mock)
resconfig_mock = Mock()
resconfig_mock.config_path = "config_path"
resconfig_mock.user_config_file = "user_config_file"
ert_mock = Mock(resConfig=Mock(return_value=resconfig_mock))
sc.SpearmanCorrelationJob(ert_mock).run(*["-t", "1.0"])
cor_matrix_file = ("config_path/reports/user_config_file/"
"SpearmanCorrelationJob/correlation_matrix.csv")
r1 = [1, 5, 3]
r2 = [2, 90, 2]
r3 = [3, 2, 1]
expected_corr_matrix = []
for a in zip(r1, r2, r3):
row = []
for b in zip(r1, r2, r3):
expected_corr = stats.spearmanr(a, b)
row.append(expected_corr[0])
expected_corr_matrix.append(row)
corr_matrix = pd.read_csv(cor_matrix_file, index_col=[0, 1], header=[0, 1])
assert (expected_corr_matrix == corr_matrix.values).all()
clusters_file = (
"config_path/reports/user_config_file/SpearmanCorrelationJob/clusters.json"
)
with open(clusters_file) as f:
cluster_reports = json.load(f)
assert len(cluster_reports) == 1
clusters = cluster_reports[0]
assert len(clusters.keys()) == 1