def test_summary_collector(self):
with ErtTestContext("python/enkf/export/summary_collector",
self.config) as context:
ert = context.getErt()
data = SummaryCollector.loadAllSummaryData(ert, "default_0")
self.assertFloatEqual(data["WWCT:OP2"][0]["2010-01-10"], 0.385549)
self.assertFloatEqual(data["WWCT:OP2"][24]["2010-01-10"], 0.498331)
self.assertFloatEqual(data["FOPR"][0]["2010-01-10"], 0.118963)
self.assertFloatEqual(data["FOPR"][0]["2015-06-23"], 0.133601)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_60 = data.loc[60]
data = SummaryCollector.loadAllSummaryData(
ert, "default_0", ["WWCT:OP1", "WWCT:OP2"])
self.assertFloatEqual(data["WWCT:OP1"][0]["2010-01-10"], 0.352953)
self.assertFloatEqual(data["WWCT:OP2"][0]["2010-01-10"], 0.385549)
with self.assertRaises(KeyError):
data["FOPR"]
def test_summary_collector(self):
with ErtTestContext(
"python/enkf/export/summary_collector", self.config
) as context:
ert = context.getErt()
data = SummaryCollector.loadAllSummaryData(ert, "default_0")
self.assertFloatEqual(data["WWCT:OP2"][0]["2010-01-10"], 0.385549)
self.assertFloatEqual(data["WWCT:OP2"][24]["2010-01-10"], 0.498331)
self.assertFloatEqual(data["FOPR"][0]["2010-01-10"], 0.118963)
self.assertFloatEqual(data["FOPR"][0]["2015-06-23"], 0.133601)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_60 = data.loc[60]
data = SummaryCollector.loadAllSummaryData(
ert, "default_0", ["WWCT:OP1", "WWCT:OP2"]
)
self.assertFloatEqual(data["WWCT:OP1"][0]["2010-01-10"], 0.352953)
self.assertFloatEqual(data["WWCT:OP2"][0]["2010-01-10"], 0.385549)
with self.assertRaises(KeyError):
data["FOPR"]
realization_index = 10
data = SummaryCollector.loadAllSummaryData(
ert,
"default_0",
["WWCT:OP1", "WWCT:OP2"],
realization_index=realization_index,
)
assert data.index.levels[0] == [realization_index]
assert len(data.index.levels[1]) == 200
assert list(data.columns) == ["WWCT:OP1", "WWCT:OP2"]
non_existing_realization_index = 150
with pytest.raises(IndexError):
data = SummaryCollector.loadAllSummaryData(
ert,
"default_0",
["WWCT:OP1", "WWCT:OP2"],
realization_index=non_existing_realization_index,
)
def test_join(self):
with ErtTestContext("python/enkf/export/export_join", self.config) as context:
dumpDesignMatrix("DesignMatrix.txt")
ert = context.getErt()
summary_data = SummaryCollector.loadAllSummaryData(ert, "default_1")
gen_kw_data = GenKwCollector.loadAllGenKwData(ert, "default_1")
misfit = MisfitCollector.loadAllMisfitData(ert, "default_1")
dm = DesignMatrixReader.loadDesignMatrix("DesignMatrix.txt")
result = summary_data.join(gen_kw_data, how="inner")
result = result.join(misfit, how="inner")
result = result.join(dm, how="inner")
first_date = "2010-01-10"
last_date = "2015-06-23"
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][0][first_date], 3.947766
)
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][first_date], 4.206698
)
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][last_date], 4.206698
)
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][first_date], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][first_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][first_date], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][last_date], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][last_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][last_date], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][1][last_date], 0.07)
self.assertEqual(result["EXTRA_INT_COLUMN"][1][last_date], 225)
self.assertEqual(result["EXTRA_STRING_COLUMN"][1][last_date], "OFF")
self.assertFloatEqual(result["MISFIT:FOPR"][0][last_date], 457.491003)
self.assertFloatEqual(result["MISFIT:FOPR"][24][last_date], 1630.774198)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][first_date], 468.469969)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][last_date], 468.469969)
self.assertFloatEqual(result["MISFIT:TOTAL"][24][last_date], 1714.662370)
with self.assertRaises(KeyError):
realization_13 = result.loc[60]
column_count = len(result.columns)
self.assertEqual(result.dtypes[0], numpy.float64)
self.assertEqual(result.dtypes[column_count - 1], numpy.object)
self.assertEqual(result.dtypes[column_count - 2], numpy.int64)
def gatherSummaryData(ert, case, key):
""" :rtype: pandas.DataFrame """
data = SummaryCollector.loadAllSummaryData(ert, case, [key])
if not data.empty:
data = data.reset_index()
if True in data.duplicated():
print("** Warning: The simulation data contains duplicate "
"timestamps. A possible explanation is that your "
"simulation timestep is less than a second.")
data = data.drop_duplicates()
data = data.pivot(index="Date", columns="Realization", values=key)
return data #.dropna()
def run(self, output_file, case_list=None, design_matrix_path=None, infer_iteration=True):
cases = []
if case_list is not None:
if case_list.strip() == "*":
cases = self.getAllCaseList()
else:
cases = case_list.split(",")
if case_list is None or len(cases) == 0:
cases = [self.ert().getEnkfFsManager().getCurrentFileSystem().getCaseName()]
if design_matrix_path is not None:
if not os.path.exists(design_matrix_path):
raise UserWarning("The design matrix file does not exists!")
if not os.path.isfile(design_matrix_path):
raise UserWarning("The design matrix is not a file!")
data = pandas.DataFrame()
for index, case in enumerate(cases):
case = case.strip()
if not self.ert().getEnkfFsManager().caseExists(case):
raise UserWarning("The case '%s' does not exist!" % case)
if not self.ert().getEnkfFsManager().caseHasData(case):
raise UserWarning("The case '%s' does not have any data!" % case)
if infer_iteration:
iteration_number = self.inferIterationNumber(case)
else:
iteration_number = index
case_data = GenKwCollector.loadAllGenKwData(self.ert(), case)
custom_kw_data = CustomKWCollector.loadAllCustomKWData(self.ert(), case)
if not custom_kw_data.empty:
case_data = case_data.join(custom_kw_data, how='outer')
if design_matrix_path is not None:
design_matrix_data = DesignMatrixReader.loadDesignMatrix(design_matrix_path)
if not design_matrix_data.empty:
case_data = case_data.join(design_matrix_data, how='outer')
misfit_data = MisfitCollector.loadAllMisfitData(self.ert(), case)
if not misfit_data.empty:
case_data = case_data.join(misfit_data, how='outer')
summary_data = SummaryCollector.loadAllSummaryData(self.ert(), case)
if not summary_data.empty:
case_data = case_data.join(summary_data, how='outer')
else:
case_data["Date"] = None
case_data.set_index(["Date"], append=True, inplace=True)
case_data["Iteration"] = iteration_number
case_data["Case"] = case
case_data.set_index(["Case", "Iteration"], append=True, inplace=True)
data = pandas.concat([data, case_data])
data = data.reorder_levels(["Realization", "Iteration", "Date", "Case"])
data.to_csv(output_file)
export_info = "Exported %d rows and %d columns to %s." % (len(data.index), len(data.columns), output_file)
return export_info
def load_all_summary_data(self, case_name, keys=None):
return SummaryCollector.loadAllSummaryData(self._enkf_main, case_name,
keys)