def test_gen_kw_collector(self):
with ErtTestContext("python/enkf/export/gen_kw_collector", self.config) as context:
ert = context.getErt()
data = GenKwCollector.loadAllGenKwData(ert, "default")
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][0], 0.018466)
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][24], 0.221049)
self.assertFloatEqual(data["GRID_PARAMS:MULTPV3"][0], 2.227307)
self.assertFloatEqual(data["GRID_PARAMS:MULTPV3"][12], 5.899703)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][0], -3.008949)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][24], -1.051446)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_60 = data.loc[60]
data = GenKwCollector.loadAllGenKwData(ert, "default", ["FLUID_PARAMS:SGCR", "LOG10_MULTFLT:F3"])
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][0], 0.018466)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][0], -3.008949)
with self.assertRaises(KeyError):
data["GRID_PARAMS:MULTPV3"]
def test_gen_kw_collector(self):
with ErtTestContext("python/enkf/export/gen_kw_collector", self.config) as context:
ert = context.getErt()
data = GenKwCollector.loadAllGenKwData(ert, "default_0")
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0], 0.047517)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][24], 0.160907)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0], 0.054539)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][12], 0.057807)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_60 = data.loc[60]
data = GenKwCollector.loadAllGenKwData(
ert, "default_0", ["SNAKE_OIL_PARAM:OP1_PERSISTENCE", "SNAKE_OIL_PARAM:OP1_OFFSET"]
)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0], 0.047517)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0], 0.054539)
with self.assertRaises(KeyError):
data["SNAKE_OIL_PARAM:OP1_DIVERGENCE_SCALE"]
def test_gen_kw_collector(self):
with ErtTestContext("python/enkf/export/gen_kw_collector",
self.config) as context:
ert = context.getErt()
data = GenKwCollector.loadAllGenKwData(ert, "default")
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][0], 0.018466)
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][24], 0.221049)
self.assertFloatEqual(data["GRID_PARAMS:MULTPV3"][0], 2.227307)
self.assertFloatEqual(data["GRID_PARAMS:MULTPV3"][12], 5.899703)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][0], -3.008949)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][24], -1.051446)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_60 = data.loc[60]
data = GenKwCollector.loadAllGenKwData(
ert, "default", ["FLUID_PARAMS:SGCR", "LOG10_MULTFLT:F3"])
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][0], 0.018466)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][0], -3.008949)
with self.assertRaises(KeyError):
data["GRID_PARAMS:MULTPV3"]
def test_gen_kw_collector(self):
with ErtTestContext("python/enkf/export/gen_kw_collector", self.config) as context:
ert = context.getErt()
data = GenKwCollector.loadAllGenKwData(ert, "default")
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][0], 0.295136)
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][24], 0.177833)
self.assertFloatEqual(data["GRID_PARAMS:MULTPV3"][0], 0.423297)
self.assertFloatEqual(data["GRID_PARAMS:MULTPV3"][12], 2.278845)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][0], -2.742916)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][24], -3.459867)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_21 = data.loc[21]
data = GenKwCollector.loadAllGenKwData(ert, "default", ["FLUID_PARAMS:SGCR", "LOG10_MULTFLT:F3"])
self.assertFloatEqual(data["FLUID_PARAMS:SGCR"][0], 0.295136)
self.assertFloatEqual(data["LOG10_MULTFLT:F3"][0], -2.742916)
with self.assertRaises(KeyError):
data["GRID_PARAMS:MULTPV3"]
def test_gen_kw_collector(self):
with ErtTestContext("python/enkf/export/gen_kw_collector",
self.config) as context:
ert = context.getErt()
data = GenKwCollector.loadAllGenKwData(ert, "default_0")
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0],
0.047517)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][24],
0.160907)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0],
0.054539)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][12],
0.057807)
realization_20 = data.loc[20]
with self.assertRaises(KeyError):
realization_60 = data.loc[60]
data = GenKwCollector.loadAllGenKwData(ert, "default_0", [
"SNAKE_OIL_PARAM:OP1_PERSISTENCE", "SNAKE_OIL_PARAM:OP1_OFFSET"
])
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0],
0.047517)
self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0],
0.054539)
with self.assertRaises(KeyError):
data["SNAKE_OIL_PARAM:OP1_DIVERGENCE_SCALE"]
def test_smoother(self):
test_config = self.createTestPath("local/custom_kw/mini_config")
with ErtShellTestContext("python/ertshell/smoother", test_config) as shell:
shell.invokeCommand("case select test_run")
self.assertTrue(shell.invokeCommand("smoother update test_run_update"))
shell.invokeCommand("case select test_run_update")
ert = shell.shellContext().ert()
data = GenKwCollector.loadAllGenKwData(ert, "test_run", keys=["PERLIN_PARAM:SCALE"])
update_data = GenKwCollector.loadAllGenKwData(ert, "test_run_update", keys=["PERLIN_PARAM:SCALE"])
self.assertTrue(data["PERLIN_PARAM:SCALE"].std() > update_data["PERLIN_PARAM:SCALE"].std())
def test_join(self):
with ErtTestContext("python/enkf/export/export_join",
self.config) as context:
dumpDesignMatrix("DesignMatrix.txt")
ert = context.getErt()
summary_data = SummaryCollector.loadAllSummaryData(ert, "default")
gen_kw_data = GenKwCollector.loadAllGenKwData(ert, "default")
misfit = MisfitCollector.loadAllMisfitData(ert, "default")
dm = DesignMatrixReader.loadDesignMatrix("DesignMatrix.txt")
result = summary_data.join(gen_kw_data, how='inner')
result = result.join(misfit, how='inner')
result = result.join(dm, how='inner')
self.assertFloatEqual(result["FLUID_PARAMS:SGCR"][0]["2000-02-01"],
0.018466)
self.assertFloatEqual(
result["FLUID_PARAMS:SGCR"][24]["2000-02-01"], 0.221049)
self.assertFloatEqual(
result["FLUID_PARAMS:SGCR"][24]["2004-12-01"], 0.221049)
self.assertFloatEqual(
result["EXTRA_FLOAT_COLUMN"][0]["2000-02-01"], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0]["2000-02-01"], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0]["2000-02-01"],
"ON")
self.assertFloatEqual(
result["EXTRA_FLOAT_COLUMN"][0]["2004-12-01"], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0]["2004-12-01"], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0]["2004-12-01"],
"ON")
self.assertFloatEqual(
result["EXTRA_FLOAT_COLUMN"][1]["2004-12-01"], 0.07)
self.assertEqual(result["EXTRA_INT_COLUMN"][1]["2004-12-01"], 225)
self.assertEqual(result["EXTRA_STRING_COLUMN"][1]["2004-12-01"],
"OFF")
self.assertFloatEqual(result["MISFIT:WWCT:OP_2"][0]["2004-12-01"],
0.617793)
self.assertFloatEqual(result["MISFIT:WWCT:OP_2"][24]["2004-12-01"],
0.256436)
self.assertFloatEqual(result["MISFIT:TOTAL"][0]["2000-02-01"],
7236.322836)
self.assertFloatEqual(result["MISFIT:TOTAL"][0]["2004-12-01"],
7236.322836)
self.assertFloatEqual(result["MISFIT:TOTAL"][24]["2004-12-01"],
2261.726621)
with self.assertRaises(KeyError):
realization_13 = result.loc[60]
column_count = len(result.columns)
self.assertEqual(result.dtypes[0], numpy.float64)
self.assertEqual(result.dtypes[column_count - 1], numpy.object)
self.assertEqual(result.dtypes[column_count - 2], numpy.int64)
def run(self, output_file, case_list=None, design_matrix_path=None, infer_iteration=True):
cases = []
if case_list is not None:
if case_list.strip() == "*":
cases = self.getAllCaseList()
else:
cases = case_list.split(",")
if case_list is None or len(cases) == 0:
cases = [self.ert().getEnkfFsManager().getCurrentFileSystem().getCaseName()]
if design_matrix_path is not None:
if not os.path.exists(design_matrix_path):
raise UserWarning("The design matrix file does not exists!")
if not os.path.isfile(design_matrix_path):
raise UserWarning("The design matrix is not a file!")
data = pandas.DataFrame()
for index, case in enumerate(cases):
case = case.strip()
if not self.ert().getEnkfFsManager().caseExists(case):
raise UserWarning("The case '%s' does not exist!" % case)
if not self.ert().getEnkfFsManager().caseHasData(case):
raise UserWarning("The case '%s' does not have any data!" % case)
if infer_iteration:
iteration_number = self.inferIterationNumber(case)
else:
iteration_number = index
case_data = GenKwCollector.loadAllGenKwData(self.ert(), case)
if design_matrix_path is not None:
design_matrix_data = DesignMatrixReader.loadDesignMatrix(design_matrix_path)
case_data = case_data.join(design_matrix_data, how='inner')
misfit_data = MisfitCollector.loadAllMisfitData(self.ert(), case)
case_data = case_data.join(misfit_data, how='inner')
summary_data = SummaryCollector.loadAllSummaryData(self.ert(), case)
summary_data["Iteration"] = iteration_number
summary_data["Case"] = case
summary_data.set_index(["Case", "Iteration"], append=True, inplace=True)
case_data = case_data.join(summary_data, how='inner')
data = pandas.concat([data, case_data])
data = data.reorder_levels(["Realization", "Iteration", "Date", "Case"])
data.to_csv(output_file)
export_info = "Exported %d rows and %d columns to %s." % (len(data.index), len(data.columns), output_file)
return export_info
def test_join(self):
with ErtTestContext("python/enkf/export/export_join", self.config) as context:
dumpDesignMatrix("DesignMatrix.txt")
ert = context.getErt()
summary_data = SummaryCollector.loadAllSummaryData(ert, "default_1")
gen_kw_data = GenKwCollector.loadAllGenKwData(ert, "default_1")
misfit = MisfitCollector.loadAllMisfitData(ert, "default_1")
dm = DesignMatrixReader.loadDesignMatrix("DesignMatrix.txt")
result = summary_data.join(gen_kw_data, how='inner')
result = result.join(misfit, how='inner')
result = result.join(dm, how='inner')
first_date = "2010-01-10"
last_date = "2015-06-23"
self.assertFloatEqual(result["SNAKE_OIL_PARAM:OP1_OCTAVES"][0][first_date], 3.947766)
self.assertFloatEqual(result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][first_date], 4.206698)
self.assertFloatEqual(result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][last_date], 4.206698)
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][first_date], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][first_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][first_date], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][last_date], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][last_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][last_date], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][1][last_date], 0.07)
self.assertEqual(result["EXTRA_INT_COLUMN"][1][last_date], 225)
self.assertEqual(result["EXTRA_STRING_COLUMN"][1][last_date], "OFF")
self.assertFloatEqual(result["MISFIT:FOPR"][0][last_date], 489.191069)
self.assertFloatEqual(result["MISFIT:FOPR"][24][last_date], 1841.906872)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][first_date], 500.170035)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][last_date], 500.170035)
self.assertFloatEqual(result["MISFIT:TOTAL"][24][last_date], 1925.793865)
with self.assertRaises(KeyError):
realization_13 = result.loc[60]
column_count = len(result.columns)
self.assertEqual(result.dtypes[0], numpy.float64)
self.assertEqual(result.dtypes[column_count - 1], numpy.object)
self.assertEqual(result.dtypes[column_count - 2], numpy.int64)
def test_join(self):
with ErtTestContext("python/enkf/export/export_join", self.config) as context:
dumpDesignMatrix("DesignMatrix.txt")
ert = context.getErt()
summary_data = SummaryCollector.loadAllSummaryData(ert, "default")
gen_kw_data = GenKwCollector.loadAllGenKwData(ert, "default")
misfit = MisfitCollector.loadAllMisfitData(ert, "default")
dm = DesignMatrixReader.loadDesignMatrix("DesignMatrix.txt")
result = summary_data.join(gen_kw_data, how='inner')
result = result.join(misfit, how='inner')
result = result.join(dm, how='inner')
self.assertFloatEqual(result["FLUID_PARAMS:SGCR"][0]["2000-02-01"], 0.295136)
self.assertFloatEqual(result["FLUID_PARAMS:SGCR"][24]["2000-02-01"], 0.177833)
self.assertFloatEqual(result["FLUID_PARAMS:SGCR"][24]["2004-12-01"], 0.177833)
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0]["2000-02-01"], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0]["2000-02-01"], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0]["2000-02-01"], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0]["2004-12-01"], 0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0]["2004-12-01"], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0]["2004-12-01"], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][1]["2004-12-01"], 0.07)
self.assertEqual(result["EXTRA_INT_COLUMN"][1]["2004-12-01"], 225)
self.assertEqual(result["EXTRA_STRING_COLUMN"][1]["2004-12-01"], "OFF")
self.assertFloatEqual(result["MISFIT:WWCT:OP_2"][0]["2004-12-01"], 3.552157351038322)
self.assertFloatEqual(result["MISFIT:WWCT:OP_2"][24]["2004-12-01"], 25.318572860839158)
self.assertFloatEqual(result["MISFIT:TOTAL"][0]["2000-02-01"], 1621.755076130249)
self.assertFloatEqual(result["MISFIT:TOTAL"][0]["2004-12-01"], 1621.755076130249)
self.assertFloatEqual(result["MISFIT:TOTAL"][24]["2004-12-01"], 2783.7582107191383)
with self.assertRaises(KeyError):
realization_13 = result.loc[13]
column_count = len(result.columns)
self.assertEqual(result.dtypes[0], numpy.float64)
self.assertEqual(result.dtypes[column_count - 1], numpy.object)
self.assertEqual(result.dtypes[column_count - 2], numpy.int64)
def gatherGenKwData(ert, case, key):
""" :rtype: pandas.DataFrame """
data = GenKwCollector.loadAllGenKwData(ert, case, [key])
return data[key].dropna()
def test_join(self):
with ErtTestContext("python/enkf/export/export_join",
self.config) as context:
dumpDesignMatrix("DesignMatrix.txt")
ert = context.getErt()
summary_data = SummaryCollector.loadAllSummaryData(
ert, "default_1")
gen_kw_data = GenKwCollector.loadAllGenKwData(ert, "default_1")
misfit = MisfitCollector.loadAllMisfitData(ert, "default_1")
dm = DesignMatrixReader.loadDesignMatrix("DesignMatrix.txt")
result = summary_data.join(gen_kw_data, how='inner')
result = result.join(misfit, how='inner')
result = result.join(dm, how='inner')
first_date = "2010-01-10"
last_date = "2015-06-23"
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][0][first_date], 3.947766)
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][first_date],
4.206698)
self.assertFloatEqual(
result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][last_date], 4.206698)
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][first_date],
0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][first_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][first_date],
"ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][last_date],
0.08)
self.assertEqual(result["EXTRA_INT_COLUMN"][0][last_date], 125)
self.assertEqual(result["EXTRA_STRING_COLUMN"][0][last_date], "ON")
self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][1][last_date],
0.07)
self.assertEqual(result["EXTRA_INT_COLUMN"][1][last_date], 225)
self.assertEqual(result["EXTRA_STRING_COLUMN"][1][last_date],
"OFF")
self.assertFloatEqual(result["MISFIT:FOPR"][0][last_date],
489.191069)
self.assertFloatEqual(result["MISFIT:FOPR"][24][last_date],
1841.906872)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][first_date],
500.170035)
self.assertFloatEqual(result["MISFIT:TOTAL"][0][last_date],
500.170035)
self.assertFloatEqual(result["MISFIT:TOTAL"][24][last_date],
1925.793865)
with self.assertRaises(KeyError):
realization_13 = result.loc[60]
column_count = len(result.columns)
self.assertEqual(result.dtypes[0], numpy.float64)
self.assertEqual(result.dtypes[column_count - 1], numpy.object)
self.assertEqual(result.dtypes[column_count - 2], numpy.int64)