def test_gen_kw_collector(self): with ErtTestContext("python/enkf/export/gen_kw_collector", self.config) as context: ert = context.getErt() data = GenKwCollector.loadAllGenKwData(ert, "default_0") self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0], 0.047517) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][24], 0.160907) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0], 0.054539) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][12], 0.057807) realization_20 = data.loc[20] with self.assertRaises(KeyError): realization_60 = data.loc[60] data = GenKwCollector.loadAllGenKwData(ert, "default_0", [ "SNAKE_OIL_PARAM:OP1_PERSISTENCE", "SNAKE_OIL_PARAM:OP1_OFFSET" ]) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0], 0.047517) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0], 0.054539) with self.assertRaises(KeyError): data["SNAKE_OIL_PARAM:OP1_DIVERGENCE_SCALE"]
def test_smoother(self): test_config = self.createTestPath("local/custom_kw/mini_config") with ErtShellTestContext("python/ertshell/smoother", test_config) as shell: print(os.getcwd()) shell.invokeCommand("case select test_run") self.assertTrue(shell.invokeCommand("smoother update test_run_update")) shell.invokeCommand("case select test_run_update") ert = shell.shellContext().ert() data = GenKwCollector.loadAllGenKwData(ert, "test_run", keys=["PERLIN_PARAM:SCALE"]) update_data = GenKwCollector.loadAllGenKwData(ert, "test_run_update", keys=["PERLIN_PARAM:SCALE"]) self.assertTrue(data["PERLIN_PARAM:SCALE"].std() > update_data["PERLIN_PARAM:SCALE"].std())
def test_smoother(self): test_config = self.createTestPath("local/custom_kw/mini_config") with ErtShellTestContext("python/ertshell/smoother", test_config) as shell: print(os.getcwd()) shell.invokeCommand("case select test_run") self.assertTrue(shell.invokeCommand("smoother update test_run_update")) shell.invokeCommand("case select test_run_update") ert = shell.shellContext().ert() data = GenKwCollector.loadAllGenKwData(ert, "test_run", keys=["PERLIN_PARAM:SCALE"]) update_data = GenKwCollector.loadAllGenKwData(ert, "test_run_update", keys=["PERLIN_PARAM:SCALE"]) self.assertTrue(data["PERLIN_PARAM:SCALE"].std() > update_data["PERLIN_PARAM:SCALE"].std())
def test_join(self): with ErtTestContext("python/enkf/export/export_join", self.config) as context: dumpDesignMatrix("DesignMatrix.txt") ert = context.getErt() summary_data = SummaryCollector.loadAllSummaryData(ert, "default_1") gen_kw_data = GenKwCollector.loadAllGenKwData(ert, "default_1") misfit = MisfitCollector.loadAllMisfitData(ert, "default_1") dm = DesignMatrixReader.loadDesignMatrix("DesignMatrix.txt") result = summary_data.join(gen_kw_data, how="inner") result = result.join(misfit, how="inner") result = result.join(dm, how="inner") first_date = "2010-01-10" last_date = "2015-06-23" self.assertFloatEqual( result["SNAKE_OIL_PARAM:OP1_OCTAVES"][0][first_date], 3.947766 ) self.assertFloatEqual( result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][first_date], 4.206698 ) self.assertFloatEqual( result["SNAKE_OIL_PARAM:OP1_OCTAVES"][24][last_date], 4.206698 ) self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][first_date], 0.08) self.assertEqual(result["EXTRA_INT_COLUMN"][0][first_date], 125) self.assertEqual(result["EXTRA_STRING_COLUMN"][0][first_date], "ON") self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][0][last_date], 0.08) self.assertEqual(result["EXTRA_INT_COLUMN"][0][last_date], 125) self.assertEqual(result["EXTRA_STRING_COLUMN"][0][last_date], "ON") self.assertFloatEqual(result["EXTRA_FLOAT_COLUMN"][1][last_date], 0.07) self.assertEqual(result["EXTRA_INT_COLUMN"][1][last_date], 225) self.assertEqual(result["EXTRA_STRING_COLUMN"][1][last_date], "OFF") self.assertFloatEqual(result["MISFIT:FOPR"][0][last_date], 457.491003) self.assertFloatEqual(result["MISFIT:FOPR"][24][last_date], 1630.774198) self.assertFloatEqual(result["MISFIT:TOTAL"][0][first_date], 468.469969) self.assertFloatEqual(result["MISFIT:TOTAL"][0][last_date], 468.469969) self.assertFloatEqual(result["MISFIT:TOTAL"][24][last_date], 1714.662370) with self.assertRaises(KeyError): realization_13 = result.loc[60] column_count = len(result.columns) self.assertEqual(result.dtypes[0], numpy.float64) self.assertEqual(result.dtypes[column_count - 1], numpy.object) self.assertEqual(result.dtypes[column_count - 2], numpy.int64)
def run(self, output_file, case_list=None, design_matrix_path=None, infer_iteration=True): cases = [] if case_list is not None: if case_list.strip() == "*": cases = self.getAllCaseList() else: cases = case_list.split(",") if case_list is None or len(cases) == 0: cases = [self.ert().getEnkfFsManager().getCurrentFileSystem().getCaseName()] if design_matrix_path is not None: if not os.path.exists(design_matrix_path): raise UserWarning("The design matrix file does not exists!") if not os.path.isfile(design_matrix_path): raise UserWarning("The design matrix is not a file!") data = pandas.DataFrame() for index, case in enumerate(cases): case = case.strip() if not self.ert().getEnkfFsManager().caseExists(case): raise UserWarning("The case '%s' does not exist!" % case) if not self.ert().getEnkfFsManager().caseHasData(case): raise UserWarning("The case '%s' does not have any data!" % case) if infer_iteration: iteration_number = self.inferIterationNumber(case) else: iteration_number = index case_data = GenKwCollector.loadAllGenKwData(self.ert(), case) custom_kw_data = CustomKWCollector.loadAllCustomKWData(self.ert(), case) if not custom_kw_data.empty: case_data = case_data.join(custom_kw_data, how='outer') if design_matrix_path is not None: design_matrix_data = DesignMatrixReader.loadDesignMatrix(design_matrix_path) if not design_matrix_data.empty: case_data = case_data.join(design_matrix_data, how='outer') misfit_data = MisfitCollector.loadAllMisfitData(self.ert(), case) if not misfit_data.empty: case_data = case_data.join(misfit_data, how='outer') summary_data = SummaryCollector.loadAllSummaryData(self.ert(), case) if not summary_data.empty: case_data = case_data.join(summary_data, how='outer') else: case_data["Date"] = None case_data.set_index(["Date"], append=True, inplace=True) case_data["Iteration"] = iteration_number case_data["Case"] = case case_data.set_index(["Case", "Iteration"], append=True, inplace=True) data = pandas.concat([data, case_data]) data = data.reorder_levels(["Realization", "Iteration", "Date", "Case"]) data.to_csv(output_file) export_info = "Exported %d rows and %d columns to %s." % (len(data.index), len(data.columns), output_file) return export_info
def gatherGenKwData(ert, case, key): """ :rtype: pandas.DataFrame """ data = GenKwCollector.loadAllGenKwData(ert, case, [key]) return data[key].dropna()
def test_gen_kw_collector(self): with ErtTestContext("python/enkf/export/gen_kw_collector", self.config) as context: ert = context.getErt() data = GenKwCollector.loadAllGenKwData(ert, "default_0") self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0], 0.047517) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][24], 0.160907) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0], 0.054539) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][12], 0.057807) realization_20 = data.loc[20] with self.assertRaises(KeyError): realization_60 = data.loc[60] data = GenKwCollector.loadAllGenKwData( ert, "default_0", [ "SNAKE_OIL_PARAM:OP1_PERSISTENCE", "SNAKE_OIL_PARAM:OP1_OFFSET" ], ) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][0], 0.047517) self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_OFFSET"][0], 0.054539) with self.assertRaises(KeyError): data["SNAKE_OIL_PARAM:OP1_DIVERGENCE_SCALE"] realization_index = 10 data = GenKwCollector.loadAllGenKwData( ert, "default_0", ["SNAKE_OIL_PARAM:OP1_PERSISTENCE"], realization_index=realization_index, ) assert data.index == [realization_index] assert len(data.index) == 1 assert list(data.columns) == ["SNAKE_OIL_PARAM:OP1_PERSISTENCE"] self.assertFloatEqual(data["SNAKE_OIL_PARAM:OP1_PERSISTENCE"][10], 0.282923) non_existing_realization_index = 150 with pytest.raises(IndexError): data = GenKwCollector.loadAllGenKwData( ert, "default_0", ["SNAKE_OIL_PARAM:OP1_PERSISTENCE"], realization_index=non_existing_realization_index, )
def run(self, target_name="analysis_case", prior_name=None, group_by="data_key"): """Perform analysis of parameters change per obs group prior to posterior of ahm""" ert = self.ert() facade = LibresFacade(self.ert()) obs_keys = [ facade.get_observation_key(nr) for nr, _ in enumerate(facade.get_observations()) ] key_map = _group_observations(facade, obs_keys, group_by) prior_name, target_name = check_names( facade.get_current_case_name(), prior_name, target_name, ) # Get the prior scalar parameter distributions prior_data = GenKwCollector.loadAllGenKwData(ert, prior_name) raise_if_empty( dataframes=[ prior_data, MisfitCollector.loadAllMisfitData(ert, prior_name) ], messages=[ "Empty prior ensemble", "Empty parameters set for History Matching", ], ) # create dataframe with observations vectors (1 by 1 obs and also all_obs) combinations = make_obs_groups(key_map) field_parameters = sorted(ert.ensembleConfig().getKeylistFromImplType( ErtImplType.FIELD)) scalar_parameters = sorted(ert.ensembleConfig().getKeylistFromImplType( ErtImplType.GEN_KW)) # identify the set of actual parameters that was updated for now just go # through scalar parameters but in future if easier access to field parameter # updates should also include field parameters dkeysf = get_updated_parameters(prior_data, scalar_parameters) # setup dataframe for calculated data kolmogorov_smirnov_data, active_obs, misfitval = ( pd.DataFrame(sorted(dkeysf), columns=["Parameters"]), pd.DataFrame(), pd.DataFrame(index=["misfit"]), ) # loop over keys and calculate the KS matrix, # conditioning one parameter at the time. field_output = {} for group_name, obs_group in combinations.items(): print("Processing:", group_name) # Use localization to evaluate change of parameters for each observation with tempfile.TemporaryDirectory() as update_log_path: _run_ministep( ert, obs_group, field_parameters + scalar_parameters, prior_name, target_name, update_log_path, ) # Get the active vs total observation info df_update_log = make_update_log_df(update_log_path) # Get the updated scalar parameter distributions self.reporter.publish_csv( group_name, GenKwCollector.loadAllGenKwData(ert, target_name)) active_obs.at["ratio", group_name] = ( str(count_active_observations(df_update_log)) + " active/" + str(len(df_update_log.index))) # Get misfit values misfitval[group_name] = [ calc_observationsgroup_misfit( group_name, df_update_log, MisfitCollector.loadAllMisfitData(ert, prior_name), ) ] # Calculate Ks matrix for scalar parameters kolmogorov_smirnov_data[group_name] = kolmogorov_smirnov_data[ "Parameters"].map( calc_kolmogorov_smirnov( dkeysf, prior_data, GenKwCollector.loadAllGenKwData(ert, target_name), )) field_output[group_name] = _get_field_params( ert, facade.get_ensemble_size(), field_parameters, target_name) kolmogorov_smirnov_data.set_index("Parameters", inplace=True) # Calculate Ks matrix for Fields parameters if field_parameters: # Get grid characteristics to be able to plot field avg maps grid_xyzcenter = load_grid_to_dataframe( ert.eclConfig().get_gridfile()) all_input_prior = _get_field_params(ert, facade.get_ensemble_size(), field_parameters, prior_name) for fieldparam in field_parameters: scaler = StandardScaler() scaler.fit(all_input_prior[fieldparam]) pca = PCA(0.98).fit( pd.DataFrame(scaler.transform( all_input_prior[fieldparam]))) pc_fieldprior_df = pd.DataFrame(data=pca.transform( scaler.transform(all_input_prior[fieldparam]))) all_kolmogorov_smirnov = pd.DataFrame( pc_fieldprior_df.columns.tolist(), columns=["PCFieldParameters"]) # Get the posterior Field parameters map_calc_properties = (grid_xyzcenter[grid_xyzcenter["KZ"] == 1].copy().reset_index()) for group_name in combinations.keys(): map_calc_properties[ "Mean_D_" + group_name] = calc_mean_delta_grid( field_output[group_name][fieldparam], all_input_prior[fieldparam], grid_xyzcenter, ) pc_fieldpost_df = pd.DataFrame(data=pca.transform( scaler.transform(field_output[group_name] [fieldparam]))) all_kolmogorov_smirnov[ group_name] = all_kolmogorov_smirnov[ "PCFieldParameters"].map( calc_kolmogorov_smirnov( pc_fieldpost_df, pc_fieldprior_df, pc_fieldpost_df, )) all_kolmogorov_smirnov.set_index("PCFieldParameters", inplace=True) # add the field max Ks to the scalar Ks matrix kolmogorov_smirnov_data.loc[ "FIELD_" + fieldparam] = all_kolmogorov_smirnov.max() self.reporter.publish_csv("delta_field" + fieldparam, map_calc_properties) # save/export the Ks matrix, active_obs, misfitval and prior data self.reporter.publish_csv("ks", kolmogorov_smirnov_data) self.reporter.publish_csv("active_obs_info", active_obs) self.reporter.publish_csv("misfit_obs_info", misfitval) self.reporter.publish_csv("prior", prior_data)