def _load_measured_record(enkf_main):
facade = LibresFacade(enkf_main)
obs_keys = [
facade.get_observation_key(nr)
for nr, _ in enumerate(facade.get_observations())
]
return MeasuredData(facade, obs_keys)
def run(self, job_config):
facade = LibresFacade(self.ert())
user_config = load_yaml(job_config)
user_config = _insert_default_group(user_config)
obs = facade.get_observations()
obs_keys = [facade.get_observation_key(nr) for nr, _ in enumerate(obs)]
obs_with_data = keys_with_data(
obs,
obs_keys,
facade.get_ensemble_size(),
facade.get_current_fs(),
)
default_values = _get_default_values(
facade.get_alpha(), facade.get_std_cutoff()
)
for config_dict in user_config:
config = ObsCorrConfig(config_dict, obs_keys, default_values)
config.validate(obs_with_data)
measured_data = _get_measured_data(
facade,
config.get_calculation_keys(),
config.get_index_lists(),
config.get_alpha(),
config.get_std_cutoff(),
)
job = ObservationScaleFactor(self.reporter, measured_data)
scale_factor = job.get_scaling_factor(config.get_threshold())
logging.info(
"Scaling factor calculated from keys: {}".format(
config.get_calculation_keys()
)
)
scale_observations(obs, scale_factor, config.get_update_keys())
def _load_measured_record(enkf_main):
facade = LibresFacade(enkf_main)
obs_keys = [
facade.get_observation_key(nr)
for nr, _ in enumerate(facade.get_observations())
]
measured_data = MeasuredData(facade, obs_keys)
measured_data.remove_failed_realizations()
measured_data.remove_inactive_observations()
measured_data.filter_ensemble_mean_obs(facade.get_alpha())
measured_data.filter_ensemble_std(facade.get_std_cutoff())
return measured_data
def run(self, job_config):
facade = LibresFacade(self.ert())
user_config = load_yaml(job_config)
user_config = _insert_default_group(user_config)
obs = facade.get_observations()
obs_keys = [facade.get_observation_key(nr) for nr, _ in enumerate(obs)]
obs_with_data = keys_with_data(
obs,
obs_keys,
facade.get_ensemble_size(),
facade.get_current_fs(),
)
for config in user_config:
job = ScalingJob(obs_keys, obs, obs_with_data, config)
measured_data = MeasuredData(facade, job.get_calc_keys(),
job.get_index_lists())
job.scale(measured_data)
def run(self, *args):
facade = LibresFacade(self.ert())
obs_keys = [
facade.get_observation_key(nr)
for nr, _ in enumerate(facade.get_observations())
]
measured_data = MeasuredData(facade, obs_keys)
parser = spearman_job_parser()
args = parser.parse_args(args)
scaling_configs = spearman_job(measured_data, args.threshold)
if not args.dry_run:
try:
CorrelatedObservationsScalingJob(
self.ert()).run(scaling_configs)
except EmptyDatasetException:
pass
def run(self,
target_name="analysis_case",
prior_name=None,
group_by="data_key"):
"""Perform analysis of parameters change per obs group
prior to posterior of ahm"""
ert = self.ert()
facade = LibresFacade(self.ert())
obs_keys = [
facade.get_observation_key(nr)
for nr, _ in enumerate(facade.get_observations())
]
key_map = _group_observations(facade, obs_keys, group_by)
prior_name, target_name = check_names(
facade.get_current_case_name(),
prior_name,
target_name,
)
# Get the prior scalar parameter distributions
prior_data = GenKwCollector.loadAllGenKwData(ert, prior_name)
raise_if_empty(
dataframes=[
prior_data,
MisfitCollector.loadAllMisfitData(ert, prior_name)
],
messages=[
"Empty prior ensemble",
"Empty parameters set for History Matching",
],
)
# create dataframe with observations vectors (1 by 1 obs and also all_obs)
combinations = make_obs_groups(key_map)
field_parameters = sorted(ert.ensembleConfig().getKeylistFromImplType(
ErtImplType.FIELD))
scalar_parameters = sorted(ert.ensembleConfig().getKeylistFromImplType(
ErtImplType.GEN_KW))
# identify the set of actual parameters that was updated for now just go
# through scalar parameters but in future if easier access to field parameter
# updates should also include field parameters
dkeysf = get_updated_parameters(prior_data, scalar_parameters)
# setup dataframe for calculated data
kolmogorov_smirnov_data, active_obs, misfitval = (
pd.DataFrame(sorted(dkeysf), columns=["Parameters"]),
pd.DataFrame(),
pd.DataFrame(index=["misfit"]),
)
# loop over keys and calculate the KS matrix,
# conditioning one parameter at the time.
field_output = {}
for group_name, obs_group in combinations.items():
print("Processing:", group_name)
# Use localization to evaluate change of parameters for each observation
with tempfile.TemporaryDirectory() as update_log_path:
_run_ministep(
ert,
obs_group,
field_parameters + scalar_parameters,
prior_name,
target_name,
update_log_path,
)
# Get the active vs total observation info
df_update_log = make_update_log_df(update_log_path)
# Get the updated scalar parameter distributions
self.reporter.publish_csv(
group_name, GenKwCollector.loadAllGenKwData(ert, target_name))
active_obs.at["ratio", group_name] = (
str(count_active_observations(df_update_log)) + " active/" +
str(len(df_update_log.index)))
# Get misfit values
misfitval[group_name] = [
calc_observationsgroup_misfit(
group_name,
df_update_log,
MisfitCollector.loadAllMisfitData(ert, prior_name),
)
]
# Calculate Ks matrix for scalar parameters
kolmogorov_smirnov_data[group_name] = kolmogorov_smirnov_data[
"Parameters"].map(
calc_kolmogorov_smirnov(
dkeysf,
prior_data,
GenKwCollector.loadAllGenKwData(ert, target_name),
))
field_output[group_name] = _get_field_params(
ert, facade.get_ensemble_size(), field_parameters, target_name)
kolmogorov_smirnov_data.set_index("Parameters", inplace=True)
# Calculate Ks matrix for Fields parameters
if field_parameters:
# Get grid characteristics to be able to plot field avg maps
grid_xyzcenter = load_grid_to_dataframe(
ert.eclConfig().get_gridfile())
all_input_prior = _get_field_params(ert,
facade.get_ensemble_size(),
field_parameters, prior_name)
for fieldparam in field_parameters:
scaler = StandardScaler()
scaler.fit(all_input_prior[fieldparam])
pca = PCA(0.98).fit(
pd.DataFrame(scaler.transform(
all_input_prior[fieldparam])))
pc_fieldprior_df = pd.DataFrame(data=pca.transform(
scaler.transform(all_input_prior[fieldparam])))
all_kolmogorov_smirnov = pd.DataFrame(
pc_fieldprior_df.columns.tolist(),
columns=["PCFieldParameters"])
# Get the posterior Field parameters
map_calc_properties = (grid_xyzcenter[grid_xyzcenter["KZ"] ==
1].copy().reset_index())
for group_name in combinations.keys():
map_calc_properties[
"Mean_D_" + group_name] = calc_mean_delta_grid(
field_output[group_name][fieldparam],
all_input_prior[fieldparam],
grid_xyzcenter,
)
pc_fieldpost_df = pd.DataFrame(data=pca.transform(
scaler.transform(field_output[group_name]
[fieldparam])))
all_kolmogorov_smirnov[
group_name] = all_kolmogorov_smirnov[
"PCFieldParameters"].map(
calc_kolmogorov_smirnov(
pc_fieldpost_df,
pc_fieldprior_df,
pc_fieldpost_df,
))
all_kolmogorov_smirnov.set_index("PCFieldParameters",
inplace=True)
# add the field max Ks to the scalar Ks matrix
kolmogorov_smirnov_data.loc[
"FIELD_" + fieldparam] = all_kolmogorov_smirnov.max()
self.reporter.publish_csv("delta_field" + fieldparam,
map_calc_properties)
# save/export the Ks matrix, active_obs, misfitval and prior data
self.reporter.publish_csv("ks", kolmogorov_smirnov_data)
self.reporter.publish_csv("active_obs_info", active_obs)
self.reporter.publish_csv("misfit_obs_info", misfitval)
self.reporter.publish_csv("prior", prior_data)