def apply_removers(config_key: str, dset: "Dataset") -> None:
"""Apply all removers for a given session
Args:
config_key: The configuration key listing which removers to apply.
dset: Dataset containing analysis data.
"""
prefix = config.analysis.get("analysis", default="").str
log.info(f"Applying removers")
keep_idxs = plugins.call_all(package_name=__name__,
config_key=config_key,
prefix=prefix,
dset=dset)
all_keep_idx = np.ones(dset.num_obs, dtype=bool)
for remover, remover_keep_idx in keep_idxs.items():
log.info(
f"Removing {sum(np.logical_not(remover_keep_idx)):5d} observations based on {remover}"
)
report.data("remover_data",
dset,
remover_name=remover,
keep_idx=remover_keep_idx)
all_keep_idx = np.logical_and(all_keep_idx, remover_keep_idx)
log.info(f"Keeping {sum(all_keep_idx)} of {dset.num_obs} observations")
dset.subset(all_keep_idx)
def detect_outliers(config_key, dset):
"""Detect all outliers for a given session
Args:
config_key (String): The configuration key listing which detectors to apply.
dset (Dataset): Dataset containing analysis data.
"""
prefix = config.analysis.get("analysis", default="").str
log.info(f"Detecting outliers")
keep_idxs = plugins.call_all(package_name=__name__,
config_key=config_key,
prefix=prefix,
dset=dset)
all_keep_idx = np.ones(dset.num_obs, dtype=bool)
for detector, detector_keep_idx in keep_idxs.items():
log.info(
f"Detecting {sum(~detector_keep_idx):5d} outliers based on {detector}"
)
report.data("detector_data",
dset,
detector_name=detector,
keep_idx=detector_keep_idx)
all_keep_idx = np.logical_and(all_keep_idx, detector_keep_idx)
log.info(f"Removing {sum(~all_keep_idx)} of {dset.num_obs} observations")
return all_keep_idx
def write(rundate, tech):
"""Call all reports specified in the configuration
The list of reports to use is taken from the config file of the given technique. Each report is passed a
:class:`~where.data.dataset.Dataset` with data for the modelrun and should write the relevant parts of the data to
file.
Args:
data: A Dataset containing model run data.
"""
prefix = config.analysis.get("analysis", default="").str
plugins.call_all(package_name=__name__,
config_key="report",
prefix=prefix,
rundate=rundate,
tech=tech)
def partial_vectors(dset, estimator_config_key):
"""Call all partials specified in the configuration and set up the corresponding state vector
The list of partials to calculate is taken from the config file of the given technique. Each partial calculator is
passed a :class:`~where.data.dataset.Dataset` with data for the modelrun and should return a tuple with the partial
vectors and their names.
Args:
dset (Dataset): A Dataset containing model run data.
estimator_config_key (String): Key in config file with the name of the estimator.
Returns:
Dict: List of names of the partial derivatives for each partial config key.
"""
partial_vectors = dict()
prefix = config.analysis.get("analysis", default="").str
for config_key in estimators.partial_config_keys(estimator_config_key):
partial_vectors[config_key] = list()
partial_data = plugins.call_all(package_name=__name__,
config_key=config_key,
prefix=prefix,
dset=dset)
for param, (data, names, data_unit) in partial_data.items():
param_unit_cfg = config.tech[param].unit
if not param_unit_cfg.str:
log.fatal(
f"No unit given for parameter {param!r} in {param_unit_cfg.source}"
)
display_unit = config.tech[param].display_unit.str
display_unit = param_unit_cfg.str if not display_unit else display_unit
partial_unit = str(
Unit("{} / ({})".format(dset.unit("calc"),
param_unit_cfg.str)).u)
factor = Unit(data_unit, partial_unit)
for values, name in zip(data.T, names):
partial_name = "{}-{}".format(param, name)
dset.add_float(
"partial_" + partial_name,
table="partial",
val=values * factor,
unit=partial_unit,
write_level="operational",
)
dset.add_to_meta("display_units", partial_name, display_unit)
partial_vectors[config_key].append(partial_name)
return partial_vectors
def apply_editors(config_key, dset):
"""Apply editors for a given session
Args:
config_key (String): The configuration key listing which editors to apply.
dset (Dataset): Dataset containing analysis data.
"""
prefix = config.analysis.get("analysis", default="").str
editors = plugins.list_all(package_name=__name__,
config_key=config_key,
prefix=prefix)
log.info(f"Applying editors")
return plugins.call_all(package_name=__name__,
config_key=config_key,
prefix=prefix,
dset=dset)
def get(dset, param_names):
"""Call an ..
Args:
dset (Dataset): Model run data.
param_names: Names of parameters to estimate
"""
constraints = plugins.call_all(
package_name=__name__, config_key="estimate_constraint", prefix="todo", dset=dset, param_names=param_names
)
h = np.concatenate([c[0] for c in constraints.values()])
sigma = np.concatenate([c[1] for c in constraints.values()])
import IPython
IPython.embed()
return h, sigma
def calculate(config_key, dset):
prefix = config.analysis.get("analysis", default="").str
return plugins.call_all(package_name=__name__, config_key=config_key, prefix=prefix, dset=dset)