def test_missing_data():
n_datasets = 5
npoints = 5
nsamples = 100
datasets = create_datasets(n_datasets, npoints, nsamples, missing=True)
metric_calculator = PairwiseIntercomparisonMetrics()
val = Validation(
datasets,
spatial_ref="0-ERA5",
metrics_calculators={(n_datasets, 2): metric_calculator.calc_metrics},
temporal_matcher=make_combined_temporal_matcher(pd.Timedelta(12, "H")),
)
gpis = list(range(npoints))
val.calc(gpis, gpis, gpis, rename_cols=False, only_with_temporal_ref=True)
def test_TripleCollocationMetrics(testdata_generator):
# tests by comparison of pairwise metrics to triplet metrics
datasets, expected = testdata_generator()
refname = "reference_name"
othernames = list(datasets.keys())
othernames.remove(refname)
triplet_metrics_calculator = TripleCollocationMetrics(refname,
bootstrap_cis=False)
matcher = make_combined_temporal_matcher(pd.Timedelta(6, "H"))
val_triplet = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=matcher,
metrics_calculators={(4, 3): triplet_metrics_calculator.calc_metrics})
results_triplet = val_triplet.calc([1], [1], [1], rename_cols=False)
if "col1_name" in datasets.keys():
# we only test the TCA results with the random data, since for the
# constant data all covariances are zero and TCA therefore doesn't
# work.
for metric in ["snr", "err_std", "beta"]:
for dset in datasets:
values = []
dkey = (dset, datasets[dset]["columns"][0])
for tkey in results_triplet:
if dkey in tkey:
values.append(results_triplet[tkey][(metric, dset)][0])
diff = np.abs(np.diff(values))
assert diff.max() / values[0] < 0.1
# check if writing to file works
results_path = Path("__test_results")
# if this throws, there's either some data left over from previous tests,
# or some data is named __test_results. Remove the __test_results directory
# from your current directory to make the test work again.
assert not results_path.exists()
results_path.mkdir(exist_ok=True, parents=True)
netcdf_results_manager(results_triplet, results_path.name)
assert results_path.exists()
for key in results_triplet:
fname = "_with_".join(map(lambda t: ".".join(t), key)) + ".nc"
assert (results_path / fname).exists()
# res = xr.open_dataset(results_path / fname)
# for metric in ["snr", "err_std", "beta"]:
# for dset, _ in key:
# mkey = metric + "__" + dset
# assert mkey in res.data_vars
shutil.rmtree(results_path)
# now with CIs, again only for random data
if "col1_name" in datasets.keys():
triplet_metrics_calculator = TripleCollocationMetrics(
refname, bootstrap_cis=True)
val_triplet = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=matcher,
metrics_calculators={
(4, 3): triplet_metrics_calculator.calc_metrics
})
results_triplet = val_triplet.calc([1], [1], [1], rename_cols=False)
for key in results_triplet:
for dset, _ in key:
for metric in ["snr", "err_std", "beta"]:
lkey = f"{metric}_ci_lower"
ukey = f"{metric}_ci_upper"
assert (lkey, dset) in results_triplet[key]
assert (ukey, dset) in results_triplet[key]
assert (results_triplet[key][(lkey, dset)] <=
results_triplet[key][(metric, dset)])
assert (results_triplet[key][(metric, dset)] <=
results_triplet[key][(ukey, dset)])
def test_PairwiseIntercomparisonMetrics_confidence_intervals():
# tests if the correct confidence intervals are returned
datasets, _ = testdata_random()
matcher = make_combined_temporal_matcher(pd.Timedelta(6, "H"))
val = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=matcher,
metrics_calculators={
(4, 2): (PairwiseIntercomparisonMetrics(
calc_spearman=True,
calc_kendall=True,
analytical_cis=True,
bootstrap_cis=True,
).calc_metrics)
})
results_pw = val.calc([1], [1], [1], rename_cols=False)
metrics_with_ci = {
"BIAS": "bias",
"R": "pearson_r",
"rho": "spearman_r",
"tau": "kendall_tau",
"RMSD": "rmsd",
"urmsd": "ubrmsd",
"mse": "msd",
"mse_bias": "mse_bias",
}
metrics_with_bs_ci = {
"mse_corr": "mse_corr",
"mse_var": "mse_var",
}
# reconstruct dataframe
frames = []
for key in datasets:
frames.append(datasets[key]["class"].data)
data = pd.concat(frames, axis=1)
data.dropna(how="any", inplace=True)
for key in results_pw:
othername = key[0][0]
other_col = othername.split("_")[0]
other = data[other_col].values
refname = key[1][0]
ref_col = refname.split("_")[0]
ref = data[ref_col].values
for metric_key in metrics_with_ci:
lower = results_pw[key][f"{metric_key}_ci_lower"]
upper = results_pw[key][f"{metric_key}_ci_upper"]
# calculate manually from data
metric_func = getattr(pairwise, metrics_with_ci[metric_key])
m, lb, ub = with_analytical_ci(metric_func, other, ref)
# difference due to float32 vs. float64
assert_almost_equal(upper, ub, 6)
assert_almost_equal(lower, lb, 6)
for metric_key in metrics_with_bs_ci:
lower = results_pw[key][f"{metric_key}_ci_lower"]
upper = results_pw[key][f"{metric_key}_ci_upper"]
# calculate manually from data
metric_func = getattr(pairwise, metrics_with_bs_ci[metric_key])
m, lb, ub = with_bootstrapped_ci(metric_func, other, ref)
assert_allclose(upper, ub, rtol=1e-1, atol=1e-4)
assert_allclose(lower, lb, rtol=1e-1, atol=1e-4)
def test_PairwiseIntercomparisonMetrics(testdata_generator):
# This test first compares the PairwiseIntercomparisonMetrics to known
# results and then confirms that it agrees with IntercomparisonMetrics as
# expected
datasets, expected = testdata_generator()
# for the pairwise intercomparison metrics it's important that we use
# make_combined_temporal_matcher
val = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=make_combined_temporal_matcher(pd.Timedelta(6, "H")),
metrics_calculators={
(4, 2):
(PairwiseIntercomparisonMetrics(calc_spearman=True,
analytical_cis=False).calc_metrics)
})
results_pw = val.calc([1], [1], [1], rename_cols=False)
# in results_pw, there are four entries with keys (("c1name", "c1"),
# ("refname", "ref"), and so on.
# Each value is a single dictionary with the values of the metrics
expected_metrics = [
"R", "p_R", "BIAS", "RMSD", "mse", "RSS", "mse_corr", "mse_bias",
"urmsd", "mse_var", "n_obs", "gpi", "lat", "lon", "rho", "p_rho",
"tau", "p_tau"
]
for key in results_pw:
assert isinstance(key, tuple)
assert len(key) == 2
assert all(map(lambda x: isinstance(x, tuple), key))
assert isinstance(results_pw[key], dict)
assert sorted(expected_metrics) == sorted(results_pw[key].keys())
for m in expected_metrics:
if m in expected[key]:
assert_equal(results_pw[key][m], expected[key][m])
# preparation of IntercomparisonMetrics run for comparison
ds_names = list(datasets.keys())
metrics = IntercomparisonMetrics(
dataset_names=ds_names,
# passing the names here explicitly, see GH issue #220
refname="reference_name",
other_names=ds_names[1:],
calc_tau=True,
)
val = Validation(
datasets,
"reference_name",
scaling=None,
temporal_matcher=None, # use default here
metrics_calculators={(4, 4): metrics.calc_metrics})
results = val.calc(1, 1, 1, rename_cols=False)
# results is a dictionary with one entry and key
# (('c1name', 'c1'), ('c2name', 'c2'), ('c3name', 'c3'), ('refname',
# 'ref')), the value is a list of length 0, which contains a dictionary
# with all the results, where the metrics are joined with "_between_" with
# the combination of datasets, which is joined with "_and_", e.g. for R
# between ``refname`` and ``c1name`` the key is
# "R_between_refname_and_c1name"
common_metrics = ["n_obs", "gpi", "lat", "lon"]
pw_metrics = list(set(expected_metrics) - set(common_metrics))
# there's some sorting done at some point in pytesmo
oldkey = tuple(sorted([(name, name.split("_")[0]) for name in ds_names]))
res_old = results[oldkey]
for key in results_pw:
res = results_pw[key]
# handle the full dataset metrics
for m in common_metrics:
assert_equal(res[m], res_old[m])
# now get the metrics and compare to the right combination
for m in pw_metrics:
othername = key[0][0]
refname = key[1][0]
if othername == "reference_name":
# sorting might be different, see GH #220
othername = key[1][0]
refname = key[0][0]
old_m_key = f"{m}_between_{refname}_and_{othername}"
if m == "BIAS":
# PairwiseIntercomparisonMetrics has the result as (other,
# ref), and therefore "bias between other and ref", compared to
# "bias between ref and bias" in IntercomparisonMetrics
# this is related to issue #220
assert_equal(np.abs(res[m]), np.abs(res_old[old_m_key]))
elif m == "urmsd":
# the old implementation differs from the new implementation
pass
else:
assert_equal(res[m], res_old[old_m_key])
def create_pytesmo_validation(validation_run):
ds_list = []
ref_name = None
scaling_ref_name = None
ds_num = 1
for dataset_config in validation_run.dataset_configurations.all():
reader = create_reader(dataset_config.dataset, dataset_config.version)
reader = setup_filtering(
reader, list(dataset_config.filters.all()),
list(dataset_config.parametrisedfilter_set.all()),
dataset_config.dataset, dataset_config.variable)
if validation_run.anomalies == ValidationRun.MOVING_AVG_35_D:
reader = AnomalyAdapter(
reader,
window_size=35,
columns=[dataset_config.variable.pretty_name])
if validation_run.anomalies == ValidationRun.CLIMATOLOGY:
# make sure our baseline period is in UTC and without timezone information
anomalies_baseline = [
validation_run.anomalies_from.astimezone(tz=pytz.UTC).replace(
tzinfo=None),
validation_run.anomalies_to.astimezone(tz=pytz.UTC).replace(
tzinfo=None)
]
reader = AnomalyClimAdapter(
reader,
columns=[dataset_config.variable.pretty_name],
timespan=anomalies_baseline)
if (validation_run.reference_configuration and
(dataset_config.id == validation_run.reference_configuration.id)):
# reference is always named "0-..."
dataset_name = '{}-{}'.format(0, dataset_config.dataset.short_name)
else:
dataset_name = '{}-{}'.format(ds_num,
dataset_config.dataset.short_name)
ds_num += 1
ds_list.append((dataset_name, {
'class': reader,
'columns': [dataset_config.variable.pretty_name]
}))
if (validation_run.reference_configuration and
(dataset_config.id == validation_run.reference_configuration.id)):
ref_name = dataset_name
ref_short_name = validation_run.reference_configuration.dataset.short_name
if (validation_run.scaling_ref
and (dataset_config.id == validation_run.scaling_ref.id)):
scaling_ref_name = dataset_name
datasets = dict(ds_list)
ds_num = len(ds_list)
period = None
if validation_run.interval_from is not None and validation_run.interval_to is not None:
# while pytesmo can't deal with timezones, normalise the validation period to utc; can be removed once pytesmo can do timezones
startdate = validation_run.interval_from.astimezone(UTC).replace(
tzinfo=None)
enddate = validation_run.interval_to.astimezone(UTC).replace(
tzinfo=None)
period = [startdate, enddate]
upscale_parms = None
if validation_run.upscaling_method != "none":
__logger.debug("Upscaling option is active")
upscale_parms = {
"upscaling_method": validation_run.upscaling_method,
"temporal_stability": validation_run.temporal_stability,
}
upscaling_lut = create_upscaling_lut(
validation_run=validation_run,
datasets=datasets,
ref_name=ref_name,
)
upscale_parms["upscaling_lut"] = upscaling_lut
__logger.debug("Lookup table for non-reference datasets " +
", ".join(upscaling_lut.keys()) + " created")
__logger.debug("{}".format(upscaling_lut))
datamanager = DataManager(
datasets,
ref_name=ref_name,
period=period,
read_ts_names='read',
upscale_parms=upscale_parms,
)
ds_names = get_dataset_names(datamanager.reference_name,
datamanager.datasets,
n=ds_num)
# set value of the metadata template according to what reference dataset is used
if ref_short_name == 'ISMN':
metadata_template = METADATA_TEMPLATE['ismn_ref']
else:
metadata_template = METADATA_TEMPLATE['other_ref']
pairwise_metrics = PairwiseIntercomparisonMetrics(
metadata_template=metadata_template,
calc_kendall=False,
)
metric_calculators = {(ds_num, 2): pairwise_metrics.calc_metrics}
if (len(ds_names) >= 3) and (validation_run.tcol is True):
tcol_metrics = TripleCollocationMetrics(
ref_name,
metadata_template=metadata_template,
)
metric_calculators.update({(ds_num, 3): tcol_metrics.calc_metrics})
if validation_run.scaling_method == validation_run.NO_SCALING:
scaling_method = None
else:
scaling_method = validation_run.scaling_method
__logger.debug(f"Scaling method: {scaling_method}")
__logger.debug(f"Scaling dataset: {scaling_ref_name}")
val = Validation(datasets=datamanager,
temporal_matcher=make_combined_temporal_matcher(
pd.Timedelta(12, "H")),
spatial_ref=ref_name,
scaling=scaling_method,
scaling_ref=scaling_ref_name,
metrics_calculators=metric_calculators,
period=period)
return val
def test_temporal_matching_ascat_ismn():
"""
This test uses a CSV file of ASCAT and ISMN data to test if the temporal
matching within the validation works as epxected in a "real" setup.
This only tests whether the number of observations matches, because this is
the main thing the temporal matching influences.
"""
# test with ASCAT and ISMN data
here = Path(__file__).resolve().parent
ascat = pd.read_csv(here / "ASCAT.csv", index_col=0, parse_dates=True)
ismn = pd.read_csv(here / "ISMN.csv", index_col=0, parse_dates=True)
dfs = {"ASCAT": ascat, "ISMN": ismn}
columns = {"ASCAT": "sm", "ISMN": "soil_moisture"}
refname = "ISMN"
window = pd.Timedelta(12, "H")
old_matcher = BasicTemporalMatching().combinatory_matcher
new_matcher = make_combined_temporal_matcher(window)
datasets = {}
for key in ["ISMN", "ASCAT"]:
all_columns = list(dfs[key].columns)
ds = {"columns": [columns[key]], "class": DummyReader(dfs[key], all_columns)}
datasets[key] = ds
new_val = Validation(
datasets,
refname,
scaling=None, # doesn't work with the constant test data
temporal_matcher=new_matcher,
metrics_calculators={
(2, 2): PairwiseIntercomparisonMetrics().calc_metrics
}
)
new_results = new_val.calc(
1, 1, 1, rename_cols=False, only_with_temporal_ref=True
)
# old setup
ds_names = list(datasets.keys())
metrics = IntercomparisonMetrics(
dataset_names=ds_names,
# passing the names here explicitly, see GH issue #220
refname=refname,
other_names=ds_names[1:],
calc_tau=True,
)
old_val = Validation(
datasets,
refname,
scaling=None, # doesn't work with the constant test data
temporal_matcher=old_matcher,
metrics_calculators={
(2, 2): metrics.calc_metrics
}
)
old_results = old_val.calc(
1, 1, 1, rename_cols=False
)
old_key = (('ASCAT', 'sm'), ('ISMN', 'soil_moisture'))
new_key = (('ASCAT', 'sm'), ('ISMN', 'soil_moisture'))
assert old_results[old_key]["n_obs"] == new_results[new_key]["n_obs"]
def test_dfdict_combined_temporal_collocation():
ref_dr = pd.date_range("2000", "2020", freq="YS")
dr1 = pd.date_range("2000", "2015", freq="YS")
dr2 = pd.date_range("2005", "2020", freq="YS")
ref_df = pd.DataFrame({"ref": np.arange(len(ref_dr))}, index=ref_dr)
df1 = pd.DataFrame({
"k1": np.arange(len(dr1)),
"k2": np.arange(len(dr1))
},
index=dr1)
df2 = pd.DataFrame({
"k1": np.arange(len(dr2)),
"k2": np.arange(len(dr2))
},
index=dr2)
dfs = {"refkey": ref_df, "df1key": df1, "df2key": df2}
window = pd.Timedelta(days=300)
matched = temporal_matchers.dfdict_combined_temporal_collocation(
dfs, "refkey", 2, window=window, n=3, combined_dropna=True)
# keys are the same, only refkey is missing
key = ("refkey", "df1key", "df2key")
assert list(matched.keys()) == [key]
# overlap is only 11 timestamps
assert matched[key].shape == (11, 5)
overlap_dr = pd.date_range("2005", "2015", freq="YS")
assert np.all(matched[key].index == overlap_dr)
# test with ASCAT and ISMN data
here = Path(__file__).resolve().parent
ascat = pd.read_csv(here / "ASCAT.csv", index_col=0, parse_dates=True)
ismn = pd.read_csv(here / "ISMN.csv", index_col=0, parse_dates=True)
dfs = {"ASCAT": ascat[["sm"]], "ISMN": ismn[["soil_moisture"]]}
refname = "ISMN"
window = pd.Timedelta(12, "H")
old_matcher = temporal_matchers.BasicTemporalMatching().combinatory_matcher
new_matcher = temporal_matchers.make_combined_temporal_matcher(window)
expected = old_matcher(dfs, refname, k=2, n=2)
new = new_matcher(dfs, refname, k=2, n=2)
key = ("ISMN", "ASCAT")
assert list(expected.keys()) == [key]
assert list(new.keys()) == [key]
# We have to do an extra dropna for the old matcher, because the old
# matcher doesn't do this by itself.
# This is normally done within validation.py, `get_data_for_result_tuple`,
# but since the combined matcher should exclude all data where even a
# single entry misses (so that all only have common data) this is done
# before in the new matcher (the combined matcher, whereas the old one is
# the combinatory matcher)
exp = expected[key].dropna()
assert exp.shape == new[key].shape
for col in new[key]:
np.testing.assert_equal(exp[col].values, new[key][col].values)