def test_validation_n3_k2():
tst_results = {
(('DS1', 'x'), ('DS3', 'y')): {
'n_obs': np.array([1000], dtype=np.int32),
'tau': np.array([np.nan], dtype=np.float32),
'gpi': np.array([4], dtype=np.int32),
'RMSD': np.array([0.], dtype=np.float32),
'lon': np.array([4.]),
'p_tau': np.array([np.nan], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)},
(('DS1', 'x'), ('DS2', 'y')): {
'n_obs': np.array([1000], dtype=np.int32),
'tau': np.array([np.nan], dtype=np.float32),
'gpi': np.array([4], dtype=np.int32),
'RMSD': np.array([0.], dtype=np.float32),
'lon': np.array([4.]),
'p_tau': np.array([np.nan], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)},
(('DS1', 'x'), ('DS3', 'x')): {
'n_obs': np.array([1000], dtype=np.int32),
'tau': np.array([np.nan], dtype=np.float32),
'gpi': np.array([4], dtype=np.int32),
'RMSD': np.array([0.], dtype=np.float32),
'lon': np.array([4.]),
'p_tau': np.array([np.nan], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)}}
datasets = setup_TestDatasets()
process = Validation(
datasets, 'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(3, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics})
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
assert sorted(list(results)) == sorted(list(tst_results))
def test_validation_n3_k2_masking_no_data_remains():
datasets = setup_TestDatasets()
# setup masking datasets
grid = grids.CellGrid(np.array([1, 2, 3, 4]), np.array([1, 2, 3, 4]),
np.array([4, 4, 2, 1]), gpis=np.array([1, 2, 3, 4]))
mds1 = GriddedTsBase("", grid, MaskingTestDataset)
mds2 = GriddedTsBase("", grid, MaskingTestDataset)
mds = {
'masking1': {
'class': mds1,
'columns': ['x'],
'args': [],
'kwargs': {'limit': 500},
'use_lut': False,
'grids_compatible': True},
'masking2': {
'class': mds2,
'columns': ['x'],
'args': [],
'kwargs': {'limit': 1000},
'use_lut': False,
'grids_compatible': True}
}
process = Validation(
datasets, 'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(3, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics},
masking_datasets=mds)
gpi_info = (1, 1, 1)
ref_df = datasets['DS1']['class'].read_ts(1)
new_ref_df = process.mask_dataset(ref_df, gpi_info)
assert len(new_ref_df) == 0
nptest.assert_allclose(new_ref_df.x.values, np.arange(1000, 1000))
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
tst = []
assert sorted(list(results)) == sorted(list(tst))
for key, tst_key in zip(sorted(results),
sorted(tst)):
nptest.assert_almost_equal(results[key]['n_obs'],
tst[tst_key]['n_obs'])
def test_validation_n2_k2_temporal_matching_no_matches():
tst_results = {}
datasets = setup_two_without_overlap()
process = Validation(
datasets, 'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics})
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
assert sorted(list(results)) == sorted(list(tst_results))
def test_validation_n3_k2():
tst_results = {
(("DS1", "x"), ("DS3", "y")): {
"n_obs": np.array([1000], dtype=np.int32),
"tau": np.array([np.nan], dtype=np.float32),
"gpi": np.array([4], dtype=np.int32),
"RMSD": np.array([0.0], dtype=np.float32),
"lon": np.array([4.0]),
"p_tau": np.array([np.nan], dtype=np.float32),
"BIAS": np.array([0.0], dtype=np.float32),
"p_rho": np.array([0.0], dtype=np.float32),
"rho": np.array([1.0], dtype=np.float32),
"lat": np.array([4.0]),
"R": np.array([1.0], dtype=np.float32),
"p_R": np.array([0.0], dtype=np.float32),
},
(("DS1", "x"), ("DS2", "y")): {
"n_obs": np.array([1000], dtype=np.int32),
"tau": np.array([np.nan], dtype=np.float32),
"gpi": np.array([4], dtype=np.int32),
"RMSD": np.array([0.0], dtype=np.float32),
"lon": np.array([4.0]),
"p_tau": np.array([np.nan], dtype=np.float32),
"BIAS": np.array([0.0], dtype=np.float32),
"p_rho": np.array([0.0], dtype=np.float32),
"rho": np.array([1.0], dtype=np.float32),
"lat": np.array([4.0]),
"R": np.array([1.0], dtype=np.float32),
"p_R": np.array([0.0], dtype=np.float32),
},
(("DS1", "x"), ("DS3", "x")): {
"n_obs": np.array([1000], dtype=np.int32),
"tau": np.array([np.nan], dtype=np.float32),
"gpi": np.array([4], dtype=np.int32),
"RMSD": np.array([0.0], dtype=np.float32),
"lon": np.array([4.0]),
"p_tau": np.array([np.nan], dtype=np.float32),
"BIAS": np.array([0.0], dtype=np.float32),
"p_rho": np.array([0.0], dtype=np.float32),
"rho": np.array([1.0], dtype=np.float32),
"lat": np.array([4.0]),
"R": np.array([1.0], dtype=np.float32),
"p_R": np.array([0.0], dtype=np.float32),
},
(("DS2", "y"), ("DS3", "x")): {
"gpi": np.array([4], dtype=np.int32),
"lon": np.array([4.0]),
"lat": np.array([4.0]),
"n_obs": np.array([1000], dtype=np.int32),
"R": np.array([1.0], dtype=np.float32),
"p_R": np.array([0.0], dtype=np.float32),
"rho": np.array([1.0], dtype=np.float32),
"p_rho": np.array([0.0], dtype=np.float32),
"RMSD": np.array([0.0], dtype=np.float32),
"BIAS": np.array([0.0], dtype=np.float32),
"tau": np.array([np.nan], dtype=np.float32),
"p_tau": np.array([np.nan], dtype=np.float32),
},
(("DS2", "y"), ("DS3", "y")): {
"gpi": np.array([4], dtype=np.int32),
"lon": np.array([4.0]),
"lat": np.array([4.0]),
"n_obs": np.array([1000], dtype=np.int32),
"R": np.array([1.0], dtype=np.float32),
"p_R": np.array([0.0], dtype=np.float32),
"rho": np.array([1.0], dtype=np.float32),
"p_rho": np.array([0.0], dtype=np.float32),
"RMSD": np.array([0.0], dtype=np.float32),
"BIAS": np.array([0.0], dtype=np.float32),
"tau": np.array([np.nan], dtype=np.float32),
"p_tau": np.array([np.nan], dtype=np.float32),
},
}
datasets = setup_TestDatasets()
dm = DataManager(
datasets,
"DS1",
read_ts_names={d: "read"
for d in ["DS1", "DS2", "DS3"]},
)
process = Validation(
dm,
"DS1",
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling="lin_cdf_match",
metrics_calculators={
(3, 2):
metrics_calculators.BasicMetrics(other_name="k1").calc_metrics
},
)
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
assert sorted(list(results)) == sorted(list(tst_results))
def getdata():
"""
handles the get request, which should contain the arguments listes under
parameters
Parameters
----------
station_id: int
id of station in database
scaling: string
chosen scaling method , for available choices see general.times_eries.scaling
snow_depth: float
mask snow depth greater than this value
st_l1: float
mask surface temperature layer1 lower than this value
air_temp: float
mask 2m air temperature lower than this value
ssf_masking: boolean
use SSF for masking true or false
"""
station_id = request.args.get('station_id')
scaling = request.args.get('scaling')
if scaling == 'noscale':
scaling = None
masking_ids = request.args.getlist('masking_ds[]')
masking_ops = request.args.getlist('masking_op[]')
masking_values = request.args.getlist('masking_values[]')
masking_values = [float(x) for x in masking_values]
anomaly = request.args.get('anomaly')
if anomaly == 'none':
anomaly = None
(depth_from,
depth_to,
sensor_id) = get_station_first_sm_layer(app.config['ISMN_PATH'],
station_id)
lon, lat = get_station_lonlat(app.config['ISMN_PATH'],
station_id)
start, end = get_station_start_end(app.config['ISMN_PATH'],
station_id, "soil moisture",
depth_from, depth_to)
period = [start, end]
masking_data = {'labels': [], 'data': []}
masking_meta = get_masking_metadata()
masking_masked_dict = None
if len(masking_ids) > 0:
# prepare masking datasets
masking_ds_dict = get_masking_ds_dict(masking_ids)
masking_masked_dict = {}
for masking_ds, masking_op, masking_value in zip(masking_ids,
masking_ops,
masking_values):
masking_masked_dict[masking_ds] = dict(masking_ds_dict[masking_ds])
new_cls = MaskingAdapter(masking_masked_dict[masking_ds]['class'],
masking_op,
masking_value)
masking_masked_dict[masking_ds]['class'] = new_cls
# use DataManager for reading masking datasets
masking_dm = DataManager(masking_ds_dict, masking_ids[0],
period=period)
masking_data = {}
valid_masking_ids = []
for mds in masking_ids:
mdata = masking_dm.read_ds(mds, lon, lat)
if mdata is not None:
masking_data[mds] = mdata
valid_masking_ids.append(mds)
else:
masking_data[mds] = pd.DataFrame()
if len(valid_masking_ids) > 1:
masking_data = BasicTemporalMatching(window=1.0).combinatory_matcher(
masking_data, masking_ids[0], n=len(masking_ids))
if len(masking_data) > 0:
labels, values = masking_data[
masking_data.keys()[0]].to_dygraph_format()
elif len(valid_masking_ids) == 1:
masking_data = masking_data[valid_masking_ids[0]]
labels, values = masking_data.to_dygraph_format()
else:
labels = [None]
values = None
for i, label in enumerate(labels):
for mid in masking_meta:
if masking_meta[mid]['variable']['name'] in label:
labels[i] = masking_meta[mid]['long_name']
masking_data = {'labels': labels, 'data': values}
ismn_iface = prepare_station_interface(app.config['ISMN_PATH'],
station_id,
"soil moisture",
depth_from, depth_to, sensor_id)
validation_ds_dict = get_validation_ds_dict()
validation_ds_dict.update({'ISMN': {'class': ismn_iface,
'columns': ['soil moisture']}})
if anomaly is not None:
adapter = {'climatology': AnomalyClimAdapter,
'average': AnomalyAdapter}
for dataset in validation_ds_dict:
validation_ds_dict[dataset]['class'] = adapter[
anomaly](validation_ds_dict[dataset]['class'],
columns=validation_ds_dict[dataset]['columns'])
mcalc = BasicMetricsPlusMSE(other_name='k1',
calc_tau=True).calc_metrics
process = Validation(validation_ds_dict, 'ISMN',
temporal_ref='cci',
scaling=scaling,
metrics_calculators={(2, 2): mcalc},
masking_datasets=masking_masked_dict,
period=period,
temporal_window=1)
df_dict = process.data_manager.get_data(1,
lon,
lat)
matched_data, result, used_data = process.perform_validation(
df_dict, (1, lon, lat))
res_key = list(result)[0]
data = used_data[res_key]
result = result[res_key][0]
# rename data to original names
rename_dict = {}
f = lambda x: "k{}".format(x) if x > 0 else 'ref'
for i, r in enumerate(res_key):
rename_dict[f(i)] = " ".join(r)
data.rename(columns=rename_dict, inplace=True)
labels, values = data.to_dygraph_format()
validation_datasets = {'labels': labels, 'data': values}
statistics = {'kendall': {'v': '%.2f' % result['tau'], 'p': '%.4f' % result['p_tau']},
'spearman': {'v': '%.2f' % result['rho'], 'p': '%.4f' % result['p_rho']},
'pearson': {'v': '%.2f' % result['R'], 'p': '%.4f' % result['p_R']},
'bias': '%.4f' % result['BIAS'],
'rmsd': {'rmsd': '%.4f' % np.sqrt(result['mse']),
'rmsd_corr': '%.4f' % np.sqrt(result['mse_corr']),
'rmsd_bias': '%.4f' % np.sqrt(result['mse_bias']),
'rmsd_var': '%.4f' % np.sqrt(result['mse_var'])},
'mse': {'mse': '%.4f' % result['mse'],
'mse_corr': '%.4f' % result['mse_corr'],
'mse_bias': '%.4f' % result['mse_bias'],
'mse_var': '%.4f' % result['mse_var']}}
scaling_options = {'noscale': 'No scaling',
'porosity': 'Scale using porosity',
'linreg': 'Linear Regression',
'mean_std': 'Mean - standard deviation',
'min_max': 'Minimum,maximum',
'lin_cdf_match': 'Piecewise <br> linear CDF matching',
'cdf_match': 'CDF matching'}
if scaling is None:
scaling = 'noscale'
masking_option_return = {}
for mid, mops, mval in zip(masking_ids,
masking_ops,
masking_values):
masking_option_return[mid] = {'op': mops,
'val': mval,
'name': masking_meta[mid]['long_name']}
settings = {'scaling': scaling_options[scaling],
'masking': masking_option_return}
output_data = {'validation_data': validation_datasets, 'masking_data': masking_data,
'statistics': statistics, 'settings': settings}
status = 1
if status == -1:
data = 'Error'
else:
data = jsonify(output_data)
resp = make_response(data)
resp.headers['Access-Control-Allow-Origin'] = '*'
return resp
def test_temporal_matching_ascat_ismn():
"""
This test uses a CSV file of ASCAT and ISMN data to test if the temporal
matching within the validation works as epxected in a "real" setup.
This only tests whether the number of observations matches, because this is
the main thing the temporal matching influences.
"""
# test with ASCAT and ISMN data
here = Path(__file__).resolve().parent
ascat = pd.read_csv(here / "ASCAT.csv", index_col=0, parse_dates=True)
ismn = pd.read_csv(here / "ISMN.csv", index_col=0, parse_dates=True)
dfs = {"ASCAT": ascat, "ISMN": ismn}
columns = {"ASCAT": "sm", "ISMN": "soil_moisture"}
refname = "ISMN"
window = pd.Timedelta(12, "H")
old_matcher = BasicTemporalMatching().combinatory_matcher
new_matcher = make_combined_temporal_matcher(window)
datasets = {}
for key in ["ISMN", "ASCAT"]:
all_columns = list(dfs[key].columns)
ds = {"columns": [columns[key]], "class": DummyReader(dfs[key], all_columns)}
datasets[key] = ds
new_val = Validation(
datasets,
refname,
scaling=None, # doesn't work with the constant test data
temporal_matcher=new_matcher,
metrics_calculators={
(2, 2): PairwiseIntercomparisonMetrics().calc_metrics
}
)
new_results = new_val.calc(
1, 1, 1, rename_cols=False, only_with_temporal_ref=True
)
# old setup
ds_names = list(datasets.keys())
metrics = IntercomparisonMetrics(
dataset_names=ds_names,
# passing the names here explicitly, see GH issue #220
refname=refname,
other_names=ds_names[1:],
calc_tau=True,
)
old_val = Validation(
datasets,
refname,
scaling=None, # doesn't work with the constant test data
temporal_matcher=old_matcher,
metrics_calculators={
(2, 2): metrics.calc_metrics
}
)
old_results = old_val.calc(
1, 1, 1, rename_cols=False
)
old_key = (('ASCAT', 'sm'), ('ISMN', 'soil_moisture'))
new_key = (('ASCAT', 'sm'), ('ISMN', 'soil_moisture'))
assert old_results[old_key]["n_obs"] == new_results[new_key]["n_obs"]
def test_PairwiseIntercomparisonMetrics_confidence_intervals():
# tests if the correct confidence intervals are returned
datasets, _ = testdata_random()
matcher = make_combined_temporal_matcher(pd.Timedelta(6, "H"))
val = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=matcher,
metrics_calculators={
(4, 2): (
PairwiseIntercomparisonMetrics(
calc_spearman=True,
calc_kendall=True,
analytical_cis=True,
bootstrap_cis=True,
).calc_metrics
)
}
)
results_pw = val.calc(
[1], [1], [1], rename_cols=False, only_with_temporal_ref=True
)
metrics_with_ci = {
"BIAS": "bias",
"R": "pearson_r",
"rho": "spearman_r",
"tau": "kendall_tau",
"RMSD": "rmsd",
"urmsd": "ubrmsd",
"mse": "msd",
"mse_bias": "mse_bias",
}
metrics_with_bs_ci = {
"mse_corr": "mse_corr",
"mse_var": "mse_var",
}
# reconstruct dataframe
frames = []
for key in datasets:
frames.append(datasets[key]["class"].data)
data = pd.concat(frames, axis=1)
data.dropna(how="any", inplace=True)
for key in results_pw:
othername = key[0][0]
other_col = othername.split("_")[0]
other = data[other_col].values
refname = key[1][0]
ref_col = refname.split("_")[0]
ref = data[ref_col].values
for metric_key in metrics_with_ci:
lower = results_pw[key][f"{metric_key}_ci_lower"]
upper = results_pw[key][f"{metric_key}_ci_upper"]
# calculate manually from data
metric_func = getattr(pairwise, metrics_with_ci[metric_key])
m, lb, ub = with_analytical_ci(
metric_func, other, ref
)
# difference due to float32 vs. float64
assert_almost_equal(upper, ub, 6)
assert_almost_equal(lower, lb, 6)
for metric_key in metrics_with_bs_ci:
lower = results_pw[key][f"{metric_key}_ci_lower"]
upper = results_pw[key][f"{metric_key}_ci_upper"]
# calculate manually from data
metric_func = getattr(pairwise, metrics_with_bs_ci[metric_key])
m, lb, ub = with_bootstrapped_ci(
metric_func, other, ref
)
assert_allclose(upper, ub, rtol=1e-1, atol=1e-4)
assert_allclose(lower, lb, rtol=1e-1, atol=1e-4)
def test_ascat_ismn_validation_metadata_rolling():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'ascat', 'netcdf',
'55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'ascat', 'netcdf',
'grid')
static_layers_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'h_saf',
'static_layer')
ascat_reader = AscatSsmCdr(ascat_data_folder,
ascat_grid_folder,
grid_filename='TUW_WARP5_grid_info_2_1.nc',
static_layer_path=static_layers_folder)
ascat_reader.read_bulk = True
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'ismn', 'multinetwork',
'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(variable='soil moisture',
min_depth=0,
max_depth=0.1)
metadata_dict_template = {
'network': np.array(['None'], dtype='U256'),
'station': np.array(['None'], dtype='U256'),
'landcover': np.float32([np.nan]),
'climate': np.array(['None'], dtype='U4')
}
for idx in ids:
metadata = ismn_reader.metadata[idx]
metadata_dict = [{
'network': metadata['network'],
'station': metadata['station'],
'landcover': metadata['landcover_2010'],
'climate': metadata['climate']
}]
jobs.append(
(idx, metadata['longitude'], metadata['latitude'], metadata_dict))
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader,
'columns': ['soil moisture']
},
'ASCAT': {
'class': ascat_reader,
'columns': ['sm'],
'kwargs': {
'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True
}
}
}
read_ts_names = {'ASCAT': 'read', 'ISMN': 'read_ts'}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(datasets,
'ISMN',
period,
read_ts_names=read_ts_names)
process = Validation(
datasets,
'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2):
metrics_calculators.RollingMetrics(
other_name='k1',
metadata_template=metadata_dict_template).calc_metrics
},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results,
save_path,
ts_vars=['R', 'p_R', 'RMSD'])
results_fname = os.path.join(save_path,
'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [
u'gpi', u'lon', u'lat', u'R', u'p_R', u'time', u'idx', u'_row_size'
]
for key, value in metadata_dict_template.items():
vars_should.append(key)
network_should = np.array([
'MAQU', 'MAQU', 'SCAN', 'SCAN', 'SCAN', 'SOILSCAPE', 'SOILSCAPE',
'SOILSCAPE'
],
dtype='U256')
reader = PointDataResults(results_fname, read_only=True)
df = reader.read_loc(None)
nptest.assert_equal(sorted(network_should), sorted(df['network'].values))
assert np.all(df.gpi.values == np.arange(8))
assert (reader.read_ts(0).index.size == 357)
assert np.all(
reader.read_ts(1).columns.values == np.array(['R', 'p_R', 'RMSD']))
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'ascat', 'netcdf',
'55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'ascat', 'netcdf',
'grid')
static_layers_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'h_saf',
'static_layer')
ascat_reader = AscatSsmCdr(ascat_data_folder,
ascat_grid_folder,
grid_filename='TUW_WARP5_grid_info_2_1.nc',
static_layer_path=static_layers_folder)
ascat_reader.read_bulk = True
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'ismn', 'multinetwork',
'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader,
'columns': ['soil moisture']
},
'ASCAT': {
'class': ascat_reader,
'columns': ['sm'],
'kwargs': {
'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True
}
}
}
read_ts_names = {'ASCAT': 'read', 'ISMN': 'read_ts'}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(datasets,
'ISMN',
period,
read_ts_names=read_ts_names)
process = Validation(
datasets,
'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2):
metrics_calculators.BasicMetrics(other_name='k1').calc_metrics
},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(save_path,
'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [
u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau', u'BIAS', u'p_rho',
u'rho', u'lat', u'R', u'p_R', u'time', u'idx', u'_row_size'
]
n_obs_should = [384, 357, 482, 141, 251, 1927, 1887, 1652]
rho_should = np.array([
0.70022893, 0.53934574, 0.69356072, 0.84189808, 0.74206454, 0.30299741,
0.53143877, 0.62204134
],
dtype=np.float32)
rmsd_should = np.array([
7.72966719, 11.58347607, 14.57700157, 13.06224251, 12.90389824,
14.24668026, 21.19682884, 17.3883934
],
dtype=np.float32)
with nc.Dataset(results_fname, mode='r') as results:
assert sorted(list(results.variables.keys())) == sorted(vars_should)
assert sorted(
results.variables['n_obs'][:].tolist()) == sorted(n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]),
rtol=1e-4)
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]),
rtol=1e-4)
def test_validation_n3_k2():
tst_results = {
(('DS1', 'x'), ('DS3', 'y')): {
'n_obs': np.array([1000], dtype=np.int32),
'tau': np.array([np.nan], dtype=np.float32),
'gpi': np.array([4], dtype=np.int32),
'RMSD': np.array([0.], dtype=np.float32),
'lon': np.array([4.]),
'p_tau': np.array([np.nan], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)
},
(('DS1', 'x'), ('DS2', 'y')): {
'n_obs': np.array([1000], dtype=np.int32),
'tau': np.array([np.nan], dtype=np.float32),
'gpi': np.array([4], dtype=np.int32),
'RMSD': np.array([0.], dtype=np.float32),
'lon': np.array([4.]),
'p_tau': np.array([np.nan], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)
},
(('DS1', 'x'), ('DS3', 'x')): {
'n_obs': np.array([1000], dtype=np.int32),
'tau': np.array([np.nan], dtype=np.float32),
'gpi': np.array([4], dtype=np.int32),
'RMSD': np.array([0.], dtype=np.float32),
'lon': np.array([4.]),
'p_tau': np.array([np.nan], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'lat': np.array([4.]),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32)
},
(('DS2', 'y'), ('DS3', 'x')): {
'gpi': np.array([4], dtype=np.int32),
'lon': np.array([4.]),
'lat': np.array([4.]),
'n_obs': np.array([1000], dtype=np.int32),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'RMSD': np.array([0.], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'tau': np.array([np.nan], dtype=np.float32),
'p_tau': np.array([np.nan], dtype=np.float32)
},
(('DS2', 'y'), ('DS3', 'y')): {
'gpi': np.array([4], dtype=np.int32),
'lon': np.array([4.]),
'lat': np.array([4.]),
'n_obs': np.array([1000], dtype=np.int32),
'R': np.array([1.], dtype=np.float32),
'p_R': np.array([0.], dtype=np.float32),
'rho': np.array([1.], dtype=np.float32),
'p_rho': np.array([0.], dtype=np.float32),
'RMSD': np.array([0.], dtype=np.float32),
'BIAS': np.array([0.], dtype=np.float32),
'tau': np.array([np.nan], dtype=np.float32),
'p_tau': np.array([np.nan], dtype=np.float32)
}
}
datasets = setup_TestDatasets()
dm = DataManager(datasets,
'DS1',
read_ts_names={d: 'read'
for d in ['DS1', 'DS2', 'DS3']})
process = Validation(
dm,
'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(3, 2):
metrics_calculators.BasicMetrics(other_name='k1').calc_metrics
})
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
assert sorted(list(results)) == sorted(list(tst_results))
def create_pytesmo_validation(validation_run):
ds_list = []
ref_name = None
scaling_ref_name = None
ds_num = 1
for dataset_config in validation_run.dataset_configurations.all():
reader = create_reader(dataset_config.dataset, dataset_config.version)
reader = setup_filtering(
reader, list(dataset_config.filters.all()),
list(dataset_config.parametrisedfilter_set.all()),
dataset_config.dataset, dataset_config.variable)
if validation_run.anomalies == ValidationRun.MOVING_AVG_35_D:
reader = AnomalyAdapter(
reader,
window_size=35,
columns=[dataset_config.variable.pretty_name])
if validation_run.anomalies == ValidationRun.CLIMATOLOGY:
# make sure our baseline period is in UTC and without timezone information
anomalies_baseline = [
validation_run.anomalies_from.astimezone(tz=pytz.UTC).replace(
tzinfo=None),
validation_run.anomalies_to.astimezone(tz=pytz.UTC).replace(
tzinfo=None)
]
reader = AnomalyClimAdapter(
reader,
columns=[dataset_config.variable.pretty_name],
timespan=anomalies_baseline)
if ((validation_run.reference_configuration) and
(dataset_config.id == validation_run.reference_configuration.id)):
# reference is always named "0-..."
dataset_name = '{}-{}'.format(0, dataset_config.dataset.short_name)
else:
dataset_name = '{}-{}'.format(ds_num,
dataset_config.dataset.short_name)
ds_num += 1
ds_list.append((dataset_name, {
'class': reader,
'columns': [dataset_config.variable.pretty_name]
}))
if ((validation_run.reference_configuration) and
(dataset_config.id == validation_run.reference_configuration.id)):
ref_name = dataset_name
if ((validation_run.scaling_ref)
and (dataset_config.id == validation_run.scaling_ref.id)):
scaling_ref_name = dataset_name
datasets = dict(ds_list)
ds_num = len(ds_list)
period = None
if validation_run.interval_from is not None and validation_run.interval_to is not None:
## while pytesmo can't deal with timezones, normalise the validation period to utc; can be removed once pytesmo can do timezones
startdate = validation_run.interval_from.astimezone(UTC).replace(
tzinfo=None)
enddate = validation_run.interval_to.astimezone(UTC).replace(
tzinfo=None)
period = [startdate, enddate]
datamanager = DataManager(datasets,
ref_name=ref_name,
period=period,
read_ts_names='read')
ds_names = get_dataset_names(datamanager.reference_name,
datamanager.datasets,
n=ds_num)
if (len(ds_names) >= 3) and (validation_run.tcol is True):
# if there are 3 or more dataset, do TC, exclude ref metrics
metrics = TCMetrics(
dataset_names=ds_names,
tc_metrics_for_ref=False,
other_names=['k{}'.format(i + 1) for i in range(ds_num - 1)])
else:
metrics = IntercomparisonMetrics(
dataset_names=ds_names,
other_names=['k{}'.format(i + 1) for i in range(ds_num - 1)])
if validation_run.scaling_method == validation_run.NO_SCALING:
scaling_method = None
else:
scaling_method = validation_run.scaling_method
__logger.debug(f"Scaling method: {scaling_method}")
__logger.debug(f"Scaling dataset: {scaling_ref_name}")
val = Validation(datasets=datamanager,
spatial_ref=ref_name,
temporal_window=0.5,
scaling=scaling_method,
scaling_ref=scaling_ref_name,
metrics_calculators={
(ds_num, ds_num): metrics.calc_metrics
},
period=period)
return val
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', '55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', 'grid')
ascat_reader = AscatH25_SSM(ascat_data_folder, ascat_grid_folder)
ascat_reader.read_bulk = True
ascat_reader._load_grid_info()
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'ismn', 'multinetwork', 'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(
variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader,
'columns': ['soil moisture']
},
'ASCAT': {
'class': ascat_reader,
'columns': ['sm'],
'kwargs': {'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True}
}}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
process = Validation(
datasets, 'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, 'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau',
u'BIAS', u'p_rho', u'rho', u'lat', u'R', u'p_R']
n_obs_should = [360, 385, 1644, 1881, 1927, 479, 140, 251]
rho_should = np.array([0.546187,
0.717398,
0.620892,
0.532465,
0.302997,
0.694713,
0.840592,
0.742065],
dtype=np.float32)
rmsd_should = np.array([11.536263,
7.545650,
17.451935,
21.193714,
14.246680,
14.494674,
13.173215,
12.903898],
dtype=np.float32)
with nc.Dataset(results_fname) as results:
assert sorted(results.variables.keys()) == sorted(vars_should)
assert sorted(results.variables['n_obs'][:].tolist()) == sorted(
n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]),
rtol=1e-4)
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]),
rtol=1e-4)
def test_validation_n3_k2_masking():
# test result for one gpi in a cell
tst_results_one = {
(('DS1', 'x'), ('DS3', 'y')): {
'n_obs': np.array([250], dtype=np.int32)},
(('DS1', 'x'), ('DS2', 'y')): {
'n_obs': np.array([250], dtype=np.int32)},
(('DS1', 'x'), ('DS3', 'x')): {
'n_obs': np.array([250], dtype=np.int32)}}
# test result for two gpis in a cell
tst_results_two = {
(('DS1', 'x'), ('DS3', 'y')): {
'n_obs': np.array([250, 250], dtype=np.int32)},
(('DS1', 'x'), ('DS2', 'y')): {
'n_obs': np.array([250, 250], dtype=np.int32)},
(('DS1', 'x'), ('DS3', 'x')): {
'n_obs': np.array([250, 250], dtype=np.int32)}}
# cell 4 in this example has two gpis so it returns different results.
tst_results = {1: tst_results_one,
1: tst_results_one,
2: tst_results_two}
datasets = setup_TestDatasets()
# setup masking datasets
grid = grids.CellGrid(np.array([1, 2, 3, 4]), np.array([1, 2, 3, 4]),
np.array([4, 4, 2, 1]), gpis=np.array([1, 2, 3, 4]))
mds1 = GriddedTsBase("", grid, MaskingTestDataset)
mds2 = GriddedTsBase("", grid, MaskingTestDataset)
mds = {
'masking1': {
'class': mds1,
'columns': ['x'],
'args': [],
'kwargs': {'limit': 500},
'use_lut': False,
'grids_compatible': True},
'masking2': {
'class': mds2,
'columns': ['x'],
'args': [],
'kwargs': {'limit': 750},
'use_lut': False,
'grids_compatible': True}
}
process = Validation(
datasets, 'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(3, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics},
masking_datasets=mds)
gpi_info = (1, 1, 1)
ref_df = datasets['DS1']['class'].read_ts(1)
new_ref_df = process.mask_dataset(ref_df, gpi_info)
assert len(new_ref_df) == 250
nptest.assert_allclose(new_ref_df.x.values, np.arange(750, 1000))
jobs = process.get_processing_jobs()
for job in jobs:
results = process.calc(*job)
tst = tst_results[len(job[0])]
assert sorted(list(results)) == sorted(list(tst))
for key, tst_key in zip(sorted(results),
sorted(tst)):
nptest.assert_almost_equal(results[key]['n_obs'],
tst[tst_key]['n_obs'])
def create_pytesmo_validation(validation_run):
ds_list = []
ref_name = None
scaling_ref_name = None
ds_num = 1
for dataset_config in validation_run.dataset_configurations.all():
reader = create_reader(dataset_config.dataset, dataset_config.version)
reader = setup_filtering(
reader, list(dataset_config.filters.all()),
list(dataset_config.parametrisedfilter_set.all()),
dataset_config.dataset, dataset_config.variable)
if validation_run.anomalies == ValidationRun.MOVING_AVG_35_D:
reader = AnomalyAdapter(
reader,
window_size=35,
columns=[dataset_config.variable.pretty_name])
if validation_run.anomalies == ValidationRun.CLIMATOLOGY:
# make sure our baseline period is in UTC and without timezone information
anomalies_baseline = [
validation_run.anomalies_from.astimezone(tz=pytz.UTC).replace(
tzinfo=None),
validation_run.anomalies_to.astimezone(tz=pytz.UTC).replace(
tzinfo=None)
]
reader = AnomalyClimAdapter(
reader,
columns=[dataset_config.variable.pretty_name],
timespan=anomalies_baseline)
if (validation_run.reference_configuration and
(dataset_config.id == validation_run.reference_configuration.id)):
# reference is always named "0-..."
dataset_name = '{}-{}'.format(0, dataset_config.dataset.short_name)
else:
dataset_name = '{}-{}'.format(ds_num,
dataset_config.dataset.short_name)
ds_num += 1
ds_list.append((dataset_name, {
'class': reader,
'columns': [dataset_config.variable.pretty_name]
}))
if (validation_run.reference_configuration and
(dataset_config.id == validation_run.reference_configuration.id)):
ref_name = dataset_name
ref_short_name = validation_run.reference_configuration.dataset.short_name
if (validation_run.scaling_ref
and (dataset_config.id == validation_run.scaling_ref.id)):
scaling_ref_name = dataset_name
datasets = dict(ds_list)
ds_num = len(ds_list)
period = None
if validation_run.interval_from is not None and validation_run.interval_to is not None:
# while pytesmo can't deal with timezones, normalise the validation period to utc; can be removed once pytesmo can do timezones
startdate = validation_run.interval_from.astimezone(UTC).replace(
tzinfo=None)
enddate = validation_run.interval_to.astimezone(UTC).replace(
tzinfo=None)
period = [startdate, enddate]
upscale_parms = None
if validation_run.upscaling_method != "none":
__logger.debug("Upscaling option is active")
upscale_parms = {
"upscaling_method": validation_run.upscaling_method,
"temporal_stability": validation_run.temporal_stability,
}
upscaling_lut = create_upscaling_lut(
validation_run=validation_run,
datasets=datasets,
ref_name=ref_name,
)
upscale_parms["upscaling_lut"] = upscaling_lut
__logger.debug("Lookup table for non-reference datasets " +
", ".join(upscaling_lut.keys()) + " created")
__logger.debug("{}".format(upscaling_lut))
datamanager = DataManager(
datasets,
ref_name=ref_name,
period=period,
read_ts_names='read',
upscale_parms=upscale_parms,
)
ds_names = get_dataset_names(datamanager.reference_name,
datamanager.datasets,
n=ds_num)
# set value of the metadata template according to what reference dataset is used
if ref_short_name == 'ISMN':
metadata_template = METADATA_TEMPLATE['ismn_ref']
else:
metadata_template = METADATA_TEMPLATE['other_ref']
pairwise_metrics = PairwiseIntercomparisonMetrics(
metadata_template=metadata_template,
calc_kendall=False,
)
metric_calculators = {(ds_num, 2): pairwise_metrics.calc_metrics}
if (len(ds_names) >= 3) and (validation_run.tcol is True):
tcol_metrics = TripleCollocationMetrics(
ref_name,
metadata_template=metadata_template,
bootstrap_cis=validation_run.bootstrap_tcol_cis)
metric_calculators.update({(ds_num, 3): tcol_metrics.calc_metrics})
if validation_run.scaling_method == validation_run.NO_SCALING:
scaling_method = None
else:
scaling_method = validation_run.scaling_method
__logger.debug(f"Scaling method: {scaling_method}")
__logger.debug(f"Scaling dataset: {scaling_ref_name}")
val = Validation(datasets=datamanager,
temporal_matcher=make_combined_temporal_matcher(
pd.Timedelta(12, "H")),
spatial_ref=ref_name,
scaling=scaling_method,
scaling_ref=scaling_ref_name,
metrics_calculators=metric_calculators,
period=period)
return val
def test_validation_n3_k2_masking_no_data_remains():
datasets = setup_TestDatasets()
# setup masking datasets
grid = grids.CellGrid(
np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4]),
np.array([4, 4, 2, 1]),
gpis=np.array([1, 2, 3, 4]),
)
mds1 = GriddedTsBase("", grid, MaskingTestDataset)
mds2 = GriddedTsBase("", grid, MaskingTestDataset)
mds = {
"masking1": {
"class": mds1,
"columns": ["x"],
"args": [],
"kwargs": {
"limit": 500
},
"use_lut": False,
"grids_compatible": True,
},
"masking2": {
"class": mds2,
"columns": ["x"],
"args": [],
"kwargs": {
"limit": 1000
},
"use_lut": False,
"grids_compatible": True,
},
}
process = Validation(
datasets,
"DS1",
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling="lin_cdf_match",
metrics_calculators={
(3, 2):
metrics_calculators.BasicMetrics(other_name="k1").calc_metrics
},
masking_datasets=mds,
)
gpi_info = (1, 1, 1)
ref_df = datasets["DS1"]["class"].read(1)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
new_ref_df = process.mask_dataset(ref_df, gpi_info)
assert len(new_ref_df) == 0
nptest.assert_allclose(new_ref_df.x.values, np.arange(1000, 1000))
jobs = process.get_processing_jobs()
for job in jobs:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
results = process.calc(*job)
tst = []
assert sorted(list(results)) == sorted(list(tst))
for key, tst_key in zip(sorted(results), sorted(tst)):
nptest.assert_almost_equal(results[key]["n_obs"],
tst[tst_key]["n_obs"])
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), 'test-data',
'sat', 'ascat', 'netcdf', '55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), 'test-data',
'sat', 'ascat', 'netcdf', 'grid')
ascat_reader = AscatH25_SSM(ascat_data_folder, ascat_grid_folder)
ascat_reader.read_bulk = True
ascat_reader._load_grid_info()
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), 'test-data',
'ismn', 'multinetwork', 'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(
variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader, 'columns': [
'soil moisture'
], 'type': 'reference', 'args': [], 'kwargs': {}
},
'ASCAT': {
'class': ascat_reader, 'columns': [
'sm'
], 'type': 'other', 'args': [], 'kwargs': {}, 'grids_compatible':
False, 'use_lut': False, 'lut_max_dist': 30000
}
}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
process = Validation(
datasets=datasets,
data_prep=DataPreparation(),
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0,
reverse=True),
scaling='lin_cdf_match',
scale_to_other=True,
metrics_calculator=metrics_calculators.BasicMetrics(),
period=period,
cell_based_jobs=False)
for job in jobs:
results = process.calc(job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, 'ISMN.soil moisture_with_ASCAT.sm.nc')
vars_should = [u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau',
u'BIAS', u'p_rho', u'rho', u'lat', u'R', u'p_R']
n_obs_should = [360, 385, 1644, 1881, 1927, 479, 140, 251]
rho_should = np.array([0.54618734, 0.71739876, 0.62089276, 0.53246528,
0.30299741, 0.69647062, 0.840593, 0.73913699],
dtype=np.float32)
rmsd_should = np.array([11.53626347, 7.54565048, 17.45193481, 21.19371414,
14.24668026, 14.27493, 13.173215, 12.59192371],
dtype=np.float32)
with nc.Dataset(results_fname) as results:
assert sorted(results.variables.keys()) == sorted(vars_should)
assert sorted(results.variables['n_obs'][:].tolist()) == sorted(
n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]))
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]))
def test_validation_n3_k2_masking():
# test result for one gpi in a cell
tst_results_one = {
(("DS1", "x"), ("DS3", "y")): {
"n_obs": np.array([250], dtype=np.int32)
},
(("DS1", "x"), ("DS2", "y")): {
"n_obs": np.array([250], dtype=np.int32)
},
(("DS1", "x"), ("DS3", "x")): {
"n_obs": np.array([250], dtype=np.int32)
},
(("DS2", "y"), ("DS3", "x")): {
"n_obs": np.array([250], dtype=np.int32)
},
(("DS2", "y"), ("DS3", "y")): {
"n_obs": np.array([250], dtype=np.int32)
},
}
# test result for two gpis in a cell
tst_results_two = {
(("DS1", "x"), ("DS3", "y")): {
"n_obs": np.array([250, 250], dtype=np.int32)
},
(("DS1", "x"), ("DS2", "y")): {
"n_obs": np.array([250, 250], dtype=np.int32)
},
(("DS1", "x"), ("DS3", "x")): {
"n_obs": np.array([250, 250], dtype=np.int32)
},
(("DS2", "y"), ("DS3", "x")): {
"n_obs": np.array([250, 250], dtype=np.int32)
},
(("DS2", "y"), ("DS3", "y")): {
"n_obs": np.array([250, 250], dtype=np.int32)
},
}
# cell 4 in this example has two gpis so it returns different results.
tst_results = {1: tst_results_one, 1: tst_results_one, 2: tst_results_two}
datasets = setup_TestDatasets()
# setup masking datasets
grid = grids.CellGrid(
np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4]),
np.array([4, 4, 2, 1]),
gpis=np.array([1, 2, 3, 4]),
)
mds1 = GriddedTsBase("", grid, MaskingTestDataset)
mds2 = GriddedTsBase("", grid, MaskingTestDataset)
mds = {
"masking1": {
"class": mds1,
"columns": ["x"],
"args": [],
"kwargs": {
"limit": 500
},
"use_lut": False,
"grids_compatible": True,
},
"masking2": {
"class": mds2,
"columns": ["x"],
"args": [],
"kwargs": {
"limit": 750
},
"use_lut": False,
"grids_compatible": True,
},
}
process = Validation(
datasets,
"DS1",
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling="lin_cdf_match",
metrics_calculators={
(3, 2):
metrics_calculators.BasicMetrics(other_name="k1").calc_metrics
},
masking_datasets=mds,
)
gpi_info = (1, 1, 1)
ref_df = datasets["DS1"]["class"].read(1)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=DeprecationWarning
) # read_ts is hard coded when using mask_data
new_ref_df = process.mask_dataset(ref_df, gpi_info)
assert len(new_ref_df) == 250
nptest.assert_allclose(new_ref_df.x.values, np.arange(750, 1000))
jobs = process.get_processing_jobs()
for job in jobs:
with warnings.catch_warnings():
# most warnings here are caused by the read_ts function that cannot
# be changed when using a masking data set
warnings.simplefilter("ignore", category=DeprecationWarning)
results = process.calc(*job)
tst = tst_results[len(job[0])]
assert sorted(list(results)) == sorted(list(tst))
for key, tst_key in zip(sorted(results), sorted(tst)):
nptest.assert_almost_equal(results[key]["n_obs"],
tst[tst_key]["n_obs"])
'kwargs': {'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True}}
}
# The datasets dictionary contains all the information about the datasets to read. The `class` is the dataset class to use which we have already initialized. The `columns` key describes which columns of the dataset interest us for validation. This a mandatory field telling the framework which other columns to ignore. In this case the columns `soil moisture_flag` and `soil moisture_orig_flag` will be ignored by the ISMN reader. We can also specify additional keywords that should be given to the `read_ts` method of the dataset reader. In this case we want the ASCAT reader to mask the ASCAT soil moisture using the included frozen and snow probabilities as well as the SSF. There are also other keys that can be used here. Please see the documentation for explanations.
# In[13]:
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
basic_metrics = metrics_calculators.BasicMetrics(other_name='k1')
process = Validation(
datasets, 'ISMN', {(2, 2): basic_metrics.calc_metrics},
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
period=period)
# During the initialization of the Validation class we can also tell it other things that it needs to know. In this case it uses the datasets we have specified earlier. The spatial reference is the `'ISMN'` dataset which is the second argument. The third argument looks a little bit strange so let's look at it in more detail.
#
# It is a dictionary with a tuple as the key and a function as the value. The key tuple `(n, k)` has the following meaning: `n` datasets are temporally matched together and then given in sets of `k` columns to the metric calculator. The metric calculator then gets a DataFrame with the columns ['ref', 'k1', 'k2' ...] and so on depending on the value of k. The value of `(2, 2)` makes sense here since we only have two datasets and all our metrics also take two inputs.
#
# This can be used in more complex scenarios to e.g. have three input datasets that are all temporally matched together and then combinations of two input datasets are given to one metric calculator while all three datasets are given to another metric calculator. This could look like this:
#
# ```python
# { (3 ,2): metric_calc,
# (3, 3): triple_collocation}
# ```
def test_validation_n3_k2_masking_no_data_remains():
datasets = setup_TestDatasets()
# setup masking datasets
grid = grids.CellGrid(np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4]),
np.array([4, 4, 2, 1]),
gpis=np.array([1, 2, 3, 4]))
mds1 = GriddedTsBase("", grid, MaskingTestDataset)
mds2 = GriddedTsBase("", grid, MaskingTestDataset)
mds = {
'masking1': {
'class': mds1,
'columns': ['x'],
'args': [],
'kwargs': {
'limit': 500
},
'use_lut': False,
'grids_compatible': True
},
'masking2': {
'class': mds2,
'columns': ['x'],
'args': [],
'kwargs': {
'limit': 1000
},
'use_lut': False,
'grids_compatible': True
}
}
process = Validation(
datasets,
'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(3, 2):
metrics_calculators.BasicMetrics(other_name='k1').calc_metrics
},
masking_datasets=mds)
gpi_info = (1, 1, 1)
ref_df = datasets['DS1']['class'].read(1)
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning)
new_ref_df = process.mask_dataset(ref_df, gpi_info)
assert len(new_ref_df) == 0
nptest.assert_allclose(new_ref_df.x.values, np.arange(1000, 1000))
jobs = process.get_processing_jobs()
for job in jobs:
with warnings.catch_warnings():
warnings.filterwarnings('ignore', category=DeprecationWarning)
results = process.calc(*job)
tst = []
assert sorted(list(results)) == sorted(list(tst))
for key, tst_key in zip(sorted(results), sorted(tst)):
nptest.assert_almost_equal(results[key]['n_obs'],
tst[tst_key]['n_obs'])
def test_ascat_ismn_validation_metadata_rolling(ascat_reader, ismn_reader):
"""
Test processing framework with some ISMN and ASCAT sample data
"""
jobs = []
ids = ismn_reader.get_dataset_ids(variable="soil moisture",
min_depth=0,
max_depth=0.1)
metadata_dict_template = {
"network": np.array(["None"], dtype="U256"),
"station": np.array(["None"], dtype="U256"),
"landcover": np.float32([np.nan]),
"climate": np.array(["None"], dtype="U4"),
}
for idx in ids:
metadata = ismn_reader.metadata[idx]
metadata_dict = [{
"network": metadata["network"],
"station": metadata["station"],
"landcover": metadata["landcover_2010"],
"climate": metadata["climate"],
}]
jobs.append(
(idx, metadata["longitude"], metadata["latitude"], metadata_dict))
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
"ISMN": {
"class": ismn_reader,
"columns": ["soil moisture"]
},
"ASCAT": {
"class": ascat_reader,
"columns": ["sm"],
"kwargs": {
"mask_frozen_prob": 80,
"mask_snow_prob": 80,
"mask_ssf": True,
},
},
}
read_ts_names = {"ASCAT": "read", "ISMN": "read_ts"}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(datasets,
"ISMN",
period,
read_ts_names=read_ts_names)
process = Validation(
datasets,
"ISMN",
temporal_ref="ASCAT",
scaling="lin_cdf_match",
scaling_ref="ASCAT",
metrics_calculators={
(2, 2):
metrics_calculators.RollingMetrics(
other_name="k1",
metadata_template=metadata_dict_template).calc_metrics
},
period=period,
)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results,
save_path,
ts_vars=["R", "p_R", "RMSD"])
results_fname = os.path.join(save_path,
"ASCAT.sm_with_ISMN.soil moisture.nc")
target_vars = {
"network":
np.array(
[
"MAQU",
"MAQU",
"SCAN",
"SCAN",
"SCAN",
"SOILSCAPE",
"SOILSCAPE",
"SOILSCAPE",
],
dtype="U256",
)
}
vars_should = [
u"gpi", u"RMSD", u"lon", u"lat", u"R", u"p_R", u"time", u"idx",
u"_row_size"
]
for key, value in metadata_dict_template.items():
vars_should.append(key)
check_results(filename=results_fname,
target_vars=target_vars,
variables=vars_should)
reader = PointDataResults(results_fname, read_only=True)
df = reader.read_loc(None)
assert np.all(df.gpi.values == np.arange(8))
assert reader.read_ts(0).index.size == 357
assert np.all(
reader.read_ts(1).columns.values == np.array(["R", "p_R", "RMSD"]))
def test_validation_n3_k2_masking():
# test result for one gpi in a cell
tst_results_one = {
(('DS1', 'x'), ('DS3', 'y')): {
'n_obs': np.array([250], dtype=np.int32)
},
(('DS1', 'x'), ('DS2', 'y')): {
'n_obs': np.array([250], dtype=np.int32)
},
(('DS1', 'x'), ('DS3', 'x')): {
'n_obs': np.array([250], dtype=np.int32)
},
(('DS2', 'y'), ('DS3', 'x')): {
'n_obs': np.array([250], dtype=np.int32)
},
(('DS2', 'y'), ('DS3', 'y')): {
'n_obs': np.array([250], dtype=np.int32)
}
}
# test result for two gpis in a cell
tst_results_two = {
(('DS1', 'x'), ('DS3', 'y')): {
'n_obs': np.array([250, 250], dtype=np.int32)
},
(('DS1', 'x'), ('DS2', 'y')): {
'n_obs': np.array([250, 250], dtype=np.int32)
},
(('DS1', 'x'), ('DS3', 'x')): {
'n_obs': np.array([250, 250], dtype=np.int32)
},
(('DS2', 'y'), ('DS3', 'x')): {
'n_obs': np.array([250, 250], dtype=np.int32)
},
(('DS2', 'y'), ('DS3', 'y')): {
'n_obs': np.array([250, 250], dtype=np.int32)
}
}
# cell 4 in this example has two gpis so it returns different results.
tst_results = {1: tst_results_one, 1: tst_results_one, 2: tst_results_two}
datasets = setup_TestDatasets()
# setup masking datasets
grid = grids.CellGrid(np.array([1, 2, 3, 4]),
np.array([1, 2, 3, 4]),
np.array([4, 4, 2, 1]),
gpis=np.array([1, 2, 3, 4]))
mds1 = GriddedTsBase("", grid, MaskingTestDataset)
mds2 = GriddedTsBase("", grid, MaskingTestDataset)
mds = {
'masking1': {
'class': mds1,
'columns': ['x'],
'args': [],
'kwargs': {
'limit': 500
},
'use_lut': False,
'grids_compatible': True
},
'masking2': {
'class': mds2,
'columns': ['x'],
'args': [],
'kwargs': {
'limit': 750
},
'use_lut': False,
'grids_compatible': True
}
}
process = Validation(
datasets,
'DS1',
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0).combinatory_matcher,
scaling='lin_cdf_match',
metrics_calculators={
(3, 2):
metrics_calculators.BasicMetrics(other_name='k1').calc_metrics
},
masking_datasets=mds)
gpi_info = (1, 1, 1)
ref_df = datasets['DS1']['class'].read(1)
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=DeprecationWarning
) # read_ts is hard coded when using mask_data
new_ref_df = process.mask_dataset(ref_df, gpi_info)
assert len(new_ref_df) == 250
nptest.assert_allclose(new_ref_df.x.values, np.arange(750, 1000))
jobs = process.get_processing_jobs()
for job in jobs:
with warnings.catch_warnings():
# most warnings here are caused by the read_ts function that cannot
# be changed when using a masking data set
warnings.simplefilter('ignore', category=DeprecationWarning)
results = process.calc(*job)
tst = tst_results[len(job[0])]
assert sorted(list(results)) == sorted(list(tst))
for key, tst_key in zip(sorted(results), sorted(tst)):
nptest.assert_almost_equal(results[key]['n_obs'],
tst[tst_key]['n_obs'])
def test_ascat_ismn_validation(ascat_reader, ismn_reader):
"""
Test processing framework with some ISMN and ASCAT sample data
"""
jobs = []
ids = ismn_reader.get_dataset_ids(variable="soil moisture",
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata["longitude"], metadata["latitude"]))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
"ISMN": {
"class": ismn_reader,
"columns": ["soil moisture"]
},
"ASCAT": {
"class": ascat_reader,
"columns": ["sm"],
"kwargs": {
"mask_frozen_prob": 80,
"mask_snow_prob": 80,
"mask_ssf": True,
},
},
}
read_ts_names = {"ASCAT": "read", "ISMN": "read_ts"}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(datasets,
"ISMN",
period,
read_ts_names=read_ts_names)
process = Validation(
datasets,
"ISMN",
temporal_ref="ASCAT",
scaling="lin_cdf_match",
scaling_ref="ASCAT",
metrics_calculators={
(2, 2):
metrics_calculators.BasicMetrics(other_name="k1").calc_metrics
},
period=period,
)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(save_path,
"ASCAT.sm_with_ISMN.soil moisture.nc")
# targets
target_vars = {
"n_obs": [357, 384, 1646, 1875, 1915, 467, 141, 251],
"rho":
np.array([
0.53934574, 0.7002289, 0.62200236, 0.53647155, 0.30413666,
0.6740655, 0.8418981, 0.74206454
],
dtype=np.float32),
"RMSD":
np.array([
11.583476, 7.729667, 17.441547, 21.125721, 14.31557, 14.187225,
13.0622425, 12.903898
],
dtype=np.float32)
}
check_results(
filename=results_fname,
target_vars=target_vars,
)
def test_PairwiseIntercomparisonMetrics(testdata_generator):
# This test first compares the PairwiseIntercomparisonMetrics to known
# results and then confirms that it agrees with IntercomparisonMetrics as
# expected
datasets, expected = testdata_generator()
# for the pairwise intercomparison metrics it's important that we use
# make_combined_temporal_matcher
val = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=make_combined_temporal_matcher(pd.Timedelta(6, "H")),
metrics_calculators={
(4, 2): (
PairwiseIntercomparisonMetrics(
calc_spearman=True, analytical_cis=False
).calc_metrics
)
}
)
results_pw = val.calc(
[1], [1], [1], rename_cols=False, only_with_temporal_ref=True
)
# in results_pw, there are four entries with keys (("c1name", "c1"),
# ("refname", "ref"), and so on.
# Each value is a single dictionary with the values of the metrics
expected_metrics = [
"R", "p_R", "BIAS", "RMSD", "mse", "RSS", "mse_corr", "mse_bias",
"urmsd", "mse_var", "n_obs", "gpi", "lat", "lon", "rho", "p_rho",
"tau", "p_tau"
]
for key in results_pw:
assert isinstance(key, tuple)
assert len(key) == 2
assert all(map(lambda x: isinstance(x, tuple), key))
assert isinstance(results_pw[key], dict)
assert sorted(expected_metrics) == sorted(results_pw[key].keys())
for m in expected_metrics:
if m in expected[key]:
assert_equal(results_pw[key][m], expected[key][m])
# preparation of IntercomparisonMetrics run for comparison
ds_names = list(datasets.keys())
metrics = IntercomparisonMetrics(
dataset_names=ds_names,
# passing the names here explicitly, see GH issue #220
refname="reference_name",
other_names=ds_names[1:],
calc_tau=True,
)
val = Validation(
datasets,
"reference_name",
scaling=None,
temporal_matcher=None, # use default here
metrics_calculators={(4, 4): metrics.calc_metrics}
)
print("running old setup")
results = val.calc(1, 1, 1, rename_cols=False)
# results is a dictionary with one entry and key
# (('c1name', 'c1'), ('c2name', 'c2'), ('c3name', 'c3'), ('refname',
# 'ref')), the value is a list of length 0, which contains a dictionary
# with all the results, where the metrics are joined with "_between_" with
# the combination of datasets, which is joined with "_and_", e.g. for R
# between ``refname`` and ``c1name`` the key is
# "R_between_refname_and_c1name"
common_metrics = ["n_obs", "gpi", "lat", "lon"]
pw_metrics = list(set(expected_metrics) - set(common_metrics))
# there's some sorting done at some point in pytesmo
oldkey = tuple(sorted([(name, name.split("_")[0]) for name in ds_names]))
res_old = results[oldkey]
for key in results_pw:
res = results_pw[key]
# handle the full dataset metrics
for m in common_metrics:
assert_equal(res[m], res_old[m])
# now get the metrics and compare to the right combination
for m in pw_metrics:
othername = key[0][0]
refname = key[1][0]
if othername == "reference_name":
# sorting might be different, see GH #220
othername = key[1][0]
refname = key[0][0]
old_m_key = f"{m}_between_{refname}_and_{othername}"
if m == "BIAS":
# PairwiseIntercomparisonMetrics has the result as (other,
# ref), and therefore "bias between other and ref", compared to
# "bias between ref and bias" in IntercomparisonMetrics
# this is related to issue #220
assert_equal(np.abs(res[m]), np.abs(res_old[old_m_key]))
elif m == "urmsd":
# the old implementation differs from the new implementation
pass
else:
assert_equal(res[m], res_old[old_m_key])
def test_ascat_ismn_validation_metadata(ascat_reader, ismn_reader):
"""
Test processing framework with some ISMN and ASCAT sample data
"""
jobs = []
ids = ismn_reader.get_dataset_ids(variable="soil moisture",
min_depth=0,
max_depth=0.1)
metadata_dict_template = {
"network": np.array(["None"], dtype="U256"),
"station": np.array(["None"], dtype="U256"),
"landcover": np.float32([np.nan]),
"climate": np.array(["None"], dtype="U4"),
}
for idx in ids:
metadata = ismn_reader.metadata[idx]
metadata_dict = [{
"network": metadata["network"],
"station": metadata["station"],
"landcover": metadata["landcover_2010"],
"climate": metadata["climate"],
}]
jobs.append(
(idx, metadata["longitude"], metadata["latitude"], metadata_dict))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
"ISMN": {
"class": ismn_reader,
"columns": ["soil moisture"],
},
"ASCAT": {
"class": ascat_reader,
"columns": ["sm"],
"kwargs": {
"mask_frozen_prob": 80,
"mask_snow_prob": 80,
"mask_ssf": True,
},
},
}
read_ts_names = {"ASCAT": "read", "ISMN": "read_ts"}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(datasets,
"ISMN",
period,
read_ts_names=read_ts_names)
process = Validation(
datasets,
"ISMN",
temporal_ref="ASCAT",
scaling="lin_cdf_match",
scaling_ref="ASCAT",
metrics_calculators={
(2, 2):
metrics_calculators.BasicMetrics(
other_name="k1",
metadata_template=metadata_dict_template).calc_metrics
},
period=period,
)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(save_path,
"ASCAT.sm_with_ISMN.soil moisture.nc")
target_vars = {
"n_obs": [357, 384, 1646, 1875, 1915, 467, 141, 251],
"rho":
np.array([
0.53934574,
0.7002289,
0.62200236,
0.53647155,
0.30413666,
0.6740655,
0.8418981,
0.74206454,
],
dtype=np.float32),
"RMSD":
np.array([
11.583476,
7.729667,
17.441547,
21.125721,
14.31557,
14.187225,
13.0622425,
12.903898,
],
dtype=np.float32),
"network":
np.array(
[
"MAQU",
"MAQU",
"SCAN",
"SCAN",
"SCAN",
"SOILSCAPE",
"SOILSCAPE",
"SOILSCAPE",
],
dtype="U256",
)
}
vars_should = [
'BIAS', 'R', 'RMSD', '_row_size', 'climate', 'gpi', 'idx', 'landcover',
'lat', 'lon', 'n_obs', 'network', 'p_R', 'p_rho', 'p_tau', 'rho',
'station', 'tau', 'time'
]
check_results(filename=results_fname,
target_vars=target_vars,
variables=vars_should)
def test_TripleCollocationMetrics(testdata_generator):
# tests by comparison of pairwise metrics to triplet metrics
datasets, expected = testdata_generator()
refname = "reference_name"
othernames = list(datasets.keys())
othernames.remove(refname)
triplet_metrics_calculator = TripleCollocationMetrics(
refname, bootstrap_cis=False
)
matcher = make_combined_temporal_matcher(pd.Timedelta(6, "H"))
val_triplet = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=matcher,
metrics_calculators={
(4, 3): triplet_metrics_calculator.calc_metrics
}
)
results_triplet = val_triplet.calc(
[1], [1], [1], rename_cols=False, only_with_temporal_ref=True
)
if "col1_name" in datasets.keys():
# we only test the TCA results with the random data, since for the
# constant data all covariances are zero and TCA therefore doesn't
# work.
for metric in ["snr", "err_std", "beta"]:
for dset in datasets:
values = []
dkey = (dset, datasets[dset]["columns"][0])
for tkey in results_triplet:
if dkey in tkey:
values.append(results_triplet[tkey][(metric, dset)][0])
diff = np.abs(np.diff(values))
assert diff.max() / values[0] < 0.1
# check if writing to file works
results_path = Path("__test_results")
# if this throws, there's either some data left over from previous tests,
# or some data is named __test_results. Remove the __test_results directory
# from your current directory to make the test work again.
assert not results_path.exists()
results_path.mkdir(exist_ok=True, parents=True)
netcdf_results_manager(results_triplet, results_path.name)
assert results_path.exists()
for key in results_triplet:
fname = "_with_".join(map(lambda t: ".".join(t), key)) + ".nc"
assert (results_path / fname).exists()
# res = xr.open_dataset(results_path / fname)
# for metric in ["snr", "err_std", "beta"]:
# for dset, _ in key:
# mkey = metric + "__" + dset
# assert mkey in res.data_vars
shutil.rmtree(results_path)
# now with CIs, again only for random data
if "col1_name" in datasets.keys():
triplet_metrics_calculator = TripleCollocationMetrics(
refname, bootstrap_cis=True
)
val_triplet = Validation(
datasets,
"reference_name",
scaling=None, # doesn't work with the constant test data
temporal_matcher=matcher,
metrics_calculators={
(4, 3): triplet_metrics_calculator.calc_metrics
}
)
results_triplet = val_triplet.calc(
[1], [1], [1], rename_cols=False, only_with_temporal_ref=True
)
for key in results_triplet:
for dset, _ in key:
for metric in ["snr", "err_std", "beta"]:
lkey = f"{metric}_ci_lower"
ukey = f"{metric}_ci_upper"
assert (lkey, dset) in results_triplet[key]
assert (ukey, dset) in results_triplet[key]
assert (
results_triplet[key][(lkey, dset)]
<= results_triplet[key][(metric, dset)]
)
assert (
results_triplet[key][(metric, dset)]
<= results_triplet[key][(ukey, dset)]
)
def test_validation_with_averager(ascat_reader, ismn_reader):
"""
Test processing framework with averaging module. ASCAT and ISMN data are used here with no geographical
considerations (the lut is provided more upstream and contains this information already)
"""
while hasattr(ascat_reader, 'cls'):
ascat_reader = ascat_reader.cls
# lookup table between the ascat and ismn points - not geographically correct
upscaling_lut = {
"ISMN": {
1814367: [(0, 102.1333, 33.8833), (1, 102.1333, 33.6666)],
1803695: [(2, -86.55, 34.783), (3, -97.083, 37.133),
(4, -105.417, 34.25)],
1856312: [(5, -120.9675, 38.43003), (6, -120.78559, 38.14956),
(7, -120.80639, 38.17353)]
}
}
gpis = (1814367, 1803695, 1856312)
lons, lats = [], []
for gpi in gpis:
lon, lat = ascat_reader.grid.gpi2lonlat(gpi)
lons.append(lon)
lats.append(lat)
jobs = [(gpis, lons, lats)]
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
"ASCAT": {
"class": ascat_reader,
"columns": ["sm"],
"kwargs": {
"mask_frozen_prob": 80,
"mask_snow_prob": 80,
"mask_ssf": True,
}
},
"ISMN": {
"class": ismn_reader,
"columns": ["soil moisture"],
},
}
read_ts_names = {"ASCAT": "read", "ISMN": "read_ts"}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(
datasets,
"ASCAT",
period,
read_ts_names=read_ts_names,
upscale_parms={
"upscaling_method": "average",
"temporal_stability": True,
"upscaling_lut": upscaling_lut,
},
)
process = Validation(
datasets,
"ASCAT",
temporal_ref="ISMN",
scaling="lin_cdf_match",
scaling_ref="ISMN",
metrics_calculators={
(2, 2):
metrics_calculators.BasicMetrics(other_name="k1").calc_metrics
},
period=period,
)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(save_path,
"ASCAT.sm_with_ISMN.soil moisture.nc")
target_vars = {
"n_obs": [764, 2392, 904],
"rho": np.array([-0.012487, 0.255156, 0.635517], dtype=np.float32),
"RMSD": np.array([0.056428, 0.056508, 0.116294], dtype=np.float32),
"R": np.array([-0.012335, 0.257671, 0.657239], dtype=np.float32)
}
check_results(
filename=results_fname,
target_vars=target_vars,
)
def test_ascat_ismn_validation(ascat_reader):
"""
Test processing framework with some ISMN and ASCAT sample data
"""
# Initialize ISMN reader
ismn_data_folder = os.path.join(
os.path.dirname(__file__),
"..",
"test-data",
"ismn",
"multinetwork",
"header_values",
)
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(
variable="soil moisture", min_depth=0, max_depth=0.1
)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata["longitude"], metadata["latitude"]))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
"ISMN": {"class": ismn_reader, "columns": ["soil moisture"]},
"ASCAT": {
"class": ascat_reader,
"columns": ["sm"],
"kwargs": {
"mask_frozen_prob": 80,
"mask_snow_prob": 80,
"mask_ssf": True,
},
},
}
read_ts_names = {"ASCAT": "read", "ISMN": "read_ts"}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
datasets = DataManager(
datasets, "ISMN", period, read_ts_names=read_ts_names
)
process = Validation(
datasets,
"ISMN",
temporal_ref="ASCAT",
scaling="lin_cdf_match",
scaling_ref="ASCAT",
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(
other_name="k1"
).calc_metrics
},
period=period,
)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, "ASCAT.sm_with_ISMN.soil moisture.nc"
)
vars_should = [
u"n_obs",
u"tau",
u"gpi",
u"RMSD",
u"lon",
u"p_tau",
u"BIAS",
u"p_rho",
u"rho",
u"lat",
u"R",
u"p_R",
u"time",
u"idx",
u"_row_size",
]
n_obs_should = [357, 384, 1646, 1875, 1915, 467, 141, 251]
rho_should = np.array(
[
0.53934574,
0.7002289,
0.62200236,
0.53647155,
0.30413666,
0.6740655,
0.8418981,
0.74206454,
],
dtype=np.float32,
)
rmsd_should = np.array(
[
11.583476,
7.729667,
17.441547,
21.125721,
14.31557,
14.187225,
13.0622425,
12.903898,
],
dtype=np.float32,
)
with nc.Dataset(results_fname, mode="r") as results:
vars = results.variables.keys()
n_obs = results.variables["n_obs"][:].tolist()
rho = results.variables["rho"][:]
rmsd = results.variables["RMSD"][:]
assert sorted(vars) == sorted(vars_should)
assert sorted(n_obs) == sorted(n_obs_should)
nptest.assert_allclose(sorted(rho), sorted(rho_should), rtol=1e-4)
nptest.assert_allclose(sorted(rmsd), sorted(rmsd_should), rtol=1e-4)
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', '55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', 'grid')
static_layers_folder = os.path.join(os.path.dirname(__file__),
'..', 'test-data', 'sat',
'h_saf', 'static_layer')
ascat_reader = AscatSsmCdr(ascat_data_folder, ascat_grid_folder,
grid_filename='TUW_WARP5_grid_info_2_1.nc',
static_layer_path=static_layers_folder)
ascat_reader.read_bulk = True
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'ismn', 'multinetwork', 'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(
variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader,
'columns': ['soil moisture']
},
'ASCAT': {
'class': ascat_reader,
'columns': ['sm'],
'kwargs': {'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True}
}}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
process = Validation(
datasets, 'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, 'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau',
u'BIAS', u'p_rho', u'rho', u'lat', u'R', u'p_R']
n_obs_should = [384, 357, 482, 141, 251, 1927, 1887, 1652]
rho_should = np.array([0.70022893, 0.53934574,
0.69356072, 0.84189808,
0.74206454, 0.30299741,
0.53143877, 0.62204134], dtype=np.float32)
rmsd_should = np.array([7.72966719, 11.58347607,
14.57700157, 13.06224251,
12.90389824, 14.24668026,
21.19682884, 17.3883934], dtype=np.float32)
with nc.Dataset(results_fname, mode='r') as results:
assert sorted(results.variables.keys()) == sorted(vars_should)
assert sorted(results.variables['n_obs'][:].tolist()) == sorted(
n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]),
rtol=1e-4)
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]),
rtol=1e-4)
