def test_DataManager_read_ts_method_names():
ds1 = TestDataset("")
datasets = {
'DS1': {
'class': ds1,
'columns': ['soil moisture'],
},
'DS2': {
'class': ds1,
'columns': ['soil moisture'],
}
}
read_ts_method_names = {'DS1': 'read_ts', 'DS2': 'read_ts_other'}
dm = DataManager(datasets, 'DS1', read_ts_names=read_ts_method_names)
data = dm.read_ds('DS1', 1)
data_other = dm.read_ds('DS2', 1)
pdtest.assert_frame_equal(data, ds1.read_ts(1))
pdtest.assert_frame_equal(data_other, ds1.read_ts_other(1))
def test_DataManager_read_ts_method_names():
ds1 = TestDataset("")
datasets = {
'DS1': {
'class': ds1,
'columns': ['soil moisture'],
},
'DS2': {
'class': ds1,
'columns': ['soil moisture'],
}
}
read_ts_method_names = {'DS1': 'read_ts',
'DS2': 'read_ts_other'}
dm = DataManager(datasets, 'DS1',
read_ts_names=read_ts_method_names)
data = dm.read_ds('DS1', 1)
data_other = dm.read_ds('DS2', 1)
pdtest.assert_frame_equal(data, ds1.read_ts(1))
pdtest.assert_frame_equal(data_other, ds1.read_ts_other(1))
def getdata():
"""
handles the get request, which should contain the arguments listes under
parameters
Parameters
----------
station_id: int
id of station in database
scaling: string
chosen scaling method , for available choices see general.times_eries.scaling
snow_depth: float
mask snow depth greater than this value
st_l1: float
mask surface temperature layer1 lower than this value
air_temp: float
mask 2m air temperature lower than this value
ssf_masking: boolean
use SSF for masking true or false
"""
station_id = request.args.get('station_id')
scaling = request.args.get('scaling')
if scaling == 'noscale':
scaling = None
masking_ids = request.args.getlist('masking_ds[]')
masking_ops = request.args.getlist('masking_op[]')
masking_values = request.args.getlist('masking_values[]')
masking_values = [float(x) for x in masking_values]
anomaly = request.args.get('anomaly')
if anomaly == 'none':
anomaly = None
(depth_from,
depth_to,
sensor_id) = get_station_first_sm_layer(app.config['ISMN_PATH'],
station_id)
lon, lat = get_station_lonlat(app.config['ISMN_PATH'],
station_id)
start, end = get_station_start_end(app.config['ISMN_PATH'],
station_id, "soil moisture",
depth_from, depth_to)
period = [start, end]
masking_data = {'labels': [], 'data': []}
masking_meta = get_masking_metadata()
masking_masked_dict = None
if len(masking_ids) > 0:
# prepare masking datasets
masking_ds_dict = get_masking_ds_dict(masking_ids)
masking_masked_dict = {}
for masking_ds, masking_op, masking_value in zip(masking_ids,
masking_ops,
masking_values):
masking_masked_dict[masking_ds] = dict(masking_ds_dict[masking_ds])
new_cls = MaskingAdapter(masking_masked_dict[masking_ds]['class'],
masking_op,
masking_value)
masking_masked_dict[masking_ds]['class'] = new_cls
# use DataManager for reading masking datasets
masking_dm = DataManager(masking_ds_dict, masking_ids[0],
period=period)
masking_data = {}
valid_masking_ids = []
for mds in masking_ids:
mdata = masking_dm.read_ds(mds, lon, lat)
if mdata is not None:
masking_data[mds] = mdata
valid_masking_ids.append(mds)
else:
masking_data[mds] = pd.DataFrame()
if len(valid_masking_ids) > 1:
masking_data = BasicTemporalMatching(window=1.0).combinatory_matcher(
masking_data, masking_ids[0], n=len(masking_ids))
if len(masking_data) > 0:
labels, values = masking_data[
masking_data.keys()[0]].to_dygraph_format()
elif len(valid_masking_ids) == 1:
masking_data = masking_data[valid_masking_ids[0]]
labels, values = masking_data.to_dygraph_format()
else:
labels = [None]
values = None
for i, label in enumerate(labels):
for mid in masking_meta:
if masking_meta[mid]['variable']['name'] in label:
labels[i] = masking_meta[mid]['long_name']
masking_data = {'labels': labels, 'data': values}
ismn_iface = prepare_station_interface(app.config['ISMN_PATH'],
station_id,
"soil moisture",
depth_from, depth_to, sensor_id)
validation_ds_dict = get_validation_ds_dict()
validation_ds_dict.update({'ISMN': {'class': ismn_iface,
'columns': ['soil moisture']}})
if anomaly is not None:
adapter = {'climatology': AnomalyClimAdapter,
'average': AnomalyAdapter}
for dataset in validation_ds_dict:
validation_ds_dict[dataset]['class'] = adapter[
anomaly](validation_ds_dict[dataset]['class'],
columns=validation_ds_dict[dataset]['columns'])
mcalc = BasicMetricsPlusMSE(other_name='k1',
calc_tau=True).calc_metrics
process = Validation(validation_ds_dict, 'ISMN',
temporal_ref='cci',
scaling=scaling,
metrics_calculators={(2, 2): mcalc},
masking_datasets=masking_masked_dict,
period=period,
temporal_window=1)
df_dict = process.data_manager.get_data(1,
lon,
lat)
matched_data, result, used_data = process.perform_validation(
df_dict, (1, lon, lat))
res_key = list(result)[0]
data = used_data[res_key]
result = result[res_key][0]
# rename data to original names
rename_dict = {}
f = lambda x: "k{}".format(x) if x > 0 else 'ref'
for i, r in enumerate(res_key):
rename_dict[f(i)] = " ".join(r)
data.rename(columns=rename_dict, inplace=True)
labels, values = data.to_dygraph_format()
validation_datasets = {'labels': labels, 'data': values}
statistics = {'kendall': {'v': '%.2f' % result['tau'], 'p': '%.4f' % result['p_tau']},
'spearman': {'v': '%.2f' % result['rho'], 'p': '%.4f' % result['p_rho']},
'pearson': {'v': '%.2f' % result['R'], 'p': '%.4f' % result['p_R']},
'bias': '%.4f' % result['BIAS'],
'rmsd': {'rmsd': '%.4f' % np.sqrt(result['mse']),
'rmsd_corr': '%.4f' % np.sqrt(result['mse_corr']),
'rmsd_bias': '%.4f' % np.sqrt(result['mse_bias']),
'rmsd_var': '%.4f' % np.sqrt(result['mse_var'])},
'mse': {'mse': '%.4f' % result['mse'],
'mse_corr': '%.4f' % result['mse_corr'],
'mse_bias': '%.4f' % result['mse_bias'],
'mse_var': '%.4f' % result['mse_var']}}
scaling_options = {'noscale': 'No scaling',
'porosity': 'Scale using porosity',
'linreg': 'Linear Regression',
'mean_std': 'Mean - standard deviation',
'min_max': 'Minimum,maximum',
'lin_cdf_match': 'Piecewise <br> linear CDF matching',
'cdf_match': 'CDF matching'}
if scaling is None:
scaling = 'noscale'
masking_option_return = {}
for mid, mops, mval in zip(masking_ids,
masking_ops,
masking_values):
masking_option_return[mid] = {'op': mops,
'val': mval,
'name': masking_meta[mid]['long_name']}
settings = {'scaling': scaling_options[scaling],
'masking': masking_option_return}
output_data = {'validation_data': validation_datasets, 'masking_data': masking_data,
'statistics': statistics, 'settings': settings}
status = 1
if status == -1:
data = 'Error'
else:
data = jsonify(output_data)
resp = make_response(data)
resp.headers['Access-Control-Allow-Origin'] = '*'
return resp