def get_station_first_sm_layer(path, stationname):
"""
Get the metadata of the first soil moisture layer of this variable.
Parameters
----------
path: string
Folder in which the ISMN data is stored
stationname: string
Name of the station
Returns
-------
depth_from: float
depth_to: float
sensor: string
"""
iface = ISMN_Interface(path)
station = iface.get_station(stationname)
depths_from, depths_to = station.get_depths("soil moisture")
s_idx = np.argsort(depths_from)
depth_from = depths_from[s_idx[0]]
depth_to = depths_to[s_idx[0]]
sensor = station.get_sensors("soil moisture", depth_from, depth_to)
return depth_from, depth_to, sensor[0]
def get_station_data(path, stationname, variable,
depth_from, depth_to, sensor_id):
"""
Read the data from the ISMN dataset
Parameters
----------
path: string
Folder in which the ISMN data is stored
stationname: string
Name of the station
variable: string
Name of the variable to read
depth_from: string
starting depth of the variable
depth_to: string
end depth of the variable
sensor_id: string
Sensor id of the sensor to read
Returns
-------
ds: pandas.DataFrame
Data
"""
iface = ISMN_Interface(path)
station = iface.get_station(stationname)
ds = station.read_variable(variable.encode('ascii'),
float(depth_from),
float(depth_to),
sensor_id.encode('ascii'))
return ds.data[variable]
def get_station_start_end(path, stationname, variable,
depth_from, depth_to):
"""
Get the start and end date for the selected insitu time series.
Parameters
----------
path: string
Folder in which the ISMN data is stored
stationname: string
Name of the station
variable: string
Name of the variable to read
depth_from: string
starting depth of the variable
depth_to: string
end depth of the variable
Returns
-------
start: datetime
end: datetime
"""
iface = ISMN_Interface(path)
station = iface.get_station(stationname)
return station.get_min_max_obs_timestamp(variable=variable,
min_depth=depth_from,
max_depth=depth_to)
def read_ISMN(plot=False):
ismn_data_folder = os.path.join(
'.', 'data', 'Data_seperate_files_2011' + '1122_20121122_2364256_oqsd')
ISMN_reader = ISMN_Interface(ismn_data_folder)
network = 'IIT-KANPUR'
station = 'IITK-Airstrip'
station_obj = ISMN_reader.get_station(station)
print "Available Variables at Station %s" % station
#get the variables that this station measures
variables = station_obj.get_variables()
print variables
depths_from, depths_to = station_obj.get_depths(variables[0])
sensors = station_obj.get_sensors(variables[0], depths_from[0],
depths_to[0])
#read the data of the variable, depth, sensor combination
time_series = station_obj.read_variable(variables[0],
depth_from=depths_from[0],
depth_to=depths_to[0],
sensor=sensors[0])
#print information about the selected time series
print "Selected time series is:"
print time_series
#plot the data
time_series.plot()
#with pandas 0.12 time_series.plot() also works
plt.legend()
plt.show()
def read_ISMN(plot=False):
ismn_data_folder = os.path.join('.', 'data', 'Data_seperate_files_2011'+
'1122_20121122_2364256_oqsd')
ISMN_reader = ISMN_Interface(ismn_data_folder)
network = 'IIT-KANPUR'
station = 'IITK-Airstrip'
station_obj = ISMN_reader.get_station(station)
print "Available Variables at Station %s"%station
#get the variables that this station measures
variables = station_obj.get_variables()
print variables
depths_from,depths_to = station_obj.get_depths(variables[0])
sensors = station_obj.get_sensors(variables[0],depths_from[0],depths_to[0])
#read the data of the variable, depth, sensor combination
time_series = station_obj.read_variable(variables[0],depth_from=depths_from[0],depth_to=depths_to[0],sensor=sensors[0])
#print information about the selected time series
print "Selected time series is:"
print time_series
#plot the data
time_series.plot()
#with pandas 0.12 time_series.plot() also works
plt.legend()
plt.show()
def get_station_lonlat(path, stationname):
"""
Get the latitude and longitude coordinates from a station.
Parameters
----------
path: string
Folder in which the ISMN data is stored
stationname: string
Name of the station
Returns
-------
lon: float
lat: float
"""
iface = ISMN_Interface(path)
station = iface.get_station(stationname)
return station.longitude, station.latitude
def prepare_station_interface(path, stationname, variable,
depth_from, depth_to, sensor_id):
"""
Prepare an interface to the requested station data that
provides the data via a read_ts(id) function. This is at the moment
necessary since the ISMN interface does not follow the standards of
other interfaces.
Parameters
----------
path: string
Folder in which the ISMN data is stored
stationname: string
Name of the station
variable: string
Name of the variable to read
depth_from: string
starting depth of the variable
depth_to: string
end depth of the variable
sensor_id: string
Sensor id of the sensor to read
Returns
-------
iface: object
interface object which has a read_ts method
"""
iface = ISMN_Interface(path)
station = iface.get_station(stationname)
def read_ts(idx):
ds = station.read_variable(variable.encode('ascii'),
float(depth_from),
float(depth_to),
sensor_id.encode('ascii'))
return ds.data
station.read_ts = read_ts
return station
def variable_list(path, stationname):
"""
Goes through a downloaded ISMN dataset and reads the necessary metadata
needed for the viewer.
Parameters
----------
path: string
Folder in which the ISMN data is stored
stationname: string
Name of the station
Returns
-------
metadata: dict
Metadata dictionary.
"""
variables = []
iface = ISMN_Interface(path)
station = iface.get_station(stationname)
for i, var in enumerate(station.variables):
name = "{}_{:.2f}_{}".format(var,
station.depth_from[i],
station.sensors[i])
var_dict = {"quantityName": var,
"unit": "",
"depthFrom": station.depth_from[i],
"depthTo": station.depth_to[i],
"sensorId": station.sensors[i],
"variableName": name
}
variables.append(var_dict)
dmin, dmax = station.get_min_max_obs_timestamp()
vl = {"maxtime": dmax.date().isoformat(),
"mintime": dmin.date().isoformat(),
"originalTimeframeValid": False,
"variables": variables}
return vl
static_layers_folder = os.path.join(testdata_folder, 'sat/h_saf/static_layer')
ascat_reader = AscatSsmCdr(ascat_data_folder,
ascat_grid_folder,
grid_filename='TUW_WARP5_grid_info_2_1.nc',
static_layer_path=static_layers_folder)
ascat_reader.read_bulk = True
# Initialize ISMN reader
# In[4]:
ismn_data_folder = os.path.join(testdata_folder,
'ismn/multinetwork/header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
# The validation is run based on jobs. A job consists of at least three lists or numpy arrays specifing the grid
# point index, its latitude and longitude. In the case of the ISMN we can use the `dataset_ids` that identify every
# time series in the downloaded ISMN data as our grid point index. We can then get longitude and latitude from the
# metadata of the dataset.
#
# **DO NOT CHANGE** the name ***jobs*** because it will be searched during the parallel processing!
# In[5]:
jobs = []
ids = ismn_reader.get_dataset_ids(variable='soil moisture',
min_depth=0,
max_depth=0.1)
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', '55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', 'grid')
static_layers_folder = os.path.join(os.path.dirname(__file__),
'..', 'test-data', 'sat',
'h_saf', 'static_layer')
ascat_reader = AscatSsmCdr(ascat_data_folder, ascat_grid_folder,
static_layer_path=static_layers_folder)
ascat_reader.read_bulk = True
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'ismn', 'multinetwork', 'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(
variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader,
'columns': ['soil moisture']
},
'ASCAT': {
'class': ascat_reader,
'columns': ['sm'],
'kwargs': {'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True}
}}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
process = Validation(
datasets, 'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, 'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau',
u'BIAS', u'p_rho', u'rho', u'lat', u'R', u'p_R']
n_obs_should = [384, 357, 482, 141, 251, 1927, 1887, 1652]
rho_should = np.array([0.70022893, 0.53934574,
0.69356072, 0.84189808,
0.74206454, 0.30299741,
0.53143877, 0.62204134], dtype=np.float32)
rmsd_should = np.array([7.72966719, 11.58347607,
14.57700157, 13.06224251,
12.90389824, 14.24668026,
21.19682884, 17.3883934], dtype=np.float32)
with nc.Dataset(results_fname) as results:
assert sorted(results.variables.keys()) == sorted(vars_should)
assert sorted(results.variables['n_obs'][:].tolist()) == sorted(
n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]),
rtol=1e-4)
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]),
rtol=1e-4)
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'ascat', 'netcdf',
'55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'sat', 'ascat', 'netcdf',
'grid')
ascat_reader = AscatH25_SSM(ascat_data_folder, ascat_grid_folder)
ascat_reader.read_bulk = True
ascat_reader._load_grid_info()
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), '..',
'test-data', 'ismn', 'multinetwork',
'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader,
'columns': ['soil moisture']
},
'ASCAT': {
'class': ascat_reader,
'columns': ['sm'],
'kwargs': {
'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True
}
}
}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
process = Validation(
datasets,
'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2):
metrics_calculators.BasicMetrics(other_name='k1').calc_metrics
},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(save_path,
'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [
u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau', u'BIAS', u'p_rho',
u'rho', u'lat', u'R', u'p_R'
]
n_obs_should = [360, 385, 1644, 1881, 1927, 479, 140, 251]
rho_should = np.array([
0.546187, 0.717398, 0.620892, 0.532465, 0.302997, 0.694713, 0.840592,
0.742065
],
dtype=np.float32)
rmsd_should = np.array([
11.536263, 7.545650, 17.451935, 21.193714, 14.246680, 14.494674,
13.173215, 12.903898
],
dtype=np.float32)
with nc.Dataset(results_fname) as results:
assert sorted(results.variables.keys()) == sorted(vars_should)
assert sorted(
results.variables['n_obs'][:].tolist()) == sorted(n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]),
rtol=1e-4)
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]),
rtol=1e-4)
ascat_data_folder = os.path.join('/media/sf_R', 'Datapool_processed', 'WARP', 'WARP5.5',
'IRMA1_WARP5.5_P2', 'R1', '080_ssm', 'netcdf')
ascat_grid_folder = os.path.join('/media/sf_R', 'Datapool_processed', 'WARP',
'ancillary', 'warp5_grid')
ascat_reader = AscatH25_SSM(ascat_data_folder, ascat_grid_folder)
ascat_reader.read_bulk = True
ascat_reader._load_grid_info()
# Initialize ISMN reader
# In[3]:
ismn_data_folder = '/data/Development/python/workspace/pytesmo/tests/test-data/ismn/format_header_values/'
ismn_reader = ISMN_Interface(ismn_data_folder)
# The validation is run based on jobs. A job consists of at least three lists or numpy arrays specifing the grid point index, its latitude and longitude. In the case of the ISMN we can use the `dataset_ids` that identify every time series in the downloaded ISMN data as our grid point index. We can then get longitude and latitude from the metadata of the dataset.
#
# **DO NOT CHANGE** the name ***jobs*** because it will be searched during the parallel processing!
# In[4]:
jobs = []
ids = ismn_reader.get_dataset_ids(variable='soil moisture', min_depth=0, max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
print jobs
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), 'test-data',
'sat', 'ascat', 'netcdf', '55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), 'test-data',
'sat', 'ascat', 'netcdf', 'grid')
ascat_reader = AscatH25_SSM(ascat_data_folder, ascat_grid_folder)
ascat_reader.read_bulk = True
ascat_reader._load_grid_info()
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), 'test-data',
'ismn', 'multinetwork', 'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(
variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader, 'columns': [
'soil moisture'
], 'type': 'reference', 'args': [], 'kwargs': {}
},
'ASCAT': {
'class': ascat_reader, 'columns': [
'sm'
], 'type': 'other', 'args': [], 'kwargs': {}, 'grids_compatible':
False, 'use_lut': False, 'lut_max_dist': 30000
}
}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
process = Validation(
datasets=datasets,
data_prep=DataPreparation(),
temporal_matcher=temporal_matchers.BasicTemporalMatching(
window=1 / 24.0,
reverse=True),
scaling='lin_cdf_match',
scale_to_other=True,
metrics_calculator=metrics_calculators.BasicMetrics(),
period=period,
cell_based_jobs=False)
for job in jobs:
results = process.calc(job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, 'ISMN.soil moisture_with_ASCAT.sm.nc')
vars_should = [u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau',
u'BIAS', u'p_rho', u'rho', u'lat', u'R', u'p_R']
n_obs_should = [360, 385, 1644, 1881, 1927, 479, 140, 251]
rho_should = np.array([0.54618734, 0.71739876, 0.62089276, 0.53246528,
0.30299741, 0.69647062, 0.840593, 0.73913699],
dtype=np.float32)
rmsd_should = np.array([11.53626347, 7.54565048, 17.45193481, 21.19371414,
14.24668026, 14.27493, 13.173215, 12.59192371],
dtype=np.float32)
with nc.Dataset(results_fname) as results:
assert sorted(results.variables.keys()) == sorted(vars_should)
assert sorted(results.variables['n_obs'][:].tolist()) == sorted(
n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]))
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]))
def test_ascat_ismn_validation():
"""
Test processing framework with some ISMN and ASCAT sample data
"""
ascat_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', '55R22')
ascat_grid_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'sat', 'ascat', 'netcdf', 'grid')
ascat_reader = AscatH25_SSM(ascat_data_folder, ascat_grid_folder)
ascat_reader.read_bulk = True
ascat_reader._load_grid_info()
# Initialize ISMN reader
ismn_data_folder = os.path.join(os.path.dirname(__file__), '..', 'test-data',
'ismn', 'multinetwork', 'header_values')
ismn_reader = ISMN_Interface(ismn_data_folder)
jobs = []
ids = ismn_reader.get_dataset_ids(
variable='soil moisture',
min_depth=0,
max_depth=0.1)
for idx in ids:
metadata = ismn_reader.metadata[idx]
jobs.append((idx, metadata['longitude'], metadata['latitude']))
# Create the variable ***save_path*** which is a string representing the
# path where the results will be saved. **DO NOT CHANGE** the name
# ***save_path*** because it will be searched during the parallel
# processing!
save_path = tempfile.mkdtemp()
# Create the validation object.
datasets = {
'ISMN': {
'class': ismn_reader,
'columns': ['soil moisture']
},
'ASCAT': {
'class': ascat_reader,
'columns': ['sm'],
'kwargs': {'mask_frozen_prob': 80,
'mask_snow_prob': 80,
'mask_ssf': True}
}}
period = [datetime(2007, 1, 1), datetime(2014, 12, 31)]
process = Validation(
datasets, 'ISMN',
temporal_ref='ASCAT',
scaling='lin_cdf_match',
scaling_ref='ASCAT',
metrics_calculators={
(2, 2): metrics_calculators.BasicMetrics(other_name='k1').calc_metrics},
period=period)
for job in jobs:
results = process.calc(*job)
netcdf_results_manager(results, save_path)
results_fname = os.path.join(
save_path, 'ASCAT.sm_with_ISMN.soil moisture.nc')
vars_should = [u'n_obs', u'tau', u'gpi', u'RMSD', u'lon', u'p_tau',
u'BIAS', u'p_rho', u'rho', u'lat', u'R', u'p_R']
n_obs_should = [360, 385, 1644, 1881, 1927, 479, 140, 251]
rho_should = np.array([0.546187,
0.717398,
0.620892,
0.532465,
0.302997,
0.694713,
0.840592,
0.742065],
dtype=np.float32)
rmsd_should = np.array([11.536263,
7.545650,
17.451935,
21.193714,
14.246680,
14.494674,
13.173215,
12.903898],
dtype=np.float32)
with nc.Dataset(results_fname) as results:
assert sorted(results.variables.keys()) == sorted(vars_should)
assert sorted(results.variables['n_obs'][:].tolist()) == sorted(
n_obs_should)
nptest.assert_allclose(sorted(rho_should),
sorted(results.variables['rho'][:]),
rtol=1e-4)
nptest.assert_allclose(sorted(rmsd_should),
sorted(results.variables['RMSD'][:]),
rtol=1e-4)
def ismn_metadata(path):
"""
Goes through a downloaded ISMN dataset and reads the necessary metadata
needed for the viewer.
Parameters
----------
path: string
Folder in which the ISMN data is stored
Returns
-------
metadata: dict
Metadata dictionary.
"""
metadata = {"Networks": []}
iface = ISMN_Interface(path)
for networkname in iface.list_networks():
network_dict = {
"networkID": networkname,
"network_abstract": "",
"network_status": "",
"network_country": "",
"network_continent": "",
"network_op_start": "",
"network_op_end": "",
"network_type": "project",
"network_constraints": "",
"network_reference": "",
"network_url_data": "",
"network_url": "",
"network_acknowledge": "",
"network_variables": "",
"network_depths": "",
"network_sensors": "",
"Stations": []}
metadata['Networks'].append(network_dict)
station_list = network_dict['Stations']
stations = iface.list_stations(network=networkname)
for stationname in stations:
station = iface.get_station(stationname, network=networkname)
dmin, dmax = station.get_min_max_obs_timestamp()
if dmin is None or dmax is None:
# No soil moisture measured at this station
continue
station_dict = {
"station_abbr": stationname,
"lat": station.latitude,
"lng": station.longitude,
"comment": None,
"stationID": stationname,
"extMetadata": None,
"station_name": stationname,
"variableText": '<br>'.join(np.unique(station.variables)),
"depthText": get_depth_text(station.depth_from,
station.depth_to,
station.variables),
"sensorText": '<br>'.join(np.unique(station.sensors)),
"maximum": dmax.isoformat(),
"minimum": dmin.isoformat()
}
station_list.append(station_dict)
return metadata
