class GeoCCISMv6Ts(SmecvTs):
# Reader implementation that uses the PATH configuration from above
# implememted subversion that have a set path configuration
_ds_implemented = [
('ESA_CCI_SM', 'v061', 'COMBINED'),
('ESA_CCI_SM', 'v061', 'ACTIVE'),
('ESA_CCI_SM', 'v061', 'PASSIVE'),
]
_t0 = 't0'
def __init__(self,
dataset_or_path,
exact_index=False,
force_path_group=None,
**kwargs):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoCCISMv6Ts, self).__init__(ts_path, **kwargs)
self.exact_index = exact_index
if exact_index and (self.parameters is not None):
self.parameters.append(self._t0)
class GeoHsafAscatSMDASTs(base_reader.HSAFAscatSMDASTs):
_ds_implemented = [('HSAF_ASCAT', 'SMDAS2', 'H14')]
def __init__(self, dataset_or_path, force_path_group=None, **kwargs):
"""
Parameters
----------
dataset_or_path : tuple or str
e.g. ('HSAF_ASCAT', 'SSM', 'H115+H116')
force_path_group : str, optional (default: None)
Select a specific path group from the path config to read.
kwargs :
kwargs that are passed to load_path and to initialise the reader.
"""
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoHsafAscatSMDASTs, self).__init__(ts_path, **kwargs)
def test_path_config():
dataset = ('GLDAS21', 'core')
conf = PathConfig(dataset, path_settings[dataset])
assert conf.os in ['win', 'lin']
# this should fall back to testdata if not on GEO:
path = conf.load_path(ignore_path_groups=None)
assert os.path.exists(path)
class GeoSMOSICTs(SMOSTs):
# Reader implementation that uses the PATH configuration from above
_ds_implemented = [('SMOS', 'IC', 'ASC'), ('SMOS', 'IC', 'DES')]
def __init__(self, dataset_or_path, force_path_group=None, **kwargs):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoSMOSICTs, self).__init__(ts_path, **kwargs)
def read_cells(self, cells):
cell_data = OrderedDict()
gpis, lons, lats = self.grid.grid_points_for_cell(list(cells))
for gpi, lon, lat in zip(gpis, lons, lats):
df = self.read(lon, lat)
cell_data[gpi] = df
return cell_data
class GeoCCISMv3Ts(SmecvTs):
# Reader implementation that uses the PATH configuration from above
# exact time variable (days) from reference date
_t0_ref = ('t0', datetime(1970, 1, 1, 0, 0, 0))
# fill values in the data columns
_col_fillvalues = {
'sm': [-9999.0],
'sm_uncertainty': [-9999.0],
_t0_ref[0]: [-3440586.5, -9999.0]
} # TODO: why has v045 another fillvalue?
# implememted subversion that have a set path configuration
_ds_implemented = [('ESA_CCI_SM', 'v033', 'COMBINED'),
('ESA_CCI_SM', 'v033', 'ACTIVE'),
('ESA_CCI_SM', 'v033', 'PASSIVE')]
def __init__(self,
dataset_or_path,
exact_index=False,
force_path_group=None,
**kwargs):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoCCISMv3Ts, self).__init__(ts_path,
exact_index=exact_index,
**kwargs)
def _replace_with_nan(self, df):
"""
Replace the fill values in columns defined in _col_fillvalues with NaN
"""
for col in df.columns:
if col in self._col_fillvalues.keys():
for fv in self._col_fillvalues[col]:
if self.scale_factors is not None and \
col in self.scale_factors.keys():
fv = fv * self.scale_factors[col]
df.loc[df[col] == fv, col] = np.nan
return df
def read(self, *args, **kwargs):
return self._replace_with_nan(
super(GeoCCISMv3Ts, self).read(*args, **kwargs))
class GeoCCIDsAmsr2Ts(CCIDs):
_ds_implemented = [('CCIDs', 'v052', 'AMSR2', 'DES')]
_t0 = 'jd0'
def __init__(self, dataset_or_path, exact_index=False,
force_path_group=None, **kwargs):
# TODO:
# freqband 0=invalid, 16=C_band_1, 32=C_band_2, 32=C_band_
"""
Parameters
----------
dataset_or_path : tuple or str
e.g. ('C3S', 'v201812', 'COMBINED', 'TCDR')
force_path_group : str, optional (default: None)
Select a specific path group from the path config to read.
kwargs :
kwargs that are passed to load_path and to initialise the reader.
"""
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
if 'parameters' in kwargs.keys():
self.parameters = kwargs.pop('parameters')
else:
self.parameters = None
self.dataset = dataset_or_path
path_config = path_settings[self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoCCIDsAmsr2Ts, self).__init__(ts_path, custom_dtype=None, **kwargs)
self.exact_index = exact_index
if self.exact_index and (self.parameters is not None):
self.parameters.append(self._t0)
def read(self, *args, **kwargs):
data = super(GeoCCIDsAmsr2Ts, self).read(*args, **kwargs,
only_valid=True)
if self.parameters is not None:
data = data[self.parameters]
if self.exact_index:
data[self._t0] = jd2dt(data[self._t0])
data.set_index(self._t0, inplace=True)
return data
class GeoMerra2Ts(MERRATs):
# Reader implementation that uses the PATH configuration from above
_ds_implemented = [('MERRA2', 'core')]
def __init__(self, dataset_or_path, force_path_group=None, **kwargs):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoMerra2Ts, self).__init__(ts_path, **kwargs)
class GeoScatSarSWIDrypanAnomsReader(ScatSarSWIDrypanReader):
_ds_implemented = [('SCATSAR', 'SWI', 'Drypan', 'Anoms')]
def __init__(self, dataset_or_path, force_path_group=None, switchflip=False):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoScatSarSWIDrypanAnomsReader, self).__init__(
path, smart_filename_class=SCATSARSWIDrypanAnomsFilename,
switchflip=switchflip)
class GeoPathEra5LandTs(ERATs):
# Reader implementation that uses the PATH configuration from above
_ds_implemented = [('ERA5-Land', 'sm_precip_lai'), ('ERA5-Land', 'snow'),
('ERA5-Land', 'temperature')]
def __init__(self, dataset_or_path, force_path_group=None, **kwargs):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoPathEra5LandTs, self).__init__(ts_path, **kwargs)
class GeoC3Sv202012Ts(base_reader.GeoC3STs):
# Reader implementation that uses the PATH configuration from above
# Flagging should be done with the masking adapter from pytesmo
_t0_ref = ('t0', datetime(1970, 1, 1, 0, 0, 0))
_col_fillvalues = {
'sm': [-9999.0],
'sm_uncertainty': [-9999.0],
_t0_ref[0]: [-3440586.5]
}
_ds_implemented = [
('C3S', 'v202012', 'COMBINED', 'DAILY', 'TCDR'),
('C3S', 'v202012', 'ACTIVE', 'DAILY', 'TCDR'),
('C3S', 'v202012', 'PASSIVE', 'DAILY', 'TCDR'),
('C3S', 'v202012', 'COMBINED', 'MONTHLY', 'TCDR'),
('C3S', 'v202012', 'ACTIVE', 'MONTHLY', 'TCDR'),
('C3S', 'v202012', 'PASSIVE', 'MONTHLY', 'TCDR'),
('C3S', 'v202012', 'COMBINED', 'DEKADAL', 'TCDR'),
('C3S', 'v202012', 'ACTIVE', 'DEKADAL', 'TCDR'),
('C3S', 'v202012', 'PASSIVE', 'DEKADAL', 'TCDR'),
]
def __init__(self, dataset_or_path, force_path_group=None, **kwargs):
"""
Parameters
----------
dataset : tuple or string
e.g. ('C3S', 'v201812', 'COMBINED', 'TCDR')
force_path_group : str, optional (default: None)
Select a specific path group from the path config to read.
kwargs :
kwargs that are passed to load_path and to initialise the reader.
"""
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoC3Sv202012Ts, self).__init__(ts_path, **kwargs)
class GeoC3Sv201706Ts(base_reader.GeoC3STs):
# Reader implementation that uses the PATH configuration from above
# Flagging should be done with the masking adapter from pytesmo
# exact time variable (days) from reference date
_t0_ref = ('t0', datetime(1970,1,1,0,0,0))
# fill values in the data columns
_col_fillvalues = {'sm': [-9999.0],
'sm_uncertainty': [-9999.0],
_t0_ref[0]: [-9999.0]}
_ds_implemented = [('C3S', 'v201706', 'COMBINED', 'DAILY', 'TCDR'),
('C3S', 'v201706', 'COMBINED', 'DAILY', 'ICDR'),
('C3S', 'v201706', 'ACTIVE', 'DAILY', 'TCDR'),
('C3S', 'v201706', 'ACTIVE', 'DAILY', 'ICDR'),
('C3S', 'v201706', 'PASSIVE', 'DAILY', 'TCDR'),
('C3S', 'v201706', 'PASSIVE', 'DAILY', 'ICDR')]
def __init__(self, dataset_or_path, exact_index=False, force_path_group=None,
**kwargs):
"""
Parameters
----------
dataset : tuple or str
e.g. ('C3S', 'v201706', 'COMBINED', 'TCDR') or path as string
kwargs
"""
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoC3Sv201706Ts, self).__init__(ts_path, **kwargs)
self.exact_index = exact_index
if exact_index and (self.parameters is not None):
self.parameters.append(self._t0_ref[0])
class GeoScatSarCglsSwiReader(ScatSarCglsSwiReader):
_ds_implemented = [('SCATSAR', 'CGLS', 'C0418', 'E7')]
def __init__(self,
dataset_or_path,
force_path_group=None,
grid_sampling=500,
tval=5):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoScatSarCglsSwiReader,
self).__init__(path, grid_sampling=grid_sampling, tval=tval)
class GeoCSarSsmTiffReader(CSarTiffReader):
_ds_implemented = [('CSAR', 'CGLS', 'SSM', '1km', 'V1.1', 'tiff')]
def __init__(self,
dataset_or_path,
force_path_group=None,
grid_sampling=500):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoCSarSsmTiffReader, self).__init__(path,
grid_sampling=grid_sampling,
param_rename='ssm')
class GeoScatSarSWIDrypanAbsReader(ScatSarSWIDrypanReader):
_ds_implemented = [('SCATSAR', 'SWI', 'Drypan', 'Abs')]
def __init__(self, dataset_or_path, force_path_group=None):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoScatSarSWIDrypanAbsReader, self).__init__(
path=path, smart_filename_class=SCATSARSWIDrypanAbsFilename)
def read(self, *args, **kwargs):
ts_point = super(GeoScatSarSWIDrypanAbsReader, self).read(*args, **kwargs)
ts_point.values[ts_point.values >= 127] = np.nan
return ts_point
class GeoSMAPLPRMv6Ts(base_reader.LPRMTs):
# Reader implementation that uses the PATH configuration from above
# implememted subversion that have a set path configuration
_ds_implemented = []
_t0 = 'SCANTIME_MJD'
def __init__(self, dataset_or_path, force_path_group=None,
**kwargs):
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
path_config = path_settings[self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoSMAPLPRMv6Ts, self).__init__(ts_path, **kwargs)
class GeoCCISMv6GenioTs(CCIDs):
# Reader implementation that uses the PATH configuration from above
# implememted subversion that have a set path configuration
_ds_implemented = []
_fillval = -999999.
def __init__(self, dataset_or_path, exact_index=False, force_path_group=None,
parameters=None, **kwargs):
if exact_index:
raise NotImplementedError
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
if isinstance(parameters, str):
parameters = [parameters]
self.parameters = parameters
self.dataset = dataset_or_path
path_config = path_settings[self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
super(GeoCCISMv6GenioTs, self).__init__(ts_path, **kwargs)
self.exact_index = exact_index
def _read_gp(self, gpi, **kwargs):
# override default args
ts = super(GeoCCISMv6GenioTs, self)._read_gp(gpi, **kwargs)
if self.parameters is not None:
ts = ts[self.parameters]
return ts.replace(self._fillval, np.nan)
class GeoISMNTs(ISMN_Interface):
"""
Modified ISMN Reader class, that reads good (G Flag) ISMN values and provides
functions for reading the station closest to a passed lonlat tuple.
"""
_ds_implemented = [('ISMN', 'v20191211'), ('ISMN', 'v20210131')]
def __init__(self,
dataset_or_path,
network=None,
force_path_group=None,
scale_factors=None):
"""
Initialize the reader for ISMN data
Parameters
----------
dataset_or_path : tuple or str
Dataset that is implemented in a path config. Or a path directly,
where the data is stored.
network : string or list, optional
Provide name of network to only load the given network
force_path_group : str, optional (default: None)
For a certain path group to load the data from.
scale_factors : dict, optional (default:None)
Apply the passed multiplicative scales to the selected columns
"""
if isinstance(dataset_or_path, list):
dataset_or_path = tuple(dataset_or_path)
self.dataset = dataset_or_path
self.network = network
path_config = path_settings[
self.dataset] if self.dataset in path_settings.keys() else None
self.path_config = PathConfig(self.dataset, path_config)
ts_path = self.path_config.load_path(force_path_group=force_path_group)
self.scale_factors = scale_factors
super(GeoISMNTs, self).__init__(ts_path, network=self.network)
self.metapath = os.path.join(self.root.root_dir, 'python_metadata')
def rebuild_metadata(self):
"""
Remove and rebuild python metadata.
"""
shutil.rmtree(self.metapath, ignore_errors=True)
super(GeoISMNTs, self).activate_network(self.network)
def read(self, *args, **kwargs):
# read by index and scale
ret = super(GeoISMNTs, self).read(*args, **kwargs)
if isinstance(ret, tuple):
df = ret[0]
meta = ret[1]
else:
df = ret
meta = None
if self.scale_factors is not None:
for n, s in self.scale_factors.items():
if n in df.columns: df[n] *= s
df.index = df.index.tz_localize(None)
if meta is None:
return df
else:
return df, meta
def read_nearest_station(self,
lon,
lat,
variable='soil_moisture',
max_dist=30000,
only_good=True,
return_flags=False,
return_distance=True,
**filter_kwargs):
"""
Read good (G-flagged) time series values for the station closest to the
passed lon, lat in the passed depths. To mask the time series from this
function, use a masking adapter. Only stations that measure the passed
variable will be considered.
Parameters
----------
lon : float
Longitude of the point to find the nearest station for
lat : float
Latitude of the point to find the nearest station for
variable : str, optional (default: soil_moisture)
Variable to read, the according flag will be used if only_good=True.
max_dist : int
Maximum allowed distance between the passed lon/lat position and the
actually found nearest station. If the distance is large, no data is read.
only_good : bool, optional (default: True)
If True, drops all lines where soil_moisture_flag is not G.
If False, returns also the flag column
return_flags : bool, optional (default: False)
Returns
return_distance : bool
Also return the distance (2 return values), else 1 return value
filter_kwargs: See ismn.components Sensor.eval() function
depth : Depth or list or tuple, optional (default: None)
filter_meta_dict : dict, optional (default: None)
check_only_sensor_depth_from : bool, optional (default: False)
Returns
-------
data : pd.DataFrame
Data measured at the station
nearest_station : ISMN_station
Station object from which the data was read.
distance : float
Distance between the passed coordinates and the read station.
"""
nearest_station, distance = self.find_nearest_station(lon, lat, True)
if distance > max_dist:
warnings.warn(
'fNo station within the selected distance found: {max_dist}')
data = None
else:
data = []
metadata = {}
for sensor in nearest_station.iter_sensors(variable=variable,
**filter_kwargs):
ts = sensor.read_data()[[variable, f'{variable}_flag']]
if only_good:
ts.loc[ts[f'{variable}_flag'] != 'G', variable] = nan
ts.loc[ts[f'{variable}_flag'] != 'G',
f'{variable}_flag'] = nan
ts = ts.dropna(how='all')
if not return_flags:
ts = ts[[variable]]
if self.scale_factors and variable in self.scale_factors.keys(
):
ts[variable] *= self.scale_factors[variable]
meta = sensor.metadata
ts.rename(columns={
c: f"{c} {str(meta['variable'].depth)}"
for c in ts.columns
},
inplace=True)
new_name = f"{meta['variable'].val} {str(meta['variable'].depth)}"
metadata[new_name] = meta
data.append(ts)
data = pd.concat(data, axis=1)
data.index = data.index.tz_localize(None)
if return_distance:
return data, nearest_station, distance
else:
return data, nearest_station
