def __init__(self,
filename,
parameters=None,
mode='r',
subgrid=SMECV_Grid_v052(None),
flatten=False,
fillval=None):
"""
Parameters
----------
filename : str
Path to the file to read
parameters : str or Iterable, optional (default: 'sm')
Names of parameters in the file to read.
If None are passed, all are read.
mode : str, optional (default: 'r')
Netcdf file mode, choosing something different to r may delete data.
subgrid : SMECV_Grid_v052
A subgrid of points to read. All other GPIS are masked (2d reading)
or ignored (when flattened).
flatten: bool, optional (default: False)
If set then the data is read into 1D arrays. This is used to e.g
reshuffle the data for a subset of points.
fillval : float or dict or None, optional (default: np.nan)
Fill Value for masked pixels, if a dict is passed, this can be
set for each parameter individually, otherwise it applies to all.
Note that choosing np.nan can lead to a change in dtype for some
(int) parameters. None will use the fill value from the netcdf file
"""
self.path = os.path.dirname(filename)
self.fname = os.path.basename(filename)
super(C3SImg, self).__init__(os.path.join(self.path, self.fname), mode=mode)
if parameters is None:
parameters = []
if type(parameters) != list:
parameters = [parameters]
self.parameters = parameters
self.subgrid = subgrid # subset to read
self.grid = SMECV_Grid_v052(None) # global input image
self.flatten = flatten
self.image_missing = False
self.img = None # to be loaded
self.glob_attrs = None
if isinstance(fillval, dict):
self.fillval = fillval
for p in self.parameters:
if p not in self.fillval:
self.fillval[p] = None
else:
self.fillval ={p: fillval for p in self.parameters}
def test_c3s_img_stack_multiple_img_reading_ICDR():
startdate, enddate = datetime(2017, 7, 1), datetime(2017, 12, 1)
parameters = ['sm']
path = os.path.join(os.path.dirname(__file__), 'c3s_sm-test-data', 'img',
'ICDR', '061_monthlyImages', 'passive')
subgrid = SMECV_Grid_v052('land').subgrid_from_bbox(-30, 30, 30, 70)
ds = C3S_Nc_Img_Stack(path,
parameters,
subgrid=subgrid,
subpath_templ=None)
row, col = None, None
for i, img in enumerate(ds.iter_images(startdate, enddate)):
test_loc_lonlat = (16.375, 48.125)
r, c = np.where((img.lon == test_loc_lonlat[0])
& (img.lat == test_loc_lonlat[1]))
if row is None:
row = r
else:
assert row == r
if col is None:
col = c
else:
assert col == c
if i == 0:
nptest.assert_almost_equal(img.data['sm'][row, col], 0.23400, 4)
if i == 1:
nptest.assert_almost_equal(img.data['sm'][row, col], 0.22680, 4)
if i == 2:
nptest.assert_almost_equal(img.data['sm'][row, col], 0.29522, 4)
def test_pretty_plot():
image = os.path.join(root_path.test_root, '00_testdata', 'plot',
'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-20100701000000-fv04.5.nc')
ds = Dataset(image)
dat = ds.variables['sm'][:]
dat = dat.filled(np.nan).flatten()
_, resampled_lons, resampled_lats, _ = SMECV_Grid_v052(None).get_grid_points()
index =pd.MultiIndex.from_arrays(np.array([resampled_lats, resampled_lons]),
names=['lats', 'lons'])
df = pd.DataFrame(index=index, data={'sm': dat}).dropna()
f, imax, im = cp_map(df, 'sm', resxy=(0.25,0.25), cbrange=(0,50.), veg_mask=True,
cmap=cm_sm, projection=ccrs.Sinusoidal(),
title='Overloaded Plot with too much Information',
ocean=True, land='grey', gridspace=(60,20), states=True,
borders=True, llc=(-179.9999, -90.), urc=(179.9999, 90),
cb_label='ESA CCI SM [$m^3/m^3$]', cb_labelsize=7, scale_factor=100,
grid_label_loc='0111', coastline_size='110m', cb_extend='both',
cb_ext_label_min='DRY', cb_ext_label_max='WET', cb_loc='right')
out_dir = tempfile.mkdtemp()
try:
filename = 'pretty_plot.png'
f.savefig(os.path.join(out_dir, 'pretty_plot.png'), dpi=200)
assert os.path.isfile(os.path.join(out_dir, filename))
finally:
shutil.rmtree(out_dir)
def test_cells_for_continent():
grid = SMECV_Grid_v052(None)
adp = GridShpAdapter(grid)
cells = adp.create_cells_for_continents(['Seven seas (open ocean)'],
out_file=None)
assert 1808 in cells['Seven seas (open ocean)']
def test_C3STs_tcdr_passive_decadal():
file = os.path.join(
os.path.join(
os.path.dirname(__file__), 'c3s_sm-test-data', 'img', 'TCDR',
'062_dekadalImages', 'passive',
'C3S-SOILMOISTURE-L3S-SSMV-PASSIVE-DEKADAL-20140101000000-TCDR-v201801.0.0.nc'
))
ds = C3SImg(file,
mode='r',
flatten=False,
fillval={
'nobs': -1,
'sm': np.nan
},
subgrid=SMECV_Grid_v052('landcover_class',
subset_value=[10, 11, 60,
70]).subgrid_from_bbox(
-14, 30, 44, 73))
image = ds.read()
test_loc_lonlat = (16.125, 48.125)
row, col = np.where((image.lon == test_loc_lonlat[0])
& (image.lat == test_loc_lonlat[1]))
assert image['nobs'].min() == -1
assert np.any(np.isnan(image['sm']))
nptest.assert_almost_equal(image['sm'][row, col], 0.50875, 4)
assert (image.metadata['sm']['long_name'] == 'Volumetric Soil Moisture')
def test_C3STs_tcdr_active_monthly():
file = os.path.join(
os.path.join(
os.path.dirname(__file__), 'c3s_sm-test-data', 'img', 'TCDR',
'061_monthlyImages', 'active',
'C3S-SOILMOISTURE-L3S-SSMS-ACTIVE-MONTHLY-20140101000000-TCDR-v201801.0.0.nc'
))
ds = C3SImg(file,
mode='r',
parameters='sm',
flatten=False,
fillval=None,
subgrid=SMECV_Grid_v052(None).subgrid_from_bbox(
-181, -91, 181, 91))
image = ds.read()
test_loc_lonlat = (16.375, 48.125)
row, col = np.where((image.lon == test_loc_lonlat[0])
& (image.lat == test_loc_lonlat[1]))
assert image.data['sm'].shape == (720, 1440)
nptest.assert_almost_equal(image.data['sm'][row, col], 47.69982, 4)
assert (image.metadata['sm']['_FillValue'] == -9999.)
assert image.data['sm'].min() == image.metadata['sm']['_FillValue']
assert (image.metadata['sm']['long_name'] ==
'Percent of Saturation Soil Moisture')
def grid_points_for_cells(areas_or_cells):
'''
Load the grid points on the grid for the passed area or cells
Parameters
----------
areas_or_cells : list
List of names of continents or countries as in continents_cells.txt
or list of cell numbers.
Returns
-------
grid_points : np.array
List of grid points in the passed cells or in the cells for the passed
area(s)
'''
grid = SMECV_Grid_v052()
grid_points = []
if isinstance(areas_or_cells, str):
areas_or_cells = [areas_or_cells]
for area in areas_or_cells:
if isinstance(area, str):
cells = read_cells_for_continent(area)
else:
cells = area
grid_points += np.ndarray.tolist(grid.grid_points_for_cell(cells)[0])
return np.array(grid_points)
def __init__(self, path, mode='r', grid=None, fn_format='{:04d}',
custom_dtype=None):
if grid is None:
grid = SMECV_Grid_v052()
super(CCIDs, self).__init__(path, grid, IndexedRaggedTs,
mode=mode,
fn_format=fn_format,
ioclass_kws={'custom_dtype': custom_dtype})
def test_subgrid_country_cont_names():
full_grid = SMECV_Grid_v052('land')
adp = GridShpAdapter(full_grid)
sgrid = adp.create_subgrid(names=['Austria', 'Seven seas (open ocean)'],
verbose=False)
gpis, lons, lats, cells = sgrid.get_grid_points()
assert 795661 in gpis
assert sgrid.gpi2lonlat(795661) == (15.375, 48.125)
assert 232835 in gpis
assert sgrid.gpi2lonlat(232835) == (68.875, -49.625)
def test_C3STs_icdr_combined_daily():
file = os.path.join(
os.path.join(
os.path.dirname(__file__), 'c3s_sm-test-data', 'img', 'ICDR',
'060_dailyImages', 'combined', '2017',
'C3S-SOILMOISTURE-L3S-SSMV-COMBINED-DAILY-20170701000000-ICDR-v201706.0.0.nc'
))
ds = C3SImg(file,
mode='r',
parameters=['sm', 't0'],
flatten=False,
subgrid=SMECV_Grid_v052('land'))
image = ds.read()
test_loc_lonlat = (16.375, 48.125)
row, col = np.where((image.lon == test_loc_lonlat[0])
& (image.lat == test_loc_lonlat[1]))
nptest.assert_almost_equal(image.data['sm'][row, col], 0.14548, 4)
assert (image.metadata['t0']['long_name'] == 'Observation Timestamp')
def test_c3s_img_stack_single_img_reading():
parameters = ['sm']
path = os.path.join(os.path.dirname(__file__), 'c3s_sm-test-data', 'img',
'TCDR', '060_dailyImages', 'combined')
subgrid = SMECV_Grid_v052('land').subgrid_from_bbox(-30, 30, 30, 70)
ds = C3S_Nc_Img_Stack(path,
parameters,
fillval={'sm': -1},
subgrid=subgrid,
subpath_templ=('%Y', ))
img = ds.read(datetime(2014, 1, 1)) # type: Image
test_loc_lonlat = (16.375, 48.125)
row, col = np.where((img.lon == test_loc_lonlat[0])
& (img.lat == test_loc_lonlat[1]))
nptest.assert_almost_equal(img.data['sm'][row, col], 0.34659, 4)
assert np.min(img.data['sm']) == -1
def cells_for_identifier(names, grid=SMECV_Grid_v052()):
'''
Return cell numbers for the passed areas (or cells)
Parameters
----------
areas_or_cells : str or list
List of cells (trivial case), list of area names or 'global'
Implemented areas:
Returns
-------
cells: np.array
List of cells on the selected grid
'''
if isinstance(names, str):
if names.lower() == 'global':
return grid.get_cells().tolist()
else:
names = [names]
adp = GridShpAdapter(grid)
return adp.create_subgrid(names)
def reshuffle(input_root,
outputpath,
startdate,
enddate,
parameters=None,
land_points=True,
bbox=None,
ignore_meta=False,
imgbuffer=500):
"""
Reshuffle method applied to C3S data.
Parameters
----------
input_root: string
input path where c3s images were downloaded.
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
parameters: list, optional (default: None)
parameters to read and convert
land_points : bool, optional (default: True)
Use the land grid to calculate time series on.
Leads to faster processing and smaller files.
bbox : tuple
Min lon, min lat, max lon, max lat
BBox to read data for.
ignore_meta : bool, optional (default: False)
Ignore metadata and reshuffle only the values. Can be used e.g. if a
version is not yet supported.
imgbuffer: int, optional (default: 50)
How many images to read at once before writing time series.
"""
if land_points:
grid = SMECV_Grid_v052('land')
else:
grid = SMECV_Grid_v052(None)
if bbox:
grid = grid.subgrid_from_bbox(*bbox)
if parameters is None:
file_args, file_vars = parse_filename(input_root)
parameters = [p for p in file_vars if p not in ['lat', 'lon', 'time']]
subpath_templ = ('%Y', ) if os.path.isdir(
os.path.join(input_root, str(startdate.year))) else None
input_dataset = C3S_Nc_Img_Stack(data_path=input_root,
parameters=parameters,
subgrid=grid,
flatten=True,
fillval=None,
subpath_templ=subpath_templ)
if not ignore_meta:
prod_args = input_dataset.fname_args
kwargs = {
'sensor_type': prod_args['prod'].lower(),
'cdr_type': prod_args['cdr'],
'product_temp_res': prod_args['temp'],
'cls': getattr(metadata, f"C3S_SM_TS_Attrs_{prod_args['vers']}")
}
if prod_args['temp'].upper() == 'DAILY':
kwargs.pop('product_temp_res')
attrs = C3S_daily_tsatt_nc(**kwargs)
else:
attrs = C3S_dekmon_tsatt_nc(**kwargs)
ts_attributes = {}
global_attributes = attrs.global_attr
for var in parameters:
ts_attributes.update(attrs.ts_attributes[var])
else:
global_attributes = None
ts_attributes = None
if not os.path.exists(outputpath):
os.makedirs(outputpath)
reshuffler = Img2Ts(input_dataset=input_dataset,
outputpath=outputpath,
startdate=startdate,
enddate=enddate,
input_grid=grid,
imgbuffer=imgbuffer,
cellsize_lat=5.0,
cellsize_lon=5.0,
global_attr=global_attributes,
zlib=True,
unlim_chunksize=1000,
ts_attributes=ts_attributes)
reshuffler.calc()
'C3S-SOILMOISTURE-L3S-SSMV-PASSIVE-DEKADAL-20170701000000-ICDR-v201706.0.0.nc'
))
ds = C3SImg(file, mode='r', parameters='sm', flatten=False, fillval=np.nan)
image = ds.read()
test_loc_lonlat = (16.375, 48.125)
row, col = np.where((image.lon == test_loc_lonlat[0])
& (image.lat == test_loc_lonlat[1]))
nptest.assert_almost_equal(image.data['sm'][row, col], 0.21000, 4)
assert (image.metadata['sm']['long_name'] == 'Volumetric Soil Moisture')
@pytest.mark.parametrize(
"subgrid,", [(SMECV_Grid_v052(None)), (SMECV_Grid_v052('land')),
(SMECV_Grid_v052('landcover_class', subset_value=[10, 11])),
(SMECV_Grid_v052('land').subgrid_from_bbox(74, 13, 78, 15))])
def test_1Dreading(subgrid):
# Test 1D reading with and without land grid, and if the results are the same
file = os.path.join(
os.path.join(
os.path.dirname(__file__), 'c3s_sm-test-data', 'img', 'ICDR',
'060_dailyImages', 'combined', '2017',
'C3S-SOILMOISTURE-L3S-SSMV-COMBINED-DAILY-20170701000000-ICDR-v201706.0.0.nc'
))
ds = C3SImg(file, mode='r', parameters=None, flatten=True, subgrid=subgrid)
image = ds.read()
def CCICellGrid():
return SMECV_Grid_v052(None)
def CCILandGrid():
return SMECV_Grid_v052('land')
return cube
else:
return data_arr, {
'lon': cell_lons,
'lat': cell_lats,
'gpi': cell_gpi
}
if __name__ == '__main__':
path = "/shares/wpreimes/radar/Datapool/ESA_CCI_SM/02_processed/ESA_CCI_SM_v05.2/timeseries/combined/"
dt_index = pd.date_range('2000-06-01', '2000-06-30', freq='D')
ds = SmecvTs(path,
grid=SMECV_Grid_v052(None),
clip_dates=(dt_index[0], dt_index[-1]))
params = ['sm', 'flag', 'dnflag', 'freqbandID', 'mode', 'sensor', 't0']
param_fill_val = {
'sm': -9999.,
'flag': 0,
'dnflag': 0,
'freqbandID': 0,
'mode': 0,
'sensor': 0,
't0': -9999.,
}
param_dtype = {
def __init__(self,
data_path,
parameters='sm',
subgrid=SMECV_Grid_v052(None),
flatten=False,
solve_ambiguity='sort_last',
fntempl=fntempl,
subpath_templ=None,
fillval=None):
"""
Parameters
----------
data_path : str
Path to directory where C3S images are stored
parameters : list or str, optional (default: 'sm')
Variables to read from the image files.
grid : pygeogrids.CellGrid, optional (default: SMECV_Grid_v052(None)
Subset of the image to read
array_1D : bool, optional (default: False)
Flatten the read image to a 1D array instead of a 2D array
solve_ambiguity : str, optional (default: 'latest')
Method to solve ambiguous time stamps, e.g. if a reprocessing
was performed.
- error: raises error in case of ambiguity
- sort_last (default): uses the last file when sorted by file
name, in case that multiple files are found.
- sort_first: uses the first file when sorted by file name
in case that multiple files are found.
filename_templ: str, optional
Filename template to parse datetime from.
subpath_templ : list or None, optional (default: None)
List of subdirectory names to build file paths. e.g. ['%Y'] if files
in collected by years.
fillval : float or dict or None, optional (default: None)
Fill Value for masked pixels, if a dict is passed, this can be
set for each parameter individually, otherwise it applies to all.
Note that choosing np.nan can lead to a change in dtype for some
parameters (int to float).
None will use the fill value from the netcdf file
"""
self.data_path = data_path
ioclass_kwargs = {'parameters': parameters,
'subgrid': subgrid,
'flatten': flatten,
'fillval': fillval}
self.fname_args = self._parse_filename(fntempl)
self.solve_ambiguity = solve_ambiguity
fn_args = self.fname_args.copy()
fn_args['subvers'] = '*'
fn_args['cdr'] = '*'
filename_templ = fntempl.format(**fn_args)
super(C3S_Nc_Img_Stack, self).__init__(path=data_path,
ioclass=C3SImg,
fname_templ=filename_templ ,
datetime_format="%Y%m%d%H%M%S",
subpath_templ=subpath_templ,
exact_templ=False,
ioclass_kws=ioclass_kwargs)