def __init__(self,
filename,
mode='r',
parameters=None,
flatten=False,
grid=EASE25CellGrid(bbox=None),
read_flags=(0, 1),
float_fillval=np.nan):
super(SMOSImg, self).__init__(filename, mode=mode)
if parameters is None:
parameters = []
if type(parameters) != list:
parameters = [parameters]
self.read_flags = read_flags
self.parameters = parameters
self.flatten = flatten
self.grid = grid
self.image_missing = False
self.img = None # to be loaded
self.glob_attrs = None
self.float_fillval = float_fillval
def main(args):
"""
Main routine used for command line interface.
Parameters
----------
args : list of str
Command line arguments.
"""
args = parse_args(args)
input_grid = EASE25CellGrid(
bbox=tuple(args.bbox) if args.bbox is not None else None)
flags = (0) if args.only_good else (0, 1)
ds_kwargs = {
'read_flags': flags,
'grid': input_grid,
'parameters': args.parameters
}
reshuffle(args.dataset_root,
args.timeseries_root,
args.start,
args.end,
imgbuffer=args.imgbuffer,
**ds_kwargs)
def reshuffle(input_root, outputpath,
startdate, enddate,
imgbuffer=200, **ds_kwargs):
"""
Reshuffle method applied to SMOS image data.
Parameters
----------
input_root: string
input path where smos ic data was downloaded to (yearly folders)
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
imgbuffer: int, optional
How many images to read at once before writing time series.
ds_kwargs: dict
Kwargs that are passed to the image datastack class
"""
ff, file_vars = firstfile(input_root)
fp, ff = os.path.split(ff)
if 'grid' not in ds_kwargs.keys():
ds_kwargs['grid'] = EASE25CellGrid(None)
if 'parameters' not in ds_kwargs.keys():
ds_kwargs['parameters'] = None
# this is only for reading the ts_attrs
input_dataset = SMOSImg(filename=os.path.join(fp, ff),
parameters=ds_kwargs['parameters'], flatten=True, read_flags=None,
grid=ds_kwargs['grid'])
_, ts_attributes = input_dataset._read_img()
global_attr = input_dataset.get_global_attrs()
if ds_kwargs['parameters'] is None:
ds_kwargs['parameters'] = input_dataset.parameters
input_dataset = SMOSDs(input_root, flatten=True, **ds_kwargs)
if not os.path.exists(outputpath):
os.makedirs(outputpath)
# get time series attributes from first day of data.
reshuffler = Img2Ts(input_dataset=input_dataset, outputpath=outputpath,
startdate=startdate, enddate=enddate,
input_grid=ds_kwargs['grid'].cut(), # drop points that are not subset
imgbuffer=imgbuffer, cellsize_lat=5.0,
cellsize_lon=5.0, global_attr=global_attr, zlib=True,
unlim_chunksize=1000, ts_attributes=ts_attributes)
reshuffler.calc()
def test_EASE25CellGrid():
grid = EASE25CellGrid()
gpis, lons, lats, cells = grid.get_grid_points()
nptest.assert_almost_equal(np.array([gpis[0], lons[0], lats[0]]),
np.array([0, -179.8703, -83.5171]), 4)
assert grid.activegpis.size == (584 * 1388)
assert grid.activegpis[316922] == 316922
nptest.assert_almost_equal(grid.activearrlat[316922], -12.55398284007352,
5)
nptest.assert_almost_equal(grid.activearrlon[316922], -61.08069164265129,
5)
assert grid.activearrcell[316922] == 843
def test_SMOS_IC_Ds_subset(stackfile, files):
# test reading and storing a spatial subset (as stack/single img)
dsname = os.path.join(os.path.dirname(__file__), 'smos-test-data',
'L3_SMOS_IC', 'ASC')
subgrid = EASE25CellGrid(bbox=(-11., 34., 43., 71.))
params = ['Soil_Moisture', 'Quality_Flag']
ds = SMOSDs(dsname, parameters=params, grid=subgrid, read_flags=(0, 1))
image = ds.read(datetime(2018, 1, 1))
# europe subset
assert image.data['Soil_Moisture'].shape == (113, 208)
assert list(image.data.keys()) == params
assert np.nanmax(np.nanmax(image.data['Quality_Flag'])) == 1
assert np.nanmax(np.nanmin(image.data['Quality_Flag'])) == 0
# index in global file [508, 772]
nptest.assert_almost_equal(image.data['Soil_Moisture'][60, 120],
0.31218,
decimal=4)
nptest.assert_almost_equal(image.lon[60, 120], 20.36023, decimal=4)
nptest.assert_almost_equal(image.lat[60, 120], 47.682177, decimal=4)
nptest.assert_almost_equal(image.lon[25, 130], 22.95389, 4)
nptest.assert_almost_equal(image.lat[25, 130], 59.1181825, 4)
assert np.isnan(image.data['Soil_Moisture'][25, 130])
np.ma.is_masked(image.data['Quality_Flag'][25, 130])
outdir = mkdtemp()
os.makedirs(outdir, exist_ok=True)
ds.write_multiple(root_path=outdir,
start_date=datetime(2018, 1, 1),
end_date=datetime(2018, 1, 3),
stackfile=stackfile)
if stackfile is None:
assert os.path.isdir(os.path.join(outdir, '2018'))
assert len(os.listdir(os.path.join(
outdir, '2018'))) == 2 # Jan 2nd has no data
else:
ds.write_multiple(root_path=outdir,
start_date=datetime(2018, 1, 1),
end_date=datetime(2018, 1, 3),
stackfile=stackfile)
assert len(os.listdir(outdir)) == 1 # 1 stack
def __init__(self,
data_path,
parameters=None,
flatten=False,
grid=EASE25CellGrid(bbox=None),
filename_templ=None,
read_flags=(0, 1),
float_fillval=np.nan):
if filename_templ is None:
filename_templ = self.default_fname_template
super().__init__(data_path,
ioclass=SMOS_IC_Img,
parameters=parameters,
flatten=flatten,
grid=grid,
filename_templ=filename_templ,
read_flags=read_flags,
float_fillval=float_fillval)
def __init__(self, data_path, parameters=None, flatten=False,
grid=EASE25CellGrid(bbox=None), filename_templ=None,
read_flags=(0, 1), float_fillval=np.nan):
ioclass_kws = {'parameters': parameters,
'flatten': flatten,
'grid': grid,
'read_flags': read_flags,
'float_fillval': float_fillval}
sub_path = ['%Y']
if filename_templ is None:
filename_templ = self.default_fname_template
super(SMOSDs, self).__init__(data_path, ioclass=SMOSImg,
fname_templ=filename_templ,
datetime_format="%Y%m%d",
subpath_templ=sub_path,
exact_templ=False,
ioclass_kws=ioclass_kws)
def test_SMOS_L4OPER_Ds_subset(stackfile, files):
# test reading and storing a spatial subset (as stack/single img)
dsname = os.path.join(os.path.dirname(__file__),
'smos-test-data', 'L4_SMOS_RZSM', 'OPER')
subgrid = EASE25CellGrid(bbox=(-11., 34., 43., 71.))
params = ['RZSM', 'Quality']
ds = SMOS_L4_Ds(dsname, parameters=params, grid=subgrid, oper=True)
image = ds.read(datetime(2020, 1, 31))
# europe subset
assert image.data['RZSM'].shape == (113, 208)
assert list(image.data.keys()) == params
assert np.nanmax(np.nanmax(image.data['Quality'])) == 1
assert np.nanmax(np.nanmin(image.data['Quality'])) == 0
# index in global file [508, 772]
nptest.assert_almost_equal(image.data['RZSM'][60, 120], 0.18237862, decimal=4)
nptest.assert_almost_equal(image.lon[60, 120], 20.36023, decimal=4)
nptest.assert_almost_equal(image.lat[60, 120], 47.682177, decimal=4)
nptest.assert_almost_equal(image.lon[25, 130], 22.95389, 4)
nptest.assert_almost_equal(image.lat[25, 130], 59.1181825, 4)
assert np.isnan(image.data['RZSM'][25, 130])
outdir = mkdtemp()
os.makedirs(outdir, exist_ok=True)
ds.write_multiple(root_path=outdir, start_date=datetime(2020, 1, 31),
end_date=datetime(2020, 2, 1), stackfile=stackfile)
if stackfile is None:
assert os.path.isdir(os.path.join(outdir, '2020'))
assert len(os.listdir(os.path.join(outdir, '2020'))) == 1 # Feb 1st has no data
else:
ds.write_multiple(root_path=outdir, start_date=datetime(2020, 1, 31),
end_date=datetime(2020, 2, 1), stackfile=stackfile)
assert len(os.listdir(outdir)) == 1 # 1 stack
print("Bazinga!")
def __init__(self, data_path, ioclass, parameters=None, flatten=False,
grid=EASE25CellGrid(bbox=None), filename_templ=None,
read_flags=(0, 1), float_fillval=np.nan, additional_kws=None):
ioclass_kws = {'parameters': parameters,
'flatten': flatten,
'grid': grid,
'read_flags': read_flags,
'float_fillval': float_fillval}
if additional_kws:
ioclass_kws.update(additional_kws)
sub_path = ['%Y']
super().__init__(data_path,
ioclass=ioclass,
fname_templ=filename_templ,
datetime_format="%Y%m%d",
subpath_templ=sub_path,
exact_templ=False,
ioclass_kws=ioclass_kws)
def __init__(self,
data_path,
parameters=None,
flatten=False,
grid=EASE25CellGrid(bbox=None),
filename_templ=None,
read_flags=np.linspace(0, 1, 6, endpoint=True),
oper=False,
float_fillval=np.nan):
if filename_templ is None:
filename_templ = self.default_fname_template
super().__init__(data_path,
ioclass=SMOS_L4_Img,
parameters=parameters,
flatten=flatten,
grid=grid,
filename_templ=filename_templ,
read_flags=read_flags,
float_fillval=float_fillval,
additional_kws={'oper': oper})
for id in country_ids:
ids.append(id)
return ids
def continent_countries(self, *continents):
if isinstance(continents, str):
continents = [continents]
names = np.array([])
for continent in continents:
n = self.df.loc[self.df.continent == continent, 'country'].values
names = np.append(names, n)
return names
if __name__ == '__main__':
country = 'Morocco'
from smos.grid import EASE25CellGrid
#from smecv_grid.grid import SMECV_Grid_v052
grid = EASE25CellGrid()
from pygeogrids.netcdf import save_grid
adapter = GridShpAdapter(grid)
print(adapter)
grid = adapter.create_subgrid([country])
save_grid(
r"R:\Projects\SMART-DRI\07_data\sm_country_data\SMOS-IC\ease25grid_{country}.nc"
.format(country=country),
grid=grid)