def reshuffle(input_root, outputpath,
startdate, enddate,
parameters=None, land_points=True, ignore_meta=False,
imgbuffer=200):
"""
Reshuffle method applied to ESA CCI SM images.
Parameters
----------
input_root: string
input path where era interim data was downloaded
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
parameters: list, optional (default: None)
parameters to read and convert
If none are passed, we read an image in the root path and use vars from
the image.
land_points : bool, optional (default: True)
Use the land grid to calculate time series on.
Leads to faster processing and smaller files.
imgbuffer: int, optional
How many images to read at once before writing time series.
"""
if land_points:
grid = CCILandGrid()
else:
grid = CCICellGrid()
if not os.path.exists(outputpath):
os.makedirs(outputpath)
file_args, file_vars = parse_filename(input_root)
if parameters is None:
parameters = [p for p in file_vars if p not in ['lat', 'lon', 'time']]
input_dataset = CCI_SM_025Ds(data_path=input_root, parameter=parameters,
subgrid=grid, array_1D=True)
if not ignore_meta:
global_attr, ts_attributes = read_metadata(sensortype=file_args['sensor_type'],
version=int(file_args['version']),
varnames=parameters)
global_attr['time_coverage_start'] = str(startdate)
global_attr['time_coverage_end'] = str(enddate)
else:
global_attr = {'product' : 'ESA CCI SM'}
ts_attributes = None
reshuffler = Img2Ts(input_dataset=input_dataset, outputpath=outputpath,
startdate=startdate, enddate=enddate, input_grid=grid,
imgbuffer=imgbuffer, cellsize_lat=5.0, cellsize_lon=5.0,
global_attr=global_attr, zlib=True,
unlim_chunksize=1000, ts_attributes=ts_attributes)
reshuffler.calc()
def test_CCI_SM_v052_025Img_img_reading_1D_combined():
"""
1D test for the read function of the CCI_SM_image class
"""
##### global grid
parameter = ['sm']
filename = os.path.join(
os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v05.2", "combined", "2016",
"ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-20160607000000-fv05.2.nc")
img_c = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=None,
array_1D=True)
image_c = img_c.read()
assert img_c.grid.find_nearest_gpi(75.625, 14.625) == (602942, 0)
ref_lat = image_c.lat[434462]
ref_lon = image_c.lon[434462]
assert ref_lon == 75.625
assert ref_lat == 14.625
assert sorted(image_c.data.keys()) == sorted(parameter)
ref_sm = image_c.data['sm'][1440 * 301 + 1022]
nptest.assert_almost_equal(ref_sm, 0.37016, 5)
###### land grid
land_grid = CCILandGrid()
img_c = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=land_grid,
array_1D=True)
assert img_c.grid.find_nearest_gpi(75.625, 14.625) == (602942, 0)
image_c = img_c.read()
sm = image_c.data['sm'][177048]
lat = image_c.lat[177048]
lon = image_c.lon[177048]
assert ref_lat == lat
assert ref_lon == lon
nptest.assert_almost_equal(ref_sm, sm, 5)
def test_landgrid(self):
grid = CCILandGrid()
gp, dist = grid.find_nearest_gpi(75.625, 14.625)
assert gp == 602942
lon, lat = grid.gpi2lonlat(602942)
assert lon == 75.625
assert lat == 14.625
assert np.where(
grid.get_grid_points()[0] == 602942)[0][0] == 177048 # index
assert grid.get_grid_points()[1][177048] == lon
assert grid.get_grid_points()[2][177048] == lat
assert grid.gpi2cell(602942) == 1856
assert grid.gpis.size == 1036800
assert grid.activegpis.size == 244243
assert grid.gpis[0] == 1035360
assert grid.activegpis[0] == 999942
assert np.unique(grid.get_grid_points()[3]).size == 1001
def test_CCI_SM_v042_025Img_img_reading_1D_active():
"""
1D test for the read function of the CCI_SM_v042_025Img class
_a for active
_p for passive
_c for combined
"""
parameter = ['sm']
filename = os.path.join(
os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v04.2", "active", "2016",
"ESACCI-SOILMOISTURE-L3S-SSMS-ACTIVE-20160606000000-fv04.2.nc")
img_a = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=None,
array_1D=True)
image_a = img_a.read()
ref_lat = image_a.lat[1440 * 93] # = 133920
ref_lon = image_a.lon[1440 * 93]
assert ref_lon == -179.875
assert ref_lat == 66.625
assert sorted(image_a.data.keys()) == sorted(parameter)
ref_sm = image_a.data['sm'][133920]
assert abs(ref_sm - 100.) <= 1e-3
###land_grid
land_grid = CCILandGrid()
img_a = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=land_grid,
array_1D=True)
assert img_a.grid.find_nearest_gpi(-179.875, 66.625) == (901440, 0)
image_a = img_a.read()
sm = image_a.data['sm'][32830]
lat = image_a.lat[32830]
lon = image_a.lon[32830]
assert ref_lat == lat
assert ref_lon == lon
nptest.assert_almost_equal(ref_sm, sm, 5)
def test_CCI_SM_v042_025Img_img_reading_1D_combined():
"""
1D test for the read function of the CCI_SM_v042_025Img class
_a for active
_p for passive
_c for combined
"""
parameter = ['sm']
filename = os.path.join(
os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v04.2", "combined", "2016",
"ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-20160607000000-fv04.2.nc")
img_c = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=None,
array_1D=True)
image_c = img_c.read()
ref_lat = image_c.lat[1440 * 85] # = 122400
ref_lon = image_c.lon[1440 * 85]
assert ref_lon == -179.875
assert ref_lat == 68.625
assert sorted(image_c.data.keys()) == sorted(parameter)
ref_sm = image_c.data['sm'][122400]
assert abs(ref_sm - 0.1755) <= 1e-3
###land_grid
land_grid = CCILandGrid()
img_c = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=land_grid,
array_1D=True)
assert img_c.grid.find_nearest_gpi(-179.875, 68.625) == (912960, 0)
image_c = img_c.read()
sm = image_c.data['sm'][24434]
lat = image_c.lat[24434]
lon = image_c.lon[24434]
assert ref_lat == lat
assert ref_lon == lon
nptest.assert_almost_equal(ref_sm, sm, 5)
def test_CCI_SM_v033_025Img_img_reading_1D_combined():
"""
1D test for the read function of the CCI_SM_v042_025Img class
"""
filename = os.path.join(
os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v03.3", "combined", "2016",
"ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-20160101000000-fv03.3.nc")
parameter = ['sm']
img_c = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=None,
array_1D=True)
image_c = img_c.read()
ref_lat = image_c.lat[1440 * 273 + 693]
ref_lon = image_c.lon[1440 * 273 + 693]
assert ref_lon == -6.625
assert ref_lat == 21.625
assert sorted(image_c.data.keys()) == sorted(parameter)
ref_sm = image_c.data['sm'][1440 * 273 + 693]
nptest.assert_almost_equal(ref_sm, 0.142998, 5)
###### land grid
land_grid = CCILandGrid()
img_c = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=land_grid,
array_1D=True)
assert img_c.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0)
image_c = img_c.read()
sm = image_c.data['sm'][164759]
lat = image_c.lat[164759]
lon = image_c.lon[164759]
assert ref_lat == lat
assert ref_lon == lon
nptest.assert_almost_equal(ref_sm, sm, 5)
def test_CCI_SM_v033_025Img_img_reading_1D_active():
filename = os.path.join(
os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v03.3", "active", "2016",
"ESACCI-SOILMOISTURE-L3S-SSMS-ACTIVE-20160101000000-fv03.3.nc")
parameter = ['sm']
img_a = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=None,
array_1D=True)
image_a = img_a.read()
ref_lat = image_a.lat[1440 * 273 + 693]
ref_lon = image_a.lon[1440 * 273 + 693]
assert ref_lon == -6.625
assert ref_lat == 21.625
assert sorted(image_a.data.keys()) == sorted(parameter)
ref_sm = image_a.data['sm'][1440 * 273 + 693]
nptest.assert_almost_equal(ref_sm, 18.92771, 5)
image_a = img_a.read()
###### land grid
land_grid = CCILandGrid()
img_a = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=land_grid,
array_1D=True)
assert img_a.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0)
image_a = img_a.read()
sm = image_a.data['sm'][164759]
lat = image_a.lat[164759]
lon = image_a.lon[164759]
assert ref_lat == lat
assert ref_lon == lon
nptest.assert_almost_equal(ref_sm, sm, 5)
def test_CCI_SM_v052_025Img_img_reading_1D_passive():
parameter = ['sm']
filename = os.path.join(
os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v05.2", "passive", "2016",
"ESACCI-SOILMOISTURE-L3S-SSMV-PASSIVE-20160606000000-fv05.2.nc")
img_p = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=None,
array_1D=True)
image_p = img_p.read()
assert img_p.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0)
ref_lat = image_p.lat[393813]
ref_lon = image_p.lon[393813]
assert ref_lon == -6.625
assert ref_lat == 21.625
assert sorted(image_p.data.keys()) == sorted(parameter)
ref_sm = image_p.data['sm'][1440 * 273 + 693]
nptest.assert_almost_equal(ref_sm, 0.0600, 5)
###### land grid
land_grid = CCILandGrid()
img_p = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=land_grid,
array_1D=True)
assert img_p.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0)
image_p = img_p.read()
sm = image_p.data['sm'][164759]
lat = image_p.lat[164759]
lon = image_p.lon[164759]
assert ref_lat == lat
assert ref_lon == lon
nptest.assert_almost_equal(ref_sm, sm, 5)
def test_CCI_SM_v042_025Img_img_reading_1D_passive():
parameter = ['sm']
filename = os.path.join(
os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v04.2", "passive", "2016",
"ESACCI-SOILMOISTURE-L3S-SSMV-PASSIVE-20160606000000-fv04.2.nc")
img_p = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=None,
array_1D=True)
image_p = img_p.read()
ref_lat = image_p.lat[1440 * 86] # = 123840
ref_lon = image_p.lon[1440 * 86]
assert ref_lon == -179.875
assert ref_lat == 68.375
assert sorted(image_p.data.keys()) == sorted(parameter)
ref_sm = image_p.data['sm'][123840]
assert abs(ref_sm - 0.31) <= 1e-3
###land_grid
land_grid = CCILandGrid()
img_a = CCI_SM_025Img(filename=filename,
parameter=parameter,
subgrid=land_grid,
array_1D=True)
assert img_a.grid.find_nearest_gpi(-179.875, 68.375) == (911520, 0)
image_a = img_a.read()
sm = image_a.data['sm'][25410]
lat = image_a.lat[25410]
lon = image_a.lon[25410]
assert ref_lat == lat
assert ref_lon == lon
nptest.assert_almost_equal(ref_sm, sm, 5)
def reshuffle(input_root,
outputpath,
startdate,
enddate,
parameters=None,
land_points=True,
ignore_meta=False,
imgbuffer=200):
"""
Reshuffle method applied to ESA CCI SM images.
Parameters
----------
input_root: string
input path where era interim data was downloaded
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
parameters: list, optional (default: None)
parameters to read and convert
If none are passed, we read an image in the root path and use vars from
the image.
land_points : bool, optional (default: True)
Use the land grid to calculate time series on.
Leads to faster processing and smaller files.
imgbuffer: int, optional
How many images to read at once before writing time series.
"""
if land_points:
grid = CCILandGrid()
else:
grid = CCICellGrid()
gpis, lons, lats, cells = grid.get_grid_points()
grid_vars = {'gpis': gpis, 'lons': lons, 'lats': lats}
# repurpose cannot handle masked arrays
for k, v in grid_vars.items(): # type v: np.ma.MaskedArray
if isinstance(v, np.ma.MaskedArray):
grid_vars[k] = v.filled()
grid = BasicGrid(lon=grid_vars['lons'],
lat=grid_vars['lats'],
gpis=grid_vars['gpis']).to_cell_grid(5.)
if not os.path.exists(outputpath):
os.makedirs(outputpath)
file_args, file_vars = parse_filename(input_root)
if parameters is None:
parameters = [p for p in file_vars if p not in ['lat', 'lon', 'time']]
input_dataset = CCI_SM_025Ds(data_path=input_root,
parameter=parameters,
subgrid=grid,
array_1D=True)
if not ignore_meta:
global_attr, ts_attributes = read_metadata(
sensortype=file_args['sensor_type'],
version=int(file_args['version']),
varnames=parameters,
subversion=file_args['sub_version'])
else:
global_attr = {'product': 'ESA CCI SM'}
ts_attributes = None
reshuffler = Img2Ts(input_dataset=input_dataset,
outputpath=outputpath,
startdate=startdate,
enddate=enddate,
input_grid=grid,
imgbuffer=imgbuffer,
cellsize_lat=5.0,
cellsize_lon=5.0,
global_attr=global_attr,
zlib=True,
unlim_chunksize=1000,
ts_attributes=ts_attributes)
reshuffler.calc()