def test_CCI_SM_v042_025Ds_img_reading():
"""
test for the reading of the CCI_SM_v042_025Ds class
_a for active
_p for passive
_c for combined
"""
parameter = ['sm']
data_path = os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v04.2", "combined")
img_c = CCI_SM_025Ds(data_path=data_path,
parameter=parameter,
subgrid=None,
array_1D=True)
image_c = img_c.read(datetime(2016, 6, 7, 0))
assert sorted(image_c.data.keys()) == sorted(parameter)
assert image_c.timestamp == datetime(2016, 6, 7, 0)
assert abs(image_c.data['sm'][122400] -
0.1755) <= 1e-3 # 1440 * 85 = 1222400
assert image_c.lon.shape == image_c.lat.shape == (720 * 1440, )
img_a = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v04.2", "active"),
parameter=parameter,
array_1D=True)
image_a = img_a.read(datetime(2016, 6, 6, 0))
assert sorted(image_a.data.keys()) == sorted(parameter)
assert image_a.timestamp == datetime(2016, 6, 6, 0)
assert abs(image_a.data['sm'][133920] - 100) <= 1e-3 # 93*1440
assert image_a.lon.shape == (360 * 180 * (1 / 0.25)**2, )
assert image_a.lon.shape == image_a.lat.shape
img_p = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v04.2", "passive"),
parameter=parameter,
array_1D=True)
image_p = img_p.read(datetime(2016, 6, 6, 0))
assert sorted(image_p.data.keys()) == sorted(parameter)
assert image_p.timestamp == datetime(2016, 6, 6, 0)
assert abs(image_p.data['sm'][123840] - 0.31) <= 1e-3 # 86*1440
assert image_p.lon.shape == (360 * 180 * (1 / 0.25)**2, )
assert image_p.lon.shape == image_p.lat.shape
def test_CCI_SM_v033_025Ds_timestamps_for_daterange():
"""
test for the tstamps_for_daterange function of the CCI_SM_v042_025Ds class
"""
parameter = ['sm']
should_datetime = [
datetime(2016, 1, 1, 0),
datetime(2016, 1, 1, 3),
datetime(2016, 1, 1, 6),
datetime(2016, 1, 1, 9),
datetime(2016, 1, 1, 12),
datetime(2016, 1, 1, 15),
datetime(2016, 1, 1, 18),
datetime(2016, 1, 1, 21)
]
img_a = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v03.3", "active"),
parameter=parameter,
array_1D=True)
tstamps_a = img_a.tstamps_for_daterange(datetime(2016, 1, 1),
datetime(2016, 1, 1))
assert len(tstamps_a) == 8
assert tstamps_a == should_datetime
img_p = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v03.3", "passive"),
parameter=parameter,
array_1D=True)
tstamps_p = img_p.tstamps_for_daterange(datetime(2016, 1, 1),
datetime(2016, 1, 1))
assert len(tstamps_p) == 8
assert tstamps_p == should_datetime
img_c = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v03.3", "combined"),
parameter=parameter,
array_1D=True)
tstamps_c = img_c.tstamps_for_daterange(datetime(2016, 1, 1),
datetime(2016, 1, 1))
assert len(tstamps_c) == 8
assert tstamps_c == should_datetime
def reshuffle(input_root, outputpath,
startdate, enddate,
parameters=None, land_points=True, ignore_meta=False,
imgbuffer=200):
"""
Reshuffle method applied to ESA CCI SM images.
Parameters
----------
input_root: string
input path where era interim data was downloaded
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
parameters: list, optional (default: None)
parameters to read and convert
If none are passed, we read an image in the root path and use vars from
the image.
land_points : bool, optional (default: True)
Use the land grid to calculate time series on.
Leads to faster processing and smaller files.
imgbuffer: int, optional
How many images to read at once before writing time series.
"""
if land_points:
grid = CCILandGrid()
else:
grid = CCICellGrid()
if not os.path.exists(outputpath):
os.makedirs(outputpath)
file_args, file_vars = parse_filename(input_root)
if parameters is None:
parameters = [p for p in file_vars if p not in ['lat', 'lon', 'time']]
input_dataset = CCI_SM_025Ds(data_path=input_root, parameter=parameters,
subgrid=grid, array_1D=True)
if not ignore_meta:
global_attr, ts_attributes = read_metadata(sensortype=file_args['sensor_type'],
version=int(file_args['version']),
varnames=parameters)
global_attr['time_coverage_start'] = str(startdate)
global_attr['time_coverage_end'] = str(enddate)
else:
global_attr = {'product' : 'ESA CCI SM'}
ts_attributes = None
reshuffler = Img2Ts(input_dataset=input_dataset, outputpath=outputpath,
startdate=startdate, enddate=enddate, input_grid=grid,
imgbuffer=imgbuffer, cellsize_lat=5.0, cellsize_lon=5.0,
global_attr=global_attr, zlib=True,
unlim_chunksize=1000, ts_attributes=ts_attributes)
reshuffler.calc()
def test_CCI_SM_v052_025Ds_img_reading():
"""
test for the reading of the CCI_SM_DS class
"""
parameter = ['sm']
data_path = os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v05.2", "combined")
img_c = CCI_SM_025Ds(data_path=data_path,
parameter=parameter,
subgrid=None,
array_1D=False)
image_c = img_c.read(datetime(2016, 6, 7, 0))
assert sorted(image_c.data.keys()) == sorted(parameter)
assert image_c.timestamp == datetime(2016, 6, 7, 0)
assert abs(image_c.data['sm'][273, 693] - 0.06405) <= 1e-3 ##
assert image_c.lon.shape == image_c.lat.shape == (720, 1440)
data_path = os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v05.2", "active")
img_a = CCI_SM_025Ds(data_path=data_path,
parameter=parameter,
subgrid=None,
array_1D=False)
image_a = img_a.read(datetime(2016, 6, 6, 0))
assert sorted(image_a.data.keys()) == sorted(parameter)
assert image_a.timestamp == datetime(2016, 6, 6, 0)
assert abs(image_a.data['sm'][273, 693] - 0.65197) <= 1e-3
assert image_a.lon.shape == image_a.lat.shape == (720, 1440)
data_path = os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v05.2", "passive")
img_p = CCI_SM_025Ds(data_path=data_path,
parameter=parameter,
subgrid=None,
array_1D=False)
image_p = img_p.read(datetime(2016, 6, 6, 0))
assert sorted(image_p.data.keys()) == sorted(parameter)
assert image_p.timestamp == datetime(2016, 6, 6, 0)
assert abs(image_p.data['sm'][273, 693] - 0.0600) <= 1e-3
assert image_p.lon.shape == image_p.lat.shape == (720, 1440)
def test_CCI_SM_v033_025Ds_img_reading():
"""
test for the reading of the CCI_SM_v033_025Ds class
"""
parameter = ['sm']
img_c = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v03.3", "combined"),
parameter=parameter,
array_1D=False)
image_c = img_c.read(datetime(2016, 1, 1, 0))
assert sorted(image_c.data.keys()) == sorted(parameter)
assert image_c.timestamp == datetime(2016, 1, 1, 0)
nptest.assert_almost_equal(image_c.data['sm'][203, 693], 0.23484, 5)
assert image_c.lon.shape == image_c.lat.shape == (720, 1440)
img_a = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v03.3", "active"),
parameter=parameter,
array_1D=False)
image_a = img_a.read(datetime(2016, 1, 1, 0))
assert sorted(image_a.data.keys()) == sorted(parameter)
assert image_a.timestamp == datetime(2016, 1, 1, 0)
nptest.assert_almost_equal(image_a.data['sm'][203, 693], 67.70157, 5)
assert image_c.lon.shape == image_c.lat.shape == (720, 1440)
img_c = CCI_SM_025Ds(data_path=os.path.join(os.path.dirname(__file__),
"esa_cci_sm-test-data",
"esa_cci_sm_dailyImages",
"v03.3", "passive"),
parameter=parameter,
array_1D=False)
image_c = img_c.read(datetime(2016, 1, 1, 0))
assert sorted(image_c.data.keys()) == sorted(parameter)
assert image_c.timestamp == datetime(2016, 1, 1, 0)
nptest.assert_almost_equal(image_c.data['sm'][203, 693], 0.322685, 5)
assert image_c.lon.shape == image_c.lat.shape == (720, 1440)
def test_CCI_SM_v052_025Ds_timestamps_for_daterange():
"""
test for the tstamps_for_daterange function of the CCI_SM_DS class
"""
parameter = ['sm']
should_datetime = [datetime(2016, 6, 6, 0), datetime(2016, 6, 7, 0)]
data_path = os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v05.2", "active")
img_a = CCI_SM_025Ds(data_path=data_path,
parameter=parameter,
subgrid=None,
array_1D=True)
tstamps_a = img_a.tstamps_for_daterange(datetime(2016, 6, 6),
datetime(2016, 6, 7))
assert len(tstamps_a) == 2
assert tstamps_a == should_datetime
data_path = os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v05.2", "passive")
img_p = CCI_SM_025Ds(data_path=data_path,
parameter=parameter,
subgrid=None,
array_1D=True)
tstamps_p = img_p.tstamps_for_daterange(datetime(2016, 6, 6),
datetime(2016, 6, 7))
assert len(tstamps_p) == 2
assert tstamps_p == should_datetime
data_path = os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data",
"esa_cci_sm_dailyImages", "v02.2", "combined")
img_c = CCI_SM_025Ds(data_path=data_path,
parameter=parameter,
subgrid=None,
array_1D=True)
tstamps_c = img_c.tstamps_for_daterange(datetime(2016, 6, 7),
datetime(2016, 6, 8))
assert len(tstamps_c) == 2
assert tstamps_c == [datetime(2016, 6, 7, 0), datetime(2016, 6, 8, 0)]
def reshuffle(input_root,
outputpath,
startdate,
enddate,
parameters=None,
land_points=True,
ignore_meta=False,
imgbuffer=200):
"""
Reshuffle method applied to ESA CCI SM images.
Parameters
----------
input_root: string
input path where era interim data was downloaded
outputpath : string
Output path.
startdate : datetime
Start date.
enddate : datetime
End date.
parameters: list, optional (default: None)
parameters to read and convert
If none are passed, we read an image in the root path and use vars from
the image.
land_points : bool, optional (default: True)
Use the land grid to calculate time series on.
Leads to faster processing and smaller files.
imgbuffer: int, optional
How many images to read at once before writing time series.
"""
if land_points:
grid = CCILandGrid()
else:
grid = CCICellGrid()
gpis, lons, lats, cells = grid.get_grid_points()
grid_vars = {'gpis': gpis, 'lons': lons, 'lats': lats}
# repurpose cannot handle masked arrays
for k, v in grid_vars.items(): # type v: np.ma.MaskedArray
if isinstance(v, np.ma.MaskedArray):
grid_vars[k] = v.filled()
grid = BasicGrid(lon=grid_vars['lons'],
lat=grid_vars['lats'],
gpis=grid_vars['gpis']).to_cell_grid(5.)
if not os.path.exists(outputpath):
os.makedirs(outputpath)
file_args, file_vars = parse_filename(input_root)
if parameters is None:
parameters = [p for p in file_vars if p not in ['lat', 'lon', 'time']]
input_dataset = CCI_SM_025Ds(data_path=input_root,
parameter=parameters,
subgrid=grid,
array_1D=True)
if not ignore_meta:
global_attr, ts_attributes = read_metadata(
sensortype=file_args['sensor_type'],
version=int(file_args['version']),
varnames=parameters,
subversion=file_args['sub_version'])
else:
global_attr = {'product': 'ESA CCI SM'}
ts_attributes = None
reshuffler = Img2Ts(input_dataset=input_dataset,
outputpath=outputpath,
startdate=startdate,
enddate=enddate,
input_grid=grid,
imgbuffer=imgbuffer,
cellsize_lat=5.0,
cellsize_lon=5.0,
global_attr=global_attr,
zlib=True,
unlim_chunksize=1000,
ts_attributes=ts_attributes)
reshuffler.calc()
