def test_resample_dtypes():
"""
Test if dtypes stay the same when resampling.
"""
data = {
'testint8': np.array([5, 5], dtype=np.int8),
'testfloat16': np.array([5, 5], dtype=np.float16)
}
fill_values = {'testint8': 0, 'testfloat16': 999.}
lons = np.array([0, 0.1])
lats = np.array([0, 0.1])
# lets resample to a 0.1 degree grid
# define the grid points in latitude and longitude
lats_dim = np.arange(-1, 1, 0.1)
lons_dim = np.arange(-1, 1, 0.1)
# make 2d grid out the 1D grid spacing
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
resampled_data = resample.resample_to_grid(data,
lons,
lats,
lons_grid,
lats_grid,
fill_values=fill_values)
for key in data:
assert resampled_data[key].shape == lons_grid.shape
assert resampled_data[key].dtype == data[key].dtype
def test_resample_dtypes():
"""
Test if dtypes stay the same when resampling.
"""
data = {'testint8': np.array([5, 5], dtype=np.int8),
'testfloat16': np.array([5, 5], dtype=np.float16)}
fill_values = {'testint8': 0,
'testfloat16': 999.}
lons = np.array([0, 0.1])
lats = np.array([0, 0.1])
# lets resample to a 0.1 degree grid
# define the grid points in latitude and longitude
lats_dim = np.arange(-1, 1, 0.1)
lons_dim = np.arange(-1, 1, 0.1)
# make 2d grid out the 1D grid spacing
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
resampled_data = resample.resample_to_grid(data, lons, lats,
lons_grid, lats_grid,
fill_values=fill_values)
for key in data:
assert resampled_data[key].shape == lons_grid.shape
assert resampled_data[key].dtype == data[key].dtype
def resample_rho_LC(input_data,
src_lons,
src_lats,
target_lons,
target_lats,
search_rad=18000):
if src_lons.shape != src_lats.shape:
src_lons, src_lats = np.meshgrid(src_lons, src_lats)
if target_lons.shape != target_lats.shape:
target_lons, target_lats = np.meshgrid(target_lons, target_lats)
data_resamp = resample_to_grid({'data': input_data},
src_lons,
src_lats,
target_lons,
target_lats,
methods='nn',
weight_funcs=None,
min_neighbours=1,
search_rad=search_rad,
neighbours=8,
fill_values=None)
return data_resamp['data']
def test_resample_to_zero_dot_one_deg(self):
data, meta, timestamp, lons, lats, time_var = self.reader.read_img(
datetime.datetime(2010, 5, 1, 8, 33, 1))
# lets resample to a 0.1 degree grid
# define the grid points in latitude and longitude
lats_dim = np.arange(25, 75, 0.1)
lons_dim = np.arange(-25, 45, 0.1)
# make 2d grid out the 1D grid spacing
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
resampled_data = resample.resample_to_grid(data, lons, lats,
lons_grid, lats_grid)
for key in data:
assert resampled_data[key].shape == lons_grid.shape
def test_resample_to_zero_dot_one_deg(self):
data, meta, timestamp, lons, lats, time_var = self.reader.read(
datetime.datetime(2010, 5, 1, 8, 33, 1))
# lets resample to a 0.1 degree grid
# define the grid points in latitude and longitude
lats_dim = np.arange(25, 75, 0.1)
lons_dim = np.arange(-25, 45, 0.1)
# make 2d grid out the 1D grid spacing
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
resampled_data = resample.resample_to_grid(data, lons, lats, lons_grid,
lats_grid)
for key in data:
assert resampled_data[key].shape == lons_grid.shape
def test_resample_hamming():
"""
Test if hamming window is applied correctly
"""
# let's do 5 points with the highest value in the middle
# -1-
# 151
# -1-
data = {'testfloat16': np.array([1, 1, 5, 1, 1], dtype=np.float16)}
fill_values = {'testfloat16': 999.}
lons = np.array([0, -0.1, 0, 0.1, 0])
lats = np.array([0.1, 0, 0, 0, -0.1])
# lets resample to a 0.1 degree grid
# define the grid points in latitude and longitude
lats_dim = np.arange(-0.1, 0.11, 0.1)
lons_dim = np.arange(-0.1, 0.11, 0.1)
# make 2d grid out the 1D grid spacing
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
# make partial function of the hamming window the radius of the hamming
# window is in meters not in degrees
hamm = functools.partial(resample.hamming_window, 15000)
resampled_data = resample.resample_to_grid(data,
lons,
lats,
lons_grid,
lats_grid,
fill_values=fill_values,
methods='custom',
weight_funcs=hamm)
resampled_should = np.array([[1.640625, 1.64160156, 1.640625],
[1.64160156, 3.11132812, 1.64160156],
[1.640625, 1.64160156, 1.640625]])
for key in data:
assert resampled_data[key].shape == lons_grid.shape
assert resampled_data[key].dtype == data[key].dtype
nptest.assert_almost_equal(resampled_data[key], resampled_should)
def test_resample_hamming():
"""
Test if hamming window is applied correctly
"""
# let's do 5 points with the highest value in the middle
# -1-
# 151
# -1-
data = {'testfloat16': np.array([1, 1, 5, 1, 1], dtype=np.float16)}
fill_values = {'testfloat16': 999.}
lons = np.array([0, -0.1, 0, 0.1, 0])
lats = np.array([0.1, 0, 0, 0, -0.1])
# lets resample to a 0.1 degree grid
# define the grid points in latitude and longitude
lats_dim = np.arange(-0.1, 0.11, 0.1)
lons_dim = np.arange(-0.1, 0.11, 0.1)
# make 2d grid out the 1D grid spacing
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
# make partial function of the hamming window the radius of the hamming
# window is in meters not in degrees
hamm = functools.partial(resample.hamming_window, 15000)
resampled_data = resample.resample_to_grid(data, lons, lats,
lons_grid, lats_grid,
fill_values=fill_values,
methods='custom', weight_funcs=hamm)
resampled_should = np.array([[1.640625, 1.64160156, 1.640625],
[1.64160156, 3.11132812, 1.64160156],
[1.640625, 1.64160156, 1.640625]])
for key in data:
assert resampled_data[key].shape == lons_grid.shape
assert resampled_data[key].dtype == data[key].dtype
nptest.assert_almost_equal(resampled_data[key], resampled_should)
def resample_to_shape(source_file,
region,
sp_res,
grid,
prefix=None,
nan_value=None,
dest_nan_value=None,
variables=None,
shapefile=None):
"""
Resamples images and clips country boundaries
Parameters
----------
source_file : str
Path to source file.
region : str
Identifier of the region in the shapefile. If the default shapefile is
used, this would be the FIPS country code.
sp_res : int or float
Spatial resolution of the shape-grid.
grid : poets.grid.RegularGrid or poets.grid.ShapeGrid
Grid to resample data to.
prefix : str, optional
Prefix for the variable in the NetCDF file, should be name of source
nan_value : int, float, optional
Not a number value of the original data as given by the data provider
dest_nan_value : int or float, optional
NaN value used in the final NetCDF file.
variables : list of str, optional
Variables to resample from original file.
shapefile : str, optional
Path to shape file, uses "world country admin boundary shapefile" by
default.
Returns
-------
res_data : dict of numpy.arrays
resampled image
dest_lon : numpy.array
longitudes of the points in the resampled image
dest_lat : numpy.array
latitudes of the points in the resampled image
gpis : numpy.array
grid point indices
timestamp : datetime.date
date of the image
metadata : dict
Metadata derived from input file.
"""
if prefix is not None:
prefix += '_'
fileExtension = os.path.splitext(source_file)[1].lower()
if region == 'global':
lon_min = -180
lon_max = 180
lat_min = -90
lat_max = 90
else:
shp = Shape(region, shapefile)
lon_min = shp.bbox[0]
lon_max = shp.bbox[2]
lat_min = shp.bbox[1]
lat_max = shp.bbox[3]
if fileExtension in ['.nc', '.nc3', '.nc4']:
data_src, lon, lat, timestamp, metadata = nc.read_image(
source_file, variables)
if (lon_min >= lon.max() or lon_max <= lon.min()
or lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in ['.h5']:
data_src, lon, lat, timestamp, metadata = h5.read_image(
source_file, variables)
if (lon_min >= lon.max() or lon_max <= lon.min()
or lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in imgfiletypes:
data, src_lon, src_lat, timestamp, metadata = bbox_img(
source_file, region, fileExtension, shapefile)
if nan_value is not None:
for key in data.keys():
data[key] = np.ma.array(data[key], mask=(data[key] == nan_value))
src_lon, src_lat = np.meshgrid(src_lon, src_lat)
lons = grid.arrlon[0:grid.shape[0]]
dest_lon, dest_lat = np.meshgrid(lons, np.unique(grid.arrlat)[::-1])
gpis = grid.get_bbox_grid_points(grid.arrlat.min(), grid.arrlat.max(),
grid.arrlon.min(), grid.arrlon.max())
search_rad = 180000 * sp_res
data = resample.resample_to_grid(data,
src_lon,
src_lat,
dest_lon,
dest_lat,
search_rad=search_rad)
res_data = {}
path = []
if region != 'global':
_, _, multipoly = shp._get_shape()
for ring in multipoly:
poly_verts = list(ring.exterior.coords)
path.append(matplotlib.path.Path(poly_verts))
coords = [grid.arrlon, grid.arrlat[::-1]]
coords2 = np.zeros((len(coords[0]), 2))
for idx in range(0, len(coords[0])):
coords2[idx] = [coords[0][idx], coords[1][idx]]
mask_old = path[0].contains_points(coords2)
for key in data.keys():
if variables is not None:
if key not in variables:
del metadata[key]
continue
if region != 'global':
for ring in path:
mask_new = (ring.contains_points(coords2))
mask_rev = scipy.logical_or(mask_old, mask_new)
mask_old = mask_rev
mask_rev = mask_rev.reshape(dest_lon.shape)
mask = np.invert(mask_rev)
mask[data[key].mask == True] = True
else:
mask = data[key].mask
if prefix is None:
var = key
else:
var = prefix + key
if metadata is not None:
metadata[var] = metadata[key]
if var != key:
del metadata[key]
res_data[var] = np.ma.masked_array(data[key],
mask=np.copy(mask),
fill_value=dest_nan_value)
dat = np.copy(res_data[var].data)
dat[mask == True] = dest_nan_value
res_data[var] = np.ma.masked_array(dat,
mask=np.copy(mask),
fill_value=dest_nan_value)
return res_data, dest_lon, dest_lat, gpis, timestamp, metadata
def resample_to_shape(source_file, region, sp_res, grid, prefix=None,
nan_value=None, dest_nan_value=None, variables=None,
shapefile=None):
"""
Resamples images and clips country boundaries
Parameters
----------
source_file : str
Path to source file.
region : str
Identifier of the region in the shapefile. If the default shapefile is
used, this would be the FIPS country code.
sp_res : int or float
Spatial resolution of the shape-grid.
grid : poets.grid.RegularGrid or poets.grid.ShapeGrid
Grid to resample data to.
prefix : str, optional
Prefix for the variable in the NetCDF file, should be name of source
nan_value : int, float, optional
Not a number value of the original data as given by the data provider
dest_nan_value : int or float, optional
NaN value used in the final NetCDF file.
variables : list of str, optional
Variables to resample from original file.
shapefile : str, optional
Path to shape file, uses "world country admin boundary shapefile" by
default.
Returns
-------
res_data : dict of numpy.arrays
resampled image
dest_lon : numpy.array
longitudes of the points in the resampled image
dest_lat : numpy.array
latitudes of the points in the resampled image
gpis : numpy.array
grid point indices
timestamp : datetime.date
date of the image
metadata : dict
Metadata derived from input file.
"""
if prefix is not None:
prefix += '_'
fileExtension = os.path.splitext(source_file)[1].lower()
if region == 'global':
lon_min = -180
lon_max = 180
lat_min = -90
lat_max = 90
else:
shp = Shape(region, shapefile)
lon_min = shp.bbox[0]
lon_max = shp.bbox[2]
lat_min = shp.bbox[1]
lat_max = shp.bbox[3]
if fileExtension in ['.nc', '.nc3', '.nc4']:
data_src, lon, lat, timestamp, metadata = nc.read_image(source_file,
variables)
if (lon_min >= lon.max() or lon_max <= lon.min() or
lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in ['.h5']:
data_src, lon, lat, timestamp, metadata = h5.read_image(source_file,
variables)
if (lon_min >= lon.max() or lon_max <= lon.min() or
lat_max <= lat.min() or lat_min >= lat.max()):
return "No data"
data, src_lon, src_lat = nc.clip_bbox(data_src, lon, lat, lon_min,
lat_min, lon_max, lat_max)
elif fileExtension in imgfiletypes:
data, src_lon, src_lat, timestamp, metadata = bbox_img(source_file,
region,
fileExtension,
shapefile)
if nan_value is not None:
for key in data.keys():
data[key] = np.ma.array(data[key], mask=(data[key] == nan_value))
src_lon, src_lat = np.meshgrid(src_lon, src_lat)
lons = grid.arrlon[0:grid.shape[0]]
dest_lon, dest_lat = np.meshgrid(lons, np.unique(grid.arrlat)[::-1])
gpis = grid.get_bbox_grid_points(grid.arrlat.min(), grid.arrlat.max(),
grid.arrlon.min(), grid.arrlon.max())
search_rad = 180000 * sp_res
data = resample.resample_to_grid(data, src_lon, src_lat, dest_lon,
dest_lat, search_rad=search_rad)
res_data = {}
path = []
if region != 'global':
_, _, multipoly = shp._get_shape()
for ring in multipoly:
poly_verts = list(ring.exterior.coords)
path.append(matplotlib.path.Path(poly_verts))
coords = [grid.arrlon, grid.arrlat[::-1]]
coords2 = np.zeros((len(coords[0]), 2))
for idx in range(0, len(coords[0])):
coords2[idx] = [coords[0][idx], coords[1][idx]]
mask_old = path[0].contains_points(coords2)
for key in data.keys():
if variables is not None:
if key not in variables:
del metadata[key]
continue
if region != 'global':
for ring in path:
mask_new = (ring.contains_points(coords2))
mask_rev = scipy.logical_or(mask_old, mask_new)
mask_old = mask_rev
mask_rev = mask_rev.reshape(dest_lon.shape)
mask = np.invert(mask_rev)
mask[data[key].mask == True] = True
else:
mask = data[key].mask
if prefix is None:
var = key
else:
var = prefix + key
if metadata is not None:
metadata[var] = metadata[key]
if var != key:
del metadata[key]
res_data[var] = np.ma.masked_array(data[key], mask=np.copy(mask),
fill_value=dest_nan_value)
dat = np.copy(res_data[var].data)
dat[mask == True] = dest_nan_value
res_data[var] = np.ma.masked_array(dat, mask=np.copy(mask),
fill_value=dest_nan_value)
return res_data, dest_lon, dest_lat, gpis, timestamp, metadata
# <rawcell>
# let's resample and plot the ssm data
# H07 data is not that easy to plot because it comes in orbit geometry and not on a fixed grid.
# <codecell>
# %matplotlib inline
# lets resample to a 0.1 degree grid
#define the grid points in latitude and logitude
lats_dim = np.arange(25, 75, 0.1)
lons_dim = np.arange(-25, 45, 0.1)
#make 2d grid out the 1D grid spacings
lons_grid, lats_grid = np.meshgrid(lons_dim, lats_dim)
resampled_data = resample.resample_to_grid(h07_data, lons, lats, lons_grid,
lats_grid)
fig = plt.figure(figsize=(10, 10))
ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])
# setup of basemap for europe
# simple mercator projection
m = Basemap(llcrnrlon=-25.0,llcrnrlat=25.0,urcrnrlon=45.0,urcrnrlat=75.0,\
resolution='l',area_thresh=1000.,projection='merc',\
lat_ts=50.,ax=ax)
# make a pseudocolor plot using the ASCAT SWI colormap
im = m.pcolormesh(lons_grid,
lats_grid,
resampled_data['ssm'],
latlon=True,
cmap=smcolormaps.load('SWI_ASCAT'))
# <rawcell>
# let's resample and plot the ssm data
# H07 data is not that easy to plot because it comes in orbit geometry and not on a fixed grid.
# <codecell>
# %matplotlib inline
# lets resample to a 0.1 degree grid
#define the grid points in latitude and logitude
lats_dim = np.arange(25,75,0.1)
lons_dim = np.arange(-25,45,0.1)
#make 2d grid out the 1D grid spacings
lons_grid, lats_grid = np.meshgrid(lons_dim,lats_dim)
resampled_data = resample.resample_to_grid(h07_data, lons, lats,
lons_grid, lats_grid)
fig = plt.figure(figsize=(10,10))
ax = fig.add_axes([0.1,0.1,0.8,0.8])
# setup of basemap for europe
# simple mercator projection
m = Basemap(llcrnrlon=-25.0,llcrnrlat=25.0,urcrnrlon=45.0,urcrnrlat=75.0,\
resolution='l',area_thresh=1000.,projection='merc',\
lat_ts=50.,ax=ax)
# make a pseudocolor plot using the ASCAT SWI colormap
im = m.pcolormesh(lons_grid, lats_grid, resampled_data['Surface Soil Moisture (Ms)'], latlon=True,
cmap=smcolormaps.load('SWI_ASCAT'))
m.drawcoastlines()
m.drawcountries()
def img_bulk(self):
"""
Yields numpy array of self.const.imgbuffer images,
start and enddate until all dates have been read
Returns
-------
img_stack_dict : dict of numpy.array
stack of daily images for each variable
startdate : date
date of first image in stack
enddate : date
date of last image in stack
datetimestack : np.array
array of the timestamps of each image
jd_stack : np.array or None
if None all observations in an image have the same
observation timestamp. Otherwise it gives the julian date
of each observation in img_stack_dict
"""
img_dict = {}
datetimes = []
jd_list = []
# set start of current imgbulk to startdate
bulkstart = self.startdate
# image counter
read_images = 0
dates = self.imgin.tstamps_for_daterange(self.startdate,
self.enddate)
for date in dates:
try:
(input_img, metadata,
image_datetime, lon,
lat, time_arr) = self.imgin.read(date, **self.input_kwargs)
except IOError as e:
msg = "I/O error({0}): {1}".format(e.errno,
e.strerror)
logging.log(logging.INFO, msg)
continue
read_images += 1
logging.log(logging.INFO, "read" + image_datetime.isoformat())
if self.resample:
if time_arr is not None:
input_img['jd'] = time_arr
input_img = resamp.resample_to_grid(input_img, lon, lat,
self.target_grid.activearrlon,
self.target_grid.activearrlat,
methods=self.r_methods,
weight_funcs=self.r_weightf,
min_neighbours=self.r_min_n,
search_rad=self.r_radius,
neighbours=self.r_neigh,
fill_values=self.r_fill_values)
time_arr = input_img.pop('jd')
if time_arr is None:
self.time_var = None
else:
self.time_var = 'jd'
if time_arr is not None:
# if time_var is not None means that each observation of the
# image has its own observation time
# this means that the resulting time series is not
# regularly spaced in time
if self.orthogonal is None:
self.orthogonal = False
if self.orthogonal:
raise Img2TsError("Images can not switch between a fixed image "
"timestamp and individual timestamps for each observation")
jd_list.append(time_arr)
if time_arr is None:
if self.orthogonal is None:
self.orthogonal = True
if not self.orthogonal:
raise Img2TsError(
"Images can not switch between a fixed image "
"timestamp and individual timestamps for each observation")
for key in input_img:
if key not in img_dict.keys():
img_dict[key] = []
img_dict[key].append(input_img[key])
datetimes.append(image_datetime)
if read_images >= self.imgbuffer - 1:
img_stack_dict = {}
if len(jd_list) != 0:
jd_stack = np.ma.vstack(jd_list)
jd_list = None
else:
jd_stack = None
for key in img_dict:
img_stack_dict[key] = np.vstack(img_dict[key])
img_dict[key] = None
datetimestack = np.array(datetimes)
img_dict = {}
datetimes = []
jd_list = []
yield (img_stack_dict, bulkstart, self.currentdate,
datetimestack, jd_stack)
# reset image counter
read_images = 0
if len(datetimes) > 0:
img_stack_dict = {}
if len(jd_list) != 0:
jd_stack = np.ma.vstack(jd_list)
else:
jd_stack = None
for key in img_dict:
img_stack_dict[key] = np.vstack(img_dict[key])
img_dict[key] = None
datetimestack = np.array(datetimes)
yield (img_stack_dict, bulkstart, self.currentdate, datetimestack,
jd_stack)