def read(self, timestamp=None):
#TODO: Replace ERA5 missing data (!=0)
return_img = {}
return_metadata = {}
try:
dataset = Dataset(self.filename)
except IOError as e:
print(e)
print(" ".join([self.filename, "can not be opened"]))
raise e
res_lat, res_lon = get_grid_resolution(
dataset.variables['latitude'][:],
dataset.variables['longitude'][:])
self.grid = ERA_RegularImgGrid(
res_lat, res_lon) if not self.subgrid else self.subgrid
for parameter, variable in dataset.variables.items():
if parameter in self.parameter:
param_metadata = {}
param_data = {}
for attrname in variable.ncattrs():
param_metadata.update(
{str(attrname): getattr(variable, attrname)})
param_data = variable[:]
param_data = param_data.flatten()
return_img.update(
{str(parameter): param_data[self.grid.activegpis]})
return_metadata.update({str(parameter): param_metadata})
try:
return_img[parameter]
except KeyError:
path, thefile = os.path.split(self.filename)
print('%s in %s is corrupt - filling'
'image with NaN values' % (parameter, thefile))
return_img[parameter] = np.empty(self.grid.n_gpi).fill(
np.nan)
return_metadata['corrupt_parameters'].append()
dataset.close()
if self.array_1D:
return Image(self.grid.activearrlon, self.grid.activearrlat,
return_img, return_metadata, timestamp)
else:
for key in return_img:
nlat = np.unique(self.grid.activearrlat).size
nlon = np.unique(self.grid.activearrlon).size
return_img[key] = return_img[key].reshape((nlat, nlon))
return Image(self.grid.activearrlon.reshape(nlat, nlon),
self.grid.activearrlat.reshape(nlat, nlon),
return_img, return_metadata, timestamp)
def read(self, timestamp=None):
"""
Read data from the loaded image file.
Parameters
---------
timestamp : datetime, optional (default: None)
Specific date (time) to read the data for.
"""
grbs = pygrib.open(self.filename)
grid = self.subgrid
return_img = {}
return_metadata = {}
var_msg_lut = {p: None for p in self.parameter}
sea_mask = None
for N in range(grbs.messages):
n = N + 1
message = grbs.message(n)
param_name = str(message.cfVarNameECMF)
if param_name == "lsm":
if self.mask_seapoints and sea_mask is None:
sea_mask = message.values.flatten()
if param_name not in self.parameter:
continue
else:
var_msg_lut[param_name] = n
# available variables
shape = None
for param_name, n in var_msg_lut.items():
if n is None:
continue
return_metadata[param_name] = {}
message = grbs.message(n)
param_data = message.values.flatten()
if not shape:
shape = param_data.shape
return_img[param_name] = param_data
if grid is None:
lats, lons = message.latlons()
try:
res_lat, res_lon = get_grid_resolution(lats, lons)
grid = ERA_RegularImgGrid(res_lat, res_lon)
except ValueError: # when grid not regular
lons_gt_180 = np.where(lons > 180.0)
lons[lons_gt_180] = lons[lons_gt_180] - 360
grid = ERA_IrregularImgGrid(lons, lats)
return_metadata[param_name]["units"] = message["units"]
return_metadata[param_name]["long_name"] = \
message["parameterName"]
if "levels" in message.keys():
return_metadata[param_name]["depth"] = "{:} cm".format(
message["levels"])
if self.mask_seapoints:
if sea_mask is None:
raise IOError(
"No land sea mask parameter (lsm) in passed image"
" for masking.")
else:
# mask the loaded data
for name in return_img.keys():
param_data = return_img[name]
param_data = np.ma.array(
param_data,
mask=np.logical_not(sea_mask),
fill_value=np.nan,
)
param_data = param_data.filled()
return_img[name] = param_data
grbs.close()
# missing variables
for param_name, n in var_msg_lut.items():
if n is not None:
continue
param_data = np.full(shape, np.nan)
warnings.warn("Cannot load variable {var} from file {thefile}. "
"Filling image with NaNs.".format(
var=param_name, thefile=self.filename))
return_img[param_name] = param_data
return_metadata[param_name] = {}
return_metadata[param_name]["long_name"] = lookup(
self.product, [param_name]).iloc[0]["long_name"]
if self.array_1D:
return Image(
grid.activearrlon,
grid.activearrlat,
return_img,
return_metadata,
timestamp,
)
else:
nlat = np.unique(grid.activearrlat).size
nlon = np.unique(grid.activearrlon).size
for key in return_img:
return_img[key] = return_img[key].reshape((nlat, nlon))
return Image(
grid.activearrlon.reshape(nlat, nlon),
grid.activearrlat.reshape(nlat, nlon),
return_img,
return_metadata,
timestamp,
)
def read(self, timestamp=None):
"""
Read data from the loaded image file.
Parameters
---------
timestamp : datetime, optional (default: None)
Specific date (time) to read the data for.
"""
return_img = {}
return_metadata = {}
try:
dataset = xr.open_dataset(self.filename,
engine="netcdf4",
mask_and_scale=True)
except IOError as e:
print(e)
print(" ".join([self.filename, "can not be opened"]))
raise e
res_lat, res_lon = get_grid_resolution(
dataset.variables["latitude"][:],
dataset.variables["longitude"][:])
grid = (ERA_RegularImgGrid(res_lat, res_lon)
if not self.subgrid else self.subgrid)
if self.mask_seapoints:
if "lsm" not in dataset.variables.keys():
raise IOError("No land sea mask parameter (lsm) in"
" passed image for masking.")
else:
sea_mask = dataset.variables["lsm"].values
for name in dataset.variables:
if name in self.parameter:
variable = dataset[name]
param_data = variable.data
if self.mask_seapoints:
param_data = np.ma.array(
param_data,
mask=np.logical_not(sea_mask),
fill_value=np.nan,
)
param_data = param_data.filled()
param_data = param_data.flatten()
return_metadata[name] = variable.attrs
return_img.update(
dict([(str(name), param_data[grid.activegpis])]))
try:
return_img[name]
except KeyError:
path, thefile = os.path.split(self.filename)
warnings.warn(
"Cannot load variable {var} from file {thefile}. "
"Filling image with NaNs.".format(var=name,
thefile=thefile))
return_img[name] = np.empty(grid.activegpis.size).fill(
np.nan)
dataset.close()
if self.array_1D:
return Image(
grid.activearrlon,
grid.activearrlat,
return_img,
return_metadata,
timestamp,
)
else:
nlat = np.unique(grid.activearrlat).size
nlon = np.unique(grid.activearrlon).size
for key in return_img:
return_img[key] = return_img[key].reshape((nlat, nlon))
return Image(
grid.activearrlon.reshape(nlat, nlon),
grid.activearrlat.reshape(nlat, nlon),
return_img,
return_metadata,
timestamp,
)
def test_ERA_regular_grid():
reg_grid = ERA_RegularImgGrid(0.3, 0.3)
assert np.unique(reg_grid.activearrlat).size == 601
assert np.unique(reg_grid.activearrlon).size == 1200
assert get_grid_resolution(reg_grid.activearrlat,
reg_grid.activearrlon) == (0.3, 0.3)
def read(self, timestamp=None):
'''
Read data from the loaded image file.
Parameters
---------
timestamp : datetime, optional (default: None)
Specific date (time) to read the data for.
'''
grbs = pygrib.open(self.filename)
grid = self.subgrid
return_img = {}
return_metadata = {}
sea_mask = None
for message in grbs:
param_name = message.short_name
if param_name == 'lsm':
if self.mask_seapoints and sea_mask is None:
sea_mask = message.values.flatten()
param_name = message.short_name
if param_name not in self.parameter:
continue
return_metadata[param_name] = {}
param_data = message.values.flatten()
return_img[param_name] = param_data
if grid is None:
lats, lons = message.latlons()
try:
res_lat, res_lon = get_grid_resolution(lats, lons)
grid = ERA_RegularImgGrid(res_lat, res_lon)
except ValueError: # when grid not regular
lons_gt_180 = np.where(lons > 180.0)
lons[lons_gt_180] = lons[lons_gt_180] - 360
grid = ERA_IrregularImgGrid(lons, lats)
return_metadata[param_name]['units'] = message['units']
return_metadata[param_name]['long_name'] = message['parameterName']
if 'levels' in message.keys():
return_metadata[param_name]['depth'] = '{:} cm'.format(
message['levels'])
if self.mask_seapoints:
if sea_mask is None:
raise IOError(
'No land sea mask parameter (lsm) in passed image for masking.'
)
else:
# mask the loaded data
for name in return_img.keys():
param_data = return_img[name]
param_data = np.ma.array(param_data,
mask=np.logical_not(sea_mask),
fill_value=np.nan)
param_data = param_data.filled()
return_img[name] = param_data
grbs.close()
if self.array_1D:
return Image(grid.activearrlon, grid.activearrlat, return_img,
return_metadata, timestamp)
else:
nlat = np.unique(grid.activearrlat).size
nlon = np.unique(grid.activearrlon).size
for key in return_img:
return_img[key] = return_img[key].reshape((nlat, nlon))
return Image(grid.activearrlon.reshape(nlat, nlon),
grid.activearrlat.reshape(nlat, nlon), return_img,
return_metadata, timestamp)
