def get_dataset(self, dataset_id, dataset_info):
"""Get dataset using the parameter_number key in dataset_info.
In a previous version of the reader, the attributes (nrows, ncols, ssp_lon) and projection information
(pdict and area_dict) were computed while initializing the file handler. Also the code would break out from
the While-loop below as soon as the correct parameter_number was found. This has now been revised becasue the
reader would sometimes give corrupt information about the number of messages in the file and the dataset
dimensions within a given message if the file was only partly read (not looping over all messages) in an earlier
instance.
"""
logger.debug(
'Reading in file to get dataset with parameter number %d.',
dataset_info['parameter_number'])
xarr = None
message_found = False
with open(self.filename, 'rb') as fh:
# Iterate over all messages and fetch data when the correct parameter number is found
while True:
gid = ec.codes_grib_new_from_file(fh)
if gid is None:
if not message_found:
# Could not obtain a valid message ID from the grib file
logger.warning(
"Could not find parameter_number %d in GRIB file, no valid Dataset created",
dataset_info['parameter_number'])
break
# Check if the parameter number in the GRIB message corresponds to the required key
parameter_number = self._get_from_msg(gid, 'parameterNumber')
if parameter_number == dataset_info['parameter_number']:
self._res = dataset_id.resolution
self._read_attributes(gid)
# Read the missing value
missing_value = self._get_from_msg(gid, 'missingValue')
# Retrieve values and metadata from the GRIB message, masking the values equal to missing_value
xarr = self._get_xarray_from_msg(gid)
xarr.data = da.where(xarr.data == missing_value, np.nan,
xarr.data)
ec.codes_release(gid)
# Combine all metadata into the dataset attributes and break out of the loop
xarr.attrs.update(dataset_info)
xarr.attrs.update(self._get_attributes())
message_found = True
else:
# The parameter number is not the correct one, release gid and skip to next message
ec.codes_release(gid)
return xarr
def cli(file_path):
with open(file_path, 'rb') as f:
handle = eccodes.codes_grib_new_from_file(f, headers_only=False)
while handle is not None:
date = eccodes.codes_get(handle, "dataDate")
type_of_level = eccodes.codes_get(handle, "typeOfLevel")
level = eccodes.codes_get(handle, "level")
points = eccodes.codes_grib_find_nearest(handle, 39.92, 116.46,
False, 1)
point = points[0]
print(date, type_of_level, level, " :", point.lat, point.lon,
point.value, point.distance)
eccodes.codes_release(handle)
handle = eccodes.codes_grib_new_from_file(f, headers_only=False)
def get_ecc_gids(filename):
f = open(filename)
msg_count = ecc.codes_count_in_file(f)
# TODO: this is only for grib files
gid_list = [ecc.codes_grib_new_from_file(f) for i in range(msg_count)]
#print(gid_list)
f.close()
return gid_list
def main():
#point = dict(lat = 49.014, lon = 8.404, name = 'Karlsruhe')
point = dict(lat = 50.822, lon = 8.920, name = 'Kirchhain')
#point = dict(lat = 65.661, lon =-18.718, name = 'Iceland')
path = dict(base = '/lsdfos/kit/imk-tro/projects/MOD/Gruppe_Knippertz/nw5893/',
data = 'forecast_archive/pamore/kirchhain/run_12Z/',
grid = 'forecast_archive/icon-eu-eps/grid/',
plots = 'plots/kirchhain/run_12Z/')
data_rain_gsp_sum = np.empty((25, 40, 75948))
data_rain_con_sum = np.empty((25, 40, 75948))
hours = list(range(0, 12+1))
filenames_all = []
for hour in hours:
filenames_all.append(['iefff0{:03d}0000.m0{:02d}'.format(hour, member) for member in range(1, 41)])
with ExitStack() as stack:
files_all = [[stack.enter_context(open(path['base'] + path['data'] + filename,'rb'))\
for filename in filenames_of_one_hour] for filenames_of_one_hour in filenames_all]
for i, files_of_one_hour in enumerate(files_all):
for j, file in enumerate(files_of_one_hour):
grib_id = eccodes.codes_grib_new_from_file(file)
data_rain_gsp_sum[i - hours[0], j, :] = eccodes.codes_get_array(grib_id, 'values')
grib_id = eccodes.codes_grib_new_from_file(file)
data_rain_con_sum[i - hours[0], j, :] = eccodes.codes_get_array(grib_id, 'values')
eccodes.codes_release(grib_id)
data_rain_tot_sum = data_rain_gsp_sum + data_rain_con_sum
del data_rain_gsp_sum, data_rain_con_sum, files_all
print(data_rain_tot_sum.max(axis=1).max(axis=1))
print('data_rain_tot_sum: {:.0f}MB'.format(data_rain_tot_sum.nbytes / 1e6))
stat_processing = 'max'
member = None
plot_rain_around_point(path, data_rain_tot_sum, 0, 12, point, stat_processing, member)
'''stat_processing = 'member_extract'
for member in range(1,41):
plot_rain_around_point(path, data_rain_tot_sum, 0, 24, point, stat_processing, member)'''
return
def load_bytes_from_index(index: GribMessageIndex) -> typing.Optional[bytes]:
with open(index.file_path, "rb") as f:
f.seek(index.offset)
message = eccodes.codes_grib_new_from_file(f)
if message is None:
return None
raw_bytes = eccodes.codes_get_message(message)
return raw_bytes
def cli(file_path):
with open(file_path, 'rb') as f:
handle = eccodes.codes_grib_new_from_file(f, headers_only=False)
if handle is None:
print("ERROR: unable to create handle from file " + file_path)
sys.exit(-1)
eccodes.codes_release(handle)
def keep_only_first_grib(fname: str):
with open(fname, "r+b") as infd:
gid = eccodes.codes_grib_new_from_file(infd)
try:
if eccodes.codes_get_message_offset(gid) != 0:
raise RuntimeError(
f"{fname}: first grib does not start at offset 0")
infd.truncate(eccodes.codes_get_message_size(gid))
finally:
eccodes.codes_release(gid)
def _index_grib_file(path, path_name=None):
import eccodes
with open(path, "rb") as f:
h = eccodes.codes_grib_new_from_file(f)
while h:
try:
field = dict()
if isinstance(path_name, str):
field["_path"] = path_name
elif path_name is False:
pass
elif path_name is None:
field["_path"] = path
else:
raise ValueError(
f"Value of path_name cannot be '{path_name}.'")
i = eccodes.codes_keys_iterator_new(h, "mars")
try:
while eccodes.codes_keys_iterator_next(i):
name = eccodes.codes_keys_iterator_get_name(i)
value = eccodes.codes_get_string(h, name)
field[name] = value
finally:
eccodes.codes_keys_iterator_delete(i)
field["_offset"] = eccodes.codes_get_long(h, "offset")
field["_length"] = eccodes.codes_get_long(h, "totalLength")
field["_param_id"] = eccodes.codes_get_string(h, "paramId")
field["param"] = eccodes.codes_get_string(h, "shortName")
yield field
finally:
eccodes.codes_release(h)
h = eccodes.codes_grib_new_from_file(f)
def from_file(cls, file, offset=None, **kwargs):
# type: (T.IO[bytes], T.Union[int, T.Tuple[int ,int], None], T.Any) -> Message
field_in_message = 0
if isinstance(offset, tuple):
offset, field_in_message = offset
if offset is not None:
file.seek(offset)
codes_id = None
if field_in_message == 0:
codes_id = eccodes.codes_grib_new_from_file(file)
else:
# MULTI-FIELD is enabled only when accessing additional fields
with multi_enabled(file):
for _ in range(field_in_message + 1):
codes_id = eccodes.codes_grib_new_from_file(file)
if codes_id is None:
raise EOFError("End of file: %r" % file)
return cls(codes_id=codes_id, **kwargs)
def _read_global_attributes(self):
"""Read the global product attributes from the first message.
Read the information about the date and time of the data product,
the projection and area definition and the number of messages.
"""
with open(self.filename, 'rb') as fh:
gid = ec.codes_grib_new_from_file(fh)
if gid is None:
# Could not obtain a valid message id: set attributes to None, number of messages to 0
logger.warning(
"Could not obtain a valid message id in GRIB file")
self._ssp_lon = None
self._nrows = None
self._ncols = None
self._pdict, self._area_dict = None, None
return
# Read SSP and date/time
self._ssp_lon = self._get_from_msg(
gid, 'longitudeOfSubSatellitePointInDegrees')
# Read number of points on the x and y axes
self._nrows = self._get_from_msg(gid, 'Ny')
self._ncols = self._get_from_msg(gid, 'Nx')
# Creates the projection and area dictionaries
self._pdict, self._area_dict = self._get_proj_area(gid)
# Determine the number of messages in the product by iterating until an invalid id is obtained
i = 1
ec.codes_release(gid)
while True:
gid = ec.codes_grib_new_from_file(fh)
if gid is None:
break
ec.codes_release(gid)
i = i + 1
def scan_for_gid(filename, short_name):
filee = open(filename, "rb")
for j in range(ec.codes_count_in_file(filee)):
gid = ec.codes_grib_new_from_file(filee, headers_only=True)
if ec.codes_get(gid, "shortName") == short_name:
filee.close()
return gid
else:
ec.codes_release(gid)
filee.close()
exit(1)
def scan_for_gid(filename, short_name, time_since_init, level):
filee = open(filename, "rb")
for j in np.arange(0, ec.codes_count_in_file(filee)):
gid = ec.codes_grib_new_from_file(filee, headers_only = True)
if ec.codes_get(gid, "shortName") == short_name and ec.codes_get(gid, "forecastTime") == time_since_init and ec.codes_get(gid, "level") == level:
filee.close()
return gid
else:
ec.codes_release(gid)
filee.close()
exit(1)
def _load_message_from_file_by_count(file_path, count):
current_index = 0
with open(file_path, "rb") as f:
while True:
message_id = eccodes.codes_grib_new_from_file(f)
if message_id is None:
return None
current_index += 1
if current_index == count:
return message_id
else:
eccodes.codes_release(message_id)
def confirm_packing_type(gribfile, packing_type):
"""Confirm that gribfile contains only GRIBs with specified packingType."""
comparisons = []
with open(gribfile) as infile:
while True:
gid = codes_grib_new_from_file(infile)
if gid is None:
break
encoded_type = codes_get(gid, "packingType")
codes_release(gid)
comparisons.append(encoded_type == packing_type)
return comparisons
def load_bytes_from_file(
file_path: str or Path,
parameter: str or typing.Dict,
level_type: str = None,
level: int = None,
) -> bytes or None:
"""
Load one message from grib file and return message's original bytes.
Parameters
----------
file_path
parameter
level_type
level
Returns
-------
bytes or None
Examples
--------
Load bytes of 850hPa temperature from GRAPES GFS GMF and create GRIB message using `eccodes.codes_new_from_message`.
>>> file_path = "/sstorage1/COMMONDATA/OPER/NWPC/GRAPES_GFS_GMF/Prod-grib/2020031721/ORIG/gmf.gra.2020031800105.grb2"
>>> message_bytes = load_bytes_from_file(
... file_path=file_path,
... parameter="t",
... level_type="pl",
... level=850,
... )
>>> message = eccodes.codes_new_from_message(message_bytes)
>>> values = eccodes.codes_get_double_array(message, "values")
>>> print(len(values))
1036800
"""
offset = 0
fixed_level_type = fix_level_type(level_type)
with open(file_path, "rb") as f:
while True:
message_id = eccodes.codes_grib_new_from_file(f)
length = eccodes.codes_get(message_id, "totalLength")
if message_id is None:
return None
if not _check_message(message_id, parameter, fixed_level_type,
level):
eccodes.codes_release(message_id)
offset += length
continue
return eccodes.codes_get_message(message_id)
def from_file(cls, file, offset=None, **kwargs):
# type: (T.IO[bytes], int, T.Any) -> Message
field_in_message = 0
if isinstance(offset, tuple):
offset, field_in_message = offset
if offset is not None:
file.seek(offset)
codes_id = None
# iterate over multi-fields in the message
for _ in range(field_in_message + 1):
codes_id = eccodes.codes_grib_new_from_file(file)
if codes_id is None:
raise EOFError("End of file: %r" % file)
return cls(codes_id=codes_id, **kwargs)
def gribs_match(left, right):
"""Check if GRIBs in both input files store the same data."""
comparisons = []
with open(left) as a, open(right) as b:
while True:
a_gid = codes_grib_new_from_file(a)
if a_gid is None:
break
b_gid = codes_grib_new_from_file(b)
if b_gid is None:
comparisons.append(False)
info("GRIBs contain unequal number of messages.")
continue
packing_errors = [0]
try:
packing_errors.append(codes_get(a_gid, "packingError"))
packing_errors.append(codes_get(b_gid, "packingError"))
except CodesInternalError:
pass
tolerance = max(packing_errors)
a_values = codes_get_values(a_gid)
b_values = codes_get_values(b_gid)
comparisons.append(np.allclose(a_values, b_values, atol=tolerance))
return comparisons
def get_grid_from_gribfile(filename, rotated=False):
f = open(filename)
# get grib message count and create gid_list, close filehandle
msg_count = codes.codes_count_in_file(f)
gid_list = [codes.codes_grib_new_from_file(f) for i in range(msg_count)]
f.close()
print("Working on grib-file: {0}".format(filename))
print("Message Count: {0}".format(msg_count))
# read grib grid details from given gid
gid = gid_list[0]
return get_grid_from_gid(gid, rotated=rotated)
def fetch_model_output(input_file, time_since_init, short_name, level): file = open(input_file, "rb") gid = ec.codes_grib_new_from_file(file) file.close() values = read_grib_array(input_file, short_name, time_since_init, level) lat = np.deg2rad(ec.codes_get_array(gid, "latitudes")) lon = np.deg2rad(ec.codes_get_array(gid, "longitudes")) no_of_columns = ec.codes_get_long(gid, "Ni") no_of_lines = ec.codes_get_long(gid, "Nj") ec.codes_release(gid) lat_vector = np.zeros([no_of_lines]) lon_vector = np.zeros([no_of_columns]) for i in range(no_of_lines): lat_vector[i] = lat[i*no_of_columns] for i in range(no_of_columns): lon_vector[i] = lon[i] return lat_vector, lon_vector, vector_2_array(values, no_of_lines, no_of_columns)
def index_grib_files(grib_files):
logging.info("Indexing grib files")
index = {}
index_keys = get_index_keys()
cnt = 0
for grib_file in grib_files[0]:
with open(grib_file) as fp:
message_no = 0
offset = 0
while True:
gid = ecc.codes_grib_new_from_file(fp)
if gid is None:
break
ref = index
for k in index_keys:
try:
val = ecc.codes_get_long(gid, k)
except gribapi.errors.KeyValueNotFoundError as e:
val = None
if val not in ref:
ref[val] = {}
ref = ref[val]
else:
ref = ref[val]
length = ecc.codes_get_long(gid, 'totalLength')
ref['file_name'] = grib_file
ref['message_no'] = message_no
ref['length'] = length
ref['offset'] = offset
# print(ref)
message_no += 1
offset += length
cnt += 1
logging.info(f"Indexed {cnt} messages from {len(grib_files[0])} file(s)")
return index
def cli(file_path):
with open(file_path, 'rb') as f:
handle = eccodes.codes_grib_new_from_file(f, headers_only=False)
date = eccodes.codes_get(handle, "dataDate")
type_of_level = eccodes.codes_get(handle, "typeOfLevel")
level = eccodes.codes_get(handle, "level")
iter_id = eccodes.codes_grib_iterator_new(handle, 0)
while 1:
result = eccodes.codes_grib_iterator_next(iter_id)
if not result:
break
[lat, lon, value] = result
print(lat, lon, value)
eccodes.codes_grib_iterator_delete(iter_id)
eccodes.codes_release(handle)
def get_dataset(self, dataset_id, dataset_info):
"""Get dataset using the parameter_number key in dataset_info."""
logger.debug(
'Reading in file to get dataset with parameter number %d.',
dataset_info['parameter_number'])
xarr = None
with open(self.filename, 'rb') as fh:
# Iterate until a message containing the correct parameter number is found
while True:
gid = ec.codes_grib_new_from_file(fh)
if gid is None:
# Could not obtain a valid message ID, break out of the loop
logger.warning(
"Could not find parameter_number %d in GRIB file, no valid Dataset created",
dataset_info['parameter_number'])
break
# Check if the parameter number in the GRIB message corresponds to the required key
parameter_number = self._get_from_msg(gid, 'parameterNumber')
if parameter_number != dataset_info['parameter_number']:
# The parameter number is not the correct one, skip to next message
ec.codes_release(gid)
continue
# Read the missing value
missing_value = self._get_from_msg(gid, 'missingValue')
# Retrieve values and metadata from the GRIB message, masking the values equal to missing_value
xarr = self._get_xarray_from_msg(gid)
xarr.where(xarr.data == missing_value, np.NaN)
ec.codes_release(gid)
# Combine all metadata into the dataset attributes and break out of the loop
xarr.attrs.update(dataset_info)
xarr.attrs.update(self._get_global_attributes())
break
return xarr
def repack(input_file, outfile, packing_type):
"""Repack infile with packing_type, write result to outfile."""
with open(input_file) as infile:
i = 1
while True:
in_gid = codes_grib_new_from_file(infile)
if in_gid is None:
break
info("Repacking GRIB #{}".format(i))
payload = codes_get_values(in_gid)
clone_id = codes_clone(in_gid)
codes_set(clone_id, "packingType", packing_type)
codes_set_values(clone_id, payload)
if i == 1:
mode = "w"
else:
mode = "a"
with open(outfile, mode) as output:
codes_write(clone_id, output)
codes_release(clone_id)
codes_release(in_gid)
i += 1
if not confirm_packing_type(outfile, packing_type):
raise EncodingError("Reencoding silently failed.")
def plot_contourmap(path, date, fcst_hours, variable, stat_processing_methods):
##### generate subpath and filename #####
subpath = 'run_{}{:02}{:02}{:02}/{}/'.format(\
date['year'], date['month'], date['day'], date['hour'], variable)
filename1 = 'icon-eu-eps_europe_icosahedral_single-level_{}{:02}{:02}{:02}_{:03}_{}.grib2'.format(\
date['year'], date['month'], date['day'], date['hour'], fcst_hours[0], variable)
filename2 = 'icon-eu-eps_europe_icosahedral_single-level_{}{:02}{:02}{:02}_{:03}_{}.grib2'.format(\
date['year'], date['month'], date['day'], date['hour'], fcst_hours[1], variable)
########################################################################
### read data ###
########################################################################
##### create empty numpy arrays #####
##### 2 fcst_hours, 40 members, 75948 eu gridpoints #####
data_raw = np.empty((2, 40, 75948))
data_members = np.empty((40, 75948))
##### every time in loop open next grib msg from grib file #####
##### grib messages in dwd file are sorted by increasing member number #####
with open(path['base'] + path['data'] + subpath + filename1, 'rb') as file:
for member in range(1, 41):
print('read data from member {}'.format(member))
grib_msg_id = eccodes.codes_grib_new_from_file(file)
data_raw[0, member - 1, :] = eccodes.codes_get_array(
grib_msg_id, 'values')
eccodes.codes_release(grib_msg_id)
with open(path['base'] + path['data'] + subpath + filename2, 'rb') as file:
for member in range(1, 41):
print('read data from member {}'.format(member))
grib_msg_id = eccodes.codes_grib_new_from_file(file)
data_raw[1, member - 1, :] = eccodes.codes_get_array(
grib_msg_id, 'values')
eccodes.codes_release(grib_msg_id)
##### take the difference of the two accumulated total precipitation arrays #####
data_members = data_raw[1, :, :] - data_raw[0, :, :]
del data_raw
##### open icon-eps grid file #####
icongrid_file = nc.Dataset(
path['base'] + path['grid'] + 'icon_grid_0028_R02B07_N02.nc', 'r')
vlat = icongrid_file.variables['clat_vertices'][:].data * 180. / np.pi
vlon = icongrid_file.variables['clon_vertices'][:].data * 180. / np.pi
clat = icongrid_file.variables['clat'][:].data * 180. / np.pi
clon = icongrid_file.variables['clon'][:].data * 180. / np.pi
icongrid_file.close()
for stat_processing in stat_processing_methods:
########################################################################
### statistically process data ###
########################################################################
if stat_processing == 'mean':
data_processed = data_members.mean(axis=0)
elif stat_processing == 'max':
data_processed = data_members.max(axis=0)
elif stat_processing == 'min':
data_processed = data_members.min(axis=0)
elif stat_processing == '90p':
data_processed = np.percentile(data_members, 90, axis=0)
elif stat_processing == '75p':
data_processed = np.percentile(data_members, 75, axis=0)
elif stat_processing == '50p':
data_processed = np.percentile(data_members, 50, axis=0)
elif stat_processing == '25p':
data_processed = np.percentile(data_members, 25, axis=0)
elif stat_processing == '10p':
data_processed = np.percentile(data_members, 10, axis=0)
#print('shape of data_members: {}'.format(np.shape(data_members)))
#print('shape of data_processed: {}'.format(np.shape(data_processed)))
########################################################################
### plot data on world map ###
########################################################################
##### set domain due to center point and radius #####
center_point = dict(lat=48.5, lon=9.0)
radius = 1800 # domain radius in km around center_point
domain = dict(
lat_min=center_point['lat'] - radius / 111.2,
lat_max=center_point['lat'] + radius / 111.2,
lon_min=center_point['lon'] - radius /
(111.2 * np.cos(center_point['lat'] * np.pi / 180)),
lon_max=center_point['lon'] + radius /
(111.2 * np.cos(center_point['lat'] * np.pi / 180)),
)
##### or set domain manually in deg N/E #####
'''domain = dict(
lat_min = 0.0,
lat_max = 20.0,
lon_min = 0.0,
lon_max = 20.0,
)'''
##### set image size (should be squared) #####
##### plotting area in pyngl can not exceed squared area even if plotting on rectangular images #####
##### for obtaining rectangular plots on has to cut manually afterwards e.g. with pillow package #####
x_resolution = 800
y_resolution = 800
wks_res = Ngl.Resources()
wks_res.wkWidth = x_resolution
wks_res.wkHeight = y_resolution
plot_name = 'contourplot_icon-eu-eps_tot_prec_{:02d}-{:02d}h_{}'.format(\
fcst_hours[0], fcst_hours[1], stat_processing)
wks_type = 'png'
wks = Ngl.open_wks(wks_type, path['base'] + path['plots'] + plot_name,
wks_res)
resources = Ngl.Resources(
) # create resources object containing all the plot settings
resources.mpProjection = 'Hammer' # projection type
resources.mpCenterLonF = (domain['lon_max'] + domain['lon_min']
) / 2 # projection center point
resources.mpCenterLatF = (domain['lat_max'] + domain['lat_min']) / 2
resources.mpLimitMode = 'latlon'
resources.mpMinLonF = domain['lon_min']
resources.mpMaxLonF = domain['lon_max']
resources.mpMinLatF = domain['lat_min']
resources.mpMaxLatF = domain['lat_max']
##### set plot area #####
resources.nglMaximize = False
resources.vpXF = 0.05
resources.vpYF = 0.9
resources.vpWidthF = 0.7
resources.vpHeightF = 0.7
##### set all map plot settings #####
resources.mpFillOn = True # turn on filled map areas
resources.mpFillColors = [
'pink', 'blue', 'white', 'white'
] # set colors for [FillValue, Ocean, Land , InlandWater]
resources.mpDataBaseVersion = 'MediumRes' # quality of national borders
resources.mpDataSetName = 'Earth..4'
resources.mpOutlineBoundarySets = 'national'
#resources.mpDataBaseVersion = 'HighRes'
#resources.mpDataResolution = 'Fine'
resources.mpGeophysicalLineThicknessF = 7.0 * x_resolution / 1000 # keep borders thickness resolution-independent
resources.mpNationalLineThicknessF = 7.0 * x_resolution / 1000
#resources.mpGridAndLimbDrawOrder = 'postdraw'
resources.mpGridAndLimbOn = False # turn off geographic coordinates grid
#resources.mpLimbLineColor = 'black'
#resources.mpLimbLineThicknessF = 10
#resources.mpGridLineColor = 'black'
#resources.mpGridLineThicknessF = 1.0
#resources.mpGridSpacingF = 1
resources.mpPerimOn = True # turn on perimeter around plot
resources.mpPerimLineColor = 'black'
resources.mpPerimLineThicknessF = 8.0 * x_resolution / 1000 # keep perimeter thickness resolution-independent
resources.tmXBOn = False # turn off location ticks around plot
resources.tmXTOn = False
resources.tmYLOn = False
resources.tmYROn = False
resources.sfDataArray = data_processed # data input file to plot
resources.sfXArray = clon # array with cell center locations
resources.sfYArray = clat
resources.sfXCellBounds = vlon # array with cell vertices locations
resources.sfYCellBounds = vlat
resources.sfMissingValueV = 9999 # in case you want to mask values
resources.cnFillOn = True
resources.cnFillMode = 'CellFill'
#resources.cnCellFillEdgeColor = 'black' # uncomment this for plotting the cell edges
resources.cnMissingValFillColor = 'black'
resources.cnFillPalette = 'WhiteBlueGreenYellowRed' # color palette
resources.cnLevelSelectionMode = 'ManualLevels'
minlevel = 0.0 # min level of colorbar
maxlevel = 50.0 # max level of colorbar
numberoflevels = 250 # number of levels of colorbar, max. 250 with this color palette
resources.cnMinLevelValF = minlevel
resources.cnMaxLevelValF = maxlevel
resources.cnLevelSpacingF = (maxlevel - minlevel) / numberoflevels
resources.cnLinesOn = False # turn off contour lines
resources.cnLineLabelsOn = False # turn off contour labels
##### set resources for a nice colorbar #####
resources.lbLabelBarOn = True
resources.lbAutoManage = False
resources.lbOrientation = 'vertical'
resources.lbLabelOffsetF = 0.05
#resources.lbBoxMinorExtentF = 0.2
resources.lbLabelStride = 25 # print a tick every 25 levels
resources.lbLabelFontHeightF = 0.016
resources.lbBoxSeparatorLinesOn = False
resources.lbBoxLineThicknessF = 4.0
#resources.lbBoxEndCapStyle = 'TriangleBothEnds'
resources.lbLabelAlignment = 'BoxCenters'
resources.lbTitleString = 'mm'
resources.lbTitleFontHeightF = 0.016
resources.lbTitlePosition = 'Right'
resources.lbTitleDirection = 'Across'
resources.lbTitleAngleF = 90.0
resources.lbTitleExtentF = 0.1
resources.lbTitleOffsetF = -0.15
resources.nglFrame = False # hold on frame because will plot text afterwards on same plot
Ngl.contour_map(wks, data_processed, resources) # plot the actual plot
##### plot title text #####
text = '{:02d}-{:02d}h Total Precipitation {}, ICON-EPS run {:02}.{:02}.{} {:02}Z'.format(\
fcst_hours[0], fcst_hours[1], stat_processing,\
date['day'], date['month'], date['year'], date['hour'])
x = 0.5
y = 0.95
text_res_1 = Ngl.Resources()
text_res_1.txFontHeightF = 0.018
text_res_1.txJust = 'TopCenter'
Ngl.text_ndc(wks, text, x, y, text_res_1)
Ngl.frame(wks) # advance frame
Ngl.destroy(
wks
) # delete workspace to free memory, relevant if plotting various plots in one script
print('plotted "{}.png"'.format(plot_name))
return
def load_message_from_file(
file_path: str or Path,
parameter: str or typing.Dict or None = None,
level_type: str or typing.Dict or None = None,
level: int or float or None = None,
**kwargs,
) -> int or None:
"""
Load the **first** message from GRIB 2 file using eccodes-python library.
Returned message is a copied one of original message and file is closed before return.
And the returned message should be released by user using `eccodes.codes_release()`.
Parameters
----------
file_path: str or Path
GRIB 2 file path.
parameter: str or typing.Dict
short name of the field or a dictionary including some GRIB keys:
- discipline
- parameterCategory
- parameterNumber
level_type: str or typing.Dict
level type.
level: int or float
level value.
kwargs: dict
ignored
Returns
-------
int or None
GRIB handler (int) if found or None if not found.
Examples
--------
Load 850hPa temperature from GRAPES GFS and get values from GRIB message.
>>> t = load_message_from_file(
... file_path="/g1/COMMONDATA/OPER/NWPC/GRAPES_GFS_GMF/Prod-grib/2020031721/ORIG/gmf.gra.2020031800105.grb2",
... parameter="t",
... level_type="isobaricInhPa",
... level=850,
... )
>>> data = eccodes.codes_get_double_array(t, "values")
>>> data = data.reshape([720, 1440])
>>> data
array([[249.19234375, 249.16234375, 249.16234375, ..., 249.15234375,
249.19234375, 249.14234375],
[249.45234375, 249.45234375, 249.42234375, ..., 249.45234375,
249.44234375, 249.44234375],
[249.69234375, 249.68234375, 249.68234375, ..., 249.70234375,
249.67234375, 249.68234375],
...,
[235.33234375, 235.45234375, 235.62234375, ..., 235.47234375,
235.63234375, 235.48234375],
[235.78234375, 235.91234375, 235.64234375, ..., 235.80234375,
235.72234375, 235.82234375],
[235.66234375, 235.86234375, 235.82234375, ..., 235.85234375,
235.68234375, 235.70234375]])
"""
fixed_level_type, _ = _fix_level(level_type, None)
with open(file_path, "rb") as f:
while True:
message_id = eccodes.codes_grib_new_from_file(f)
if message_id is None:
return None
if not _check_message(message_id, parameter, fixed_level_type, level):
eccodes.codes_release(message_id)
continue
# clone message
new_message_id = eccodes.codes_clone(message_id)
eccodes.codes_release(message_id)
return new_message_id
return None
def load_messages_from_file(
file_path: str or Path,
parameter: str or typing.Dict,
level_type: str or typing.Dict or typing.List or None = None,
level: int or float or typing.List or None = None,
**kwargs,
) -> typing.List or None:
"""
Load multiply messages from file.
This function will scan all messages in GRIB 2 file and return all messages which fit conditions.
Parameters
----------
file_path: str or Path
parameter: str or typing.Dict
see ``load_message_from_file``, required option.
level_type: str or typing.List or None
level type.
- string, same as ``load_message_from_file``
- typing.List, level type should be in the list.
- None, don't check level type.
level: int or float or typing.Dict or typing.List or None
level value.
- string, same as ``load_message_from_file``
- typing.Dict, same as ``load_message_from_file``
- typing.List, level value should be in the list.
- None, don't check level value. For example, load all messages of some typeOfLevel.
kwargs: dict
other parameters
Returns
-------
typing.List or None:
a list of message number or None if no message is found.
"""
fixed_level_type, _ = _fix_level(level_type, None)
messages = []
# print("count...")
# with open(file_path, "rb") as f:
# total_count = eccodes.codes_count_in_file(f)
# print(total_count)
# print("count..done")
with open(file_path, "rb") as f:
# pbar = tqdm(total=total_count)
while True:
message_id = eccodes.codes_grib_new_from_file(f)
if message_id is None:
break
# pbar.update(1)
if not _check_parameter(message_id, parameter):
eccodes.codes_release(message_id)
continue
if not _check_level_type(message_id, fixed_level_type):
eccodes.codes_release(message_id)
continue
if not _check_level_value(message_id, level):
eccodes.codes_release(message_id)
continue
# clone message
new_message_id = eccodes.codes_clone(message_id)
eccodes.codes_release(message_id)
messages.append(new_message_id)
# pbar.close()
if len(messages) == 0:
return None
return messages
def load_field_from_file(
file_path: str or Path,
parameter: str or typing.Dict or None = None,
level_type: str or typing.Dict or None = None,
level: int or float or typing.List or None = None,
level_dim: str or None = None,
show_progress: bool = False,
) -> xr.DataArray or None:
"""
Load **one** field from local GRIB2 file using eccodes-python.
Or load multi levels into one field.
Parameters
----------
file_path: str or Path
parameter: str or typing.Dict
parameter name.
Use GRIB key `shortName` or a dict of filter conditions such as:
{
"discipline": 0,
"parameterCategory": 2,
"parameterNumber": 225,
}
level_type: str or typing.Dict or None
level type.
- Use "pl", "ml" or "sfc". They will be converted into dict.
- Use GRIB key `typeOfLevel`, such as
- "isobaricInhPa"
- "isobaricInPa"
- "surface"
- "heightAboveGround"
- ...
See https://apps.ecmwf.int/codes/grib/format/edition-independent/3/ for more values.
- If `typeOfLevel` is not available, use dict to specify filter conditions.
For example, to get one filed from GRAPES GFS modelvar GRIB2 file, use:
{
"typeOfFirstFixedSurface": 131
}
level: int or float or typing.List or None
level value(s).
- If use a scalar, level will be a non-dimension coordinate.
- If your want to extract multi levels, use a list and level will be a dimension (level, lat, lon).
- If use `None`, all levels of level_type will be packed in the result field.
level_dim: str or None
name of level dimension.
If none, function will generate a name for level dim.
If `level_type="pl"`, some values can be used:
- `None` or `pl` or `isobaricInhPa`: level_dim is a float number with unit hPa.
- `isobaricInPa`: level_dim is a float number with unit Pa.
show_progress: bool
show progress bar.
Returns
-------
DataArray or None:
DataArray if found, or None if not.
Examples
--------
Load 850hPa temperature field from a GRIB2 file generated by GRAPES GFS.
>>> load_field_from_file(
... file_path="/sstorage1/COMMONDATA/OPER/NWPC/GRAPES_GFS_GMF/Prod-grib/2020031721/ORIG/gmf.gra.2020031800105.grb2",
... parameter="t",
... level_type="isobaricInhPa",
... level=850,
... )
<xarray.DataArray 't' (latitude: 720, longitude: 1440)>
array([[249.19234375, 249.16234375, 249.16234375, ..., 249.15234375,
249.19234375, 249.14234375],
[249.45234375, 249.45234375, 249.42234375, ..., 249.45234375,
249.44234375, 249.44234375],
[249.69234375, 249.68234375, 249.68234375, ..., 249.70234375,
249.67234375, 249.68234375],
...,
[235.33234375, 235.45234375, 235.62234375, ..., 235.47234375,
235.63234375, 235.48234375],
[235.78234375, 235.91234375, 235.64234375, ..., 235.80234375,
235.72234375, 235.82234375],
[235.66234375, 235.86234375, 235.82234375, ..., 235.85234375,
235.68234375, 235.70234375]])
Coordinates:
time datetime64[ns] 2020-03-18
step timedelta64[ns] 4 days 09:00:00
valid_time datetime64[ns] 2020-03-22T09:00:00
isobaricInhPa int64 850
* latitude (latitude) float64 89.88 89.62 89.38 ... -89.38 -89.62 -89.88
* longitude (longitude) float64 0.0 0.25 0.5 0.75 ... 359.2 359.5 359.8
Attributes:
GRIB_edition: 2
GRIB_centre: babj
GRIB_subCentre: 0
GRIB_tablesVersion: 4
GRIB_localTablesVersion: 1
GRIB_dataType: fc
GRIB_dataDate: 20200318
GRIB_dataTime: 0
GRIB_validityDate: 20200322
GRIB_validityTime: 900
GRIB_step: 105
GRIB_stepType: instant
GRIB_stepUnits: 1
GRIB_stepRange: 105
GRIB_endStep: 105
GRIB_name: Temperature
GRIB_shortName: t
GRIB_cfName: air_temperature
GRIB_discipline: 0
GRIB_parameterCategory: 0
GRIB_parameterNumber: 0
GRIB_gridType: regular_ll
GRIB_gridDefinitionDescription: Latitude/longitude
GRIB_typeOfFirstFixedSurface: pl
GRIB_typeOfLevel: isobaricInhPa
GRIB_level: 850
GRIB_numberOfPoints: 1036800
GRIB_missingValue: 9999
GRIB_units: K
long_name: Temperature
units: K
"""
messages = []
fixed_level_type, fixed_level_dim = _fix_level(level_type, level_dim)
if show_progress:
with open(file_path, "rb") as f:
total_count = eccodes.codes_count_in_file(f)
with open(file_path, "rb") as f:
if show_progress:
pbar = tqdm(
total=total_count,
desc="Filtering",
)
while True:
message_id = eccodes.codes_grib_new_from_file(f)
if message_id is None:
break
if show_progress:
pbar.update(1)
if not _check_message(message_id, parameter, fixed_level_type, level):
eccodes.codes_release(message_id)
continue
messages.append(message_id)
if isinstance(level, typing.List) or level is None:
continue
else:
break
if show_progress:
pbar.close()
if len(messages) == 0:
return None
if len(messages) == 1:
message_id = messages[0]
data = create_data_array_from_message(message_id, level_dim_name=fixed_level_dim)
eccodes.codes_release(message_id)
return data
if len(messages) > 1:
if show_progress:
pbar = tqdm(
total=len(messages),
desc="Decoding",
)
def creat_array(message):
array = create_data_array_from_message(message, level_dim_name=fixed_level_dim)
if show_progress:
pbar.update(1)
return array
xarray_messages = [creat_array(message) for message in messages]
for m in messages:
eccodes.codes_release(m)
if show_progress:
pbar.close()
if level_dim is None:
if isinstance(level_type, str):
level_dim_name = level_type
elif isinstance(level_type, typing.Dict):
level_dim_name = get_level_coordinate_name(xarray_messages[0])
else:
raise ValueError(f"level_type is not supported: {level_type}")
elif isinstance(level_dim, str):
level_dim_name = level_dim
else:
raise ValueError(f"level_type is not supported: {level_type}")
if show_progress:
print("Packing...")
data = xr.concat(xarray_messages, level_dim_name)
return data
return None
def read_grid_coordinates(model, grid):
path = dict(base='/')
path['grid'] = 'data/model_data/{}/grid/'.format(model)
if model == 'icon-eu-eps':
if grid == 'icosahedral':
filename = 'icon_grid_0028_R02B07_N02.nc'
elif grid == 'latlon_0.2':
filename = 'icon-eu-eps_latlon_0.2_grid_coordinates.nc'
elif model == 'icon-global-eps':
filename = 'icon_grid_0024_R02B06_G.nc'
elif model == 'icon-eu-det':
filename_clat = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2'
filename_clon = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2'
elif model == 'icon-global-det':
if grid == 'icosahedral':
filename = 'icon_grid_0026_R03B07_G.nc'
elif grid == 'latlon_0.25':
filename = 'icon-global-det_latlon_0.25_grid_coordinates.nc'
elif grid == 'latlon_0.1':
filename = 'icon-global-det_latlon_0.1_grid_coordinates.nc'
if grid == 'icosahedral':
with xr.open_dataset(path['base'] + path['grid'] + filename) as ds:
clat = ds['clat'].values * 180 / np.pi
clon = ds['clon'].values * 180 / np.pi
vlat = ds['clat_vertices'].values * 180 / np.pi
vlon = ds['clon_vertices'].values * 180 / np.pi
return clat, clon, vlat, vlon
elif grid == 'latlon_0.0625':
with open(path['base'] + path['grid'] + filename_clat, 'rb') as file:
grib_id = eccodes.codes_grib_new_from_file(file)
clat = eccodes.codes_get_array(grib_id, 'values')
eccodes.codes_release(grib_id)
with open(path['base'] + path['grid'] + filename_clon, 'rb') as file:
grib_id = eccodes.codes_grib_new_from_file(file)
clon = eccodes.codes_get_array(grib_id, 'values')
eccodes.codes_release(grib_id)
return clat.reshape((657, 1097)), clon.reshape((657, 1097))
elif grid == 'latlon_0.2':
with xr.open_dataset(path['base'] + path['grid'] + filename) as ds:
clat = ds['lat'].values
clon = ds['lon'].values
return clat, clon
elif grid == 'latlon_0.25':
with xr.open_dataset(path['base'] + path['grid'] + filename) as ds:
clat = ds['lat'].values
clon = ds['lon'].values
clon = np.where(clon > 180, clon - 360, clon)
clon_new = np.empty_like(clon)
clon_new[719:] = clon[:721]
clon_new[:719] = clon[721:]
return clat, clon_new
elif grid == 'latlon_0.1':
with xr.open_dataset(path['base'] + path['grid'] + filename) as ds:
clat = ds['lat'].values
clon = ds['lon'].values
clon = np.where(clon > 180, clon - 360, clon)
clon_new = np.empty_like(clon)
clon_new[1799:] = clon[:1801]
clon_new[:1799] = clon[1801:]
return clat, clon_new
def field(self, gribvar, time):
if eccodes is None:
raise Exception("eccodes not found. Needed for reading grib files")
"""
"""
geography = ["bitmapPresent",
"Nx",
"Ny",
"latitudeOfFirstGridPointInDegrees",
"longitudeOfFirstGridPointInDegrees",
"LoVInDegrees",
"DxInMetres",
"DyInMetres",
"iScansNegatively",
"jScansPositively",
"jPointsAreConsecutive",
"Latin1InDegrees",
"LaDInDegrees",
"Latin2InDegrees",
"latitudeOfSouthernPoleInDegrees",
"longitudeOfSouthernPoleInDegrees",
"gridType"
]
geo_out = None
fh = open(self.fname)
while 1:
gid = eccodes.codes_grib_new_from_file(fh)
if gid is None:
print("\nCould not find key")
gribvar.print_keys()
fh.close()
return None
else:
# print("\n Next key")
# print_grib_id(gid)
if gribvar.matches(gid):
# print("Found key")
# gribvar.print_keys()
geo = {}
for key in geography:
try:
geo.update({key: eccodes.codes_get(gid, key)})
except eccodes.CodesInternalError as err:
print('Error with key="%s" : %s' % (key, err.msg))
# print('There are %d values, average is %f, min is %f, max is %f' % (
# codes_get_size(gid, 'values'),
# codes_get(gid, 'average'),
# codes_get(gid, 'min'),
# codes_get(gid, 'max')
# ))
if geo["gridType"].lower() == "lambert":
values = eccodes.codes_get_values(gid)
print(values)
nx = geo["Nx"]
ny = geo["Ny"]
lon0 = geo["LoVInDegrees"]
lat0 = geo["LaDInDegrees"]
ll_lon = geo["longitudeOfFirstGridPointInDegrees"]
ll_lat = geo["latitudeOfFirstGridPointInDegrees"]
dx = geo["DxInMetres"]
dy = geo["DyInMetres"]
# TODO Check time consistency
print("Hopefullly valid for time ", time)
earth = 6.37122e+6
proj4 = "+proj=lcc +lat_0=" + str(lat0) + " +lon_0=" + str(lon0) + " +lat_1=" + \
str(lat0) + " +lat_2=" + str(lat0) + " +units=m +no_defs +R=" + str(earth)
proj = Proj(proj4)
x0, y0 = proj(ll_lon, ll_lat)
xc = x0 + 0.5 * (nx - 1) * dx
yc = y0 + 0.5 * (ny - 1) * dy
lonc, latc = proj(xc, yc, inverse=True)
field = np.reshape(values, [nx, ny], order="F")
if geo_out is None:
domain = {
"nam_conf_proj": {
"xlon0": lon0,
"xlat0": lat0
},
"nam_conf_proj_grid": {
"xloncen": lonc,
"xlatcen": latc,
"nimax": nx,
"njmax": ny,
"xdx": dx,
"xdy": dy,
"ilone": 0,
"ilate": 0
}
}
geo_out = surfex.geo.ConfProj(domain)
else:
raise NotImplementedError(geo["gridType"] + " not implemented yet!")
eccodes.codes_release(gid)
fh.close()
# print lons
# print lats
if geo_out is None:
raise Exception("No geometry is found in file")
return field, geo_out
eccodes.codes_release(gid)
def preprocess(self, inqueue: mp.Queue, consecutive: bool) -> None:
"""
Will read a request with rawfilepath and date from the queue.
If consecutive is set to true then it checks whether the requested date is consecutive
to the content of the netcdf file and puts the request back if it is not.
If set to False, the eventual netcdf time axis can have missing days and be non-monotonic.
This function then decodes the raw grib file, extracts hourly fields,
does the operation (even if just a simple hhUTC timeslice)
And will write to the netcdf.
"""
def extract_from_message(
messageid: int) -> Tuple[Dict[str, np.ma.MaskedArray], str]:
"""
Eccodes tools to extract the field and do a sanity check on the name or cfVarName
Returns a dictionary of the field, with endStep key, and additionally the units string
EndStep is important in case the self.operation wants a certain timestep extracted.
"""
name = ec.codes_get(messageid, 'name').lower().split(' ')
datestamp = str(ec.codes_get(messageid, 'dataDate'))
assert ('_'.join(name)
== self.encoding.loc['variable']) or (ec.codes_get(
messageid, 'cfVarName') == self.encoding.name)
assert pd.Timestamp(
year=int(datestamp[:4]),
month=int(datestamp[4:6]),
day=int(datestamp[6:])) == date # Date comes from one level up
# Extract the gridded values, reshape and mask.
values = ec.codes_get_values(messageid) # One dimensional array
lat_fastest_changing = (ec.codes_get(messageid,
'jPointsAreConsecutive') == 1)
values = values.reshape(
(ec.codes_get(messageid, 'Nj'), ec.codes_get(messageid, 'Ni')),
order='F' if lat_fastest_changing else
'C') # order C means last index fastest changing
if ec.codes_get(
messageid,
'latitudeOfFirstGridPointInDegrees') > ec.codes_get(
messageid, 'latitudeOfLastGridPointInDegrees'):
values = values[::
-1, :] # In my eventual netcdf storage I want the latitudes to increase with index
masked_values = np.ma.MaskedArray(data=values,
mask=(values == ec.codes_get(
messageid, 'missingValue')))
units = ec.codes_get(messageid, 'units')
timeinfo = str(
ec.codes_get(messageid, self.encoding.loc['timevariable']))
if timeinfo[-2:] == '00':
timeinfo = timeinfo[:
-2] # Remove the trailing zeros of the minutes
if len(timeinfo) == 1:
timeinfo = '0' + timeinfo[
0] # Prepending to match the hhUTC codes
return ({timeinfo: masked_values}, units)
while True:
# Handling the queued tuples and the termination command
request = inqueue.get()
if (request[0] == 'STOP'):
logging.debug('terminating this preprocess subprocess')
break
date, rawfilepath = request # A valid request contains a pd.DatetimeIndex and a Path
# Check file time content, and make sure that we are not requested to write an existing date
with nc.Dataset(self.datapath, mode='a') as ds:
times = ds[self.operation]['time'][:]
if times.size == 0: # Empty timeaxis
presentdates = []
lastdate = date - pd.Timedelta(
1,
'D') # Set because of possible consecitive criterium
else:
presentdates = nc.num2date(
times,
units=ds[self.operation]['time'].units,
calendar=ds[self.operation]['time'].calendar)
lastdate = pd.Timestamp(presentdates[-1])
writedate = date.to_pydatetime()
assert not (writedate in presentdates)
# Check whether the writedate does not follow the presentdates, given consecutive is set to True
# Because then the request should be given back until the consecutive turns up
if consecutive and (lastdate + pd.Timedelta(1, 'D') != date):
inqueue.put(request)
try: # Checking if we have one single cycling request, if not or if the comparison fails then log and set a new previous request
if request != oldreq:
logging.debug(
f'Placed back request for {date} as it does not follow the files last day {pd.Timestamp(presentdates[-1])}'
)
oldreq = request
except NameError:
oldreq = request
logging.debug(
f'Placed back request for {date} as it does not follow the files last day {pd.Timestamp(presentdates[-1])}'
)
else:
# Let ECcodes loop through as many messages as there are in the raw file of this date (maximum 24)
content = OrderedDict()
with open(rawfilepath, mode='rb') as f:
while True:
gid = ec.codes_grib_new_from_file(f)
if gid is None:
break
stepfield, units = extract_from_message(gid)
content.update(stepfield)
ec.codes_release(gid)
# Do the actual operations
if self.operation.endswith('UTC'):
try:
dayfield = content[self.operation[:2]]
except KeyError:
logging.error(
f'field to extract at: {self.operation} is not present in {rawfilepath}'
)
else:
if len(content) < 24:
logging.warning(
f'less than 24 hours present in {rawfilepath} for {self.operation}'
)
stacked_array = np.ma.stack(content.values(), axis=0)
operation_method = getattr(stacked_array, self.operation)
dayfield = operation_method(axis=0)
self.writer.append_one_day(writedate=writedate,
dayfield=dayfield,
index=len(presentdates),
units=units)
def read_field(self):
geography = [
"bitmapPresent", "Nx", "Ny", "latitudeOfFirstGridPointInDegrees",
"longitudeOfFirstGridPointInDegrees", "LoVInDegrees", "DxInMetres",
"DyInMetres", "iScansNegatively", "jScansPositively",
"jPointsAreConsecutive", "Latin1InDegrees", "LaDInDegrees",
"Latin2InDegrees", "latitudeOfSouthernPoleInDegrees",
"longitudeOfSouthernPoleInDegrees", "gridType"
]
if self.fname == None or not os.path.isfile(self.fname):
print("The file " + str(self.fname) + " does not exist!")
sys.exit(1)
print("Reading file: " + self.fname)
f = open(self.fname, "r")
while 1:
gid = ec.codes_grib_new_from_file(f)
if gid is None:
break
par = ec.codes_get(gid, "indicatorOfParameter")
lev = ec.codes_get(gid, "level")
typ = ec.codes_get(gid, "indicatorOfTypeOfLevel")
tri = ec.codes_get(gid, "timeRangeIndicator")
#print("Search::", w_par, w_lev, w_typ, w_tri)
if self.par == par and self.lev == lev and self.typ == typ and self.tri == tri:
print("Found:", self.par, self.lev, self.typ, self.tri)
geo = {}
for key in geography:
try:
geo.update({key: ec.codes_get(gid, key)})
except ec.CodesInternalError as err:
print('Error with key="%s" : %s' % (key, err.msg))
print(
'There are %d values, average is %f, min is %f, max is %f'
% (ec.codes_get_size(gid, 'values'),
ec.codes_get(gid, 'average'), ec.codes_get(
gid, 'min'), ec.codes_get(gid, 'max')))
# Date/time
d = ec.codes_get(gid, "validityDate")
t = ec.codes_get(gid, "validityTime")
h = int(t) / 100
m = t % h
s = (h * 3600) + (m * 60)
date = datetime.strptime(str(d), "%Y%m%d")
time = timedelta(seconds=s)
dt = date + time
# Missing values
mv = None
try:
mv = ec.codes_get(gid, "missingValue")
except:
print("Field does not contanin missing values")
if geo["gridType"].lower() == "lambert":
values = ec.codes_get_values(gid)
nx = geo["Nx"]
ny = geo["Ny"]
lonCenter = geo["LoVInDegrees"]
latCenter = geo["LaDInDegrees"]
latRef = geo["Latin2InDegrees"]
lon0 = geo["longitudeOfFirstGridPointInDegrees"]
lat0 = geo["latitudeOfFirstGridPointInDegrees"]
dx = geo["DxInMetres"]
dy = geo["DyInMetres"]
proj4_string = "+proj=lcc +lat_0=" + str(
latCenter) + " +lon_0=" + str(
lonCenter) + " +lat_1=" + str(
latRef) + " +lat_2=" + str(
latRef) + " +no_defs +units=m +R=6.371e+06"
proj4 = Proj(proj4_string)
x0, y0 = proj4(lon0, lat0)
x0 = int(round(x0))
y0 = int(round(y0))
field = np.empty([nx, ny])
lons = np.empty([nx, ny])
lats = np.empty([nx, ny])
X = np.arange(x0, x0 + (nx * dx), dx)
Y = np.arange(y0, y0 + (ny * dy), dy)
ii = 0
for i in range(0, nx):
for j in range(0, ny):
field[i, j] = values[ii]
lons[i, j], lats[i, j] = proj4(X[i],
Y[j],
inverse=True)
# print i,j,lons[i, j], lats[i, j]
ii = ii + 1
if mv is not None:
field[field == mv] = np.nan
ec.codes_release(gid)
f.close()
return (lons, lats, X, Y, dt, field)
else:
print(geo["gridType"] + " not implemented yet!")
ec.codes_release(gid)
f.close()
def load_grib(path):
f = open(path)
gid = codes_grib_new_from_file(f)
yield gid
codes_release(gid)
f.close()
def get_point_index(model, point):
# if known named point get grid point location #
if not 'lat' in point:
#print('pointname is known: {}'.format(point['name']))
point = which_grid_point(point['name'], model)
path = dict(base='/')
path['subdir'] = 'data/model_data/{}/grid/'.format(model)
if model == 'icon-eu-eps':
filename_clat = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clat.grib2'
filename_clon = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clon.grib2'
elif model == 'icon-global-eps':
filename_clat = 'icon-eps_global_icosahedral_time-invariant_2019010800_clat.grib2'
filename_clon = 'icon-eps_global_icosahedral_time-invariant_2019010800_clon.grib2'
elif model == 'icon-eu-det':
filename_clat = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2'
filename_clon = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2'
elif model == 'icon-global-det':
filename_clat = 'icon_global_icosahedral_time-invariant_2020020700_CLAT.grib2'
filename_clon = 'icon_global_icosahedral_time-invariant_2020020700_CLON.grib2'
# get clat and clon 1D arrays #
with open(path['base'] + path['subdir'] + filename_clat, 'rb') as file:
grib_id = eccodes.codes_grib_new_from_file(file)
clat = eccodes.codes_get_array(grib_id, 'values')
eccodes.codes_release(grib_id)
with open(path['base'] + path['subdir'] + filename_clon, 'rb') as file:
grib_id = eccodes.codes_grib_new_from_file(file)
clon = eccodes.codes_get_array(grib_id, 'values')
eccodes.codes_release(grib_id)
# read out index of nearest icosahedral point #
filter_distance = get_latlon_filter_distance(model)
lat_near = list(np.where(abs(clat - point['lat']) < filter_distance)[0])
lon_near = list(np.where(abs(clon - point['lon']) < filter_distance)[0])
id_near = list(set(lat_near).intersection(lon_near))
id_near.sort()
distances = np.sqrt( np.square(abs(clat[id_near] - point['lat']) * 111.2) \
+ np.square(abs(clon[id_near] - point['lon']) * 111.2 \
* np.cos(point['lat']*np.pi/180)) )
index_nearest = id_near[np.argmin(distances)]
#print(id_near)
#print(distances)
#print(index_nearest)
#print(clat[index_nearest], clon[index_nearest])
if model == 'icon-eu-det':
# this model data is on regular lat-lon-grid #
lat_index = int((clat[index_nearest] - 29.5) / 0.0625)
lon_index = int((clon[index_nearest] + 23.5) / 0.0625)
point_index = [lat_index, lon_index]
else:
# all other model data is on icosahedral grid with single index #
point_index = [index_nearest]
return point_index
]
filePrefix = 'rmf.hgra.2019032100'
fileSuffix = '.grb2'
filenames = []
filePaths = []
for i in range(nfiles):
filenames.append(filePrefix + '{:03d}'.format(i) + fileSuffix)
filePaths.append(os.path.join(inDir, filenames[i]))
f = open(filePaths[0], 'r')
print(filePaths[0])
eccodes.codes_grib_multi_support_on()
mcount = eccodes.codes_count_in_file(f)
print(mcount)
gids = [eccodes.codes_grib_new_from_file(f) for i in range(mcount)]
f.close()
varNames = []
levels = []
varParCat = []
varParNum = []
prodDefinTemNum = []
for i in range(mcount):
gid = gids[i]
varNames.append(eccodes.codes_get(gid, 'shortName'))
levels.append(eccodes.codes_get(gid, 'level'))
varParCat.append(eccodes.codes_get(gid, 'parameterCategory'))
varParNum.append(eccodes.codes_get(gid, 'parameterNumber'))
prodDefinTemNum.append(
eccodes.codes_get(gid, 'productDefinitionTemplateNumber'))
def __enter__(self):
self.fd = open(self.fname, "rb")
self.gid = eccodes.codes_grib_new_from_file(self.fd)
return self
