def get_dataset(self, dataset_id, dataset_info): """Get dataset using the parameter_number key in dataset_info. In a previous version of the reader, the attributes (nrows, ncols, ssp_lon) and projection information (pdict and area_dict) were computed while initializing the file handler. Also the code would break out from the While-loop below as soon as the correct parameter_number was found. This has now been revised becasue the reader would sometimes give corrupt information about the number of messages in the file and the dataset dimensions within a given message if the file was only partly read (not looping over all messages) in an earlier instance. """ logger.debug( 'Reading in file to get dataset with parameter number %d.', dataset_info['parameter_number']) xarr = None message_found = False with open(self.filename, 'rb') as fh: # Iterate over all messages and fetch data when the correct parameter number is found while True: gid = ec.codes_grib_new_from_file(fh) if gid is None: if not message_found: # Could not obtain a valid message ID from the grib file logger.warning( "Could not find parameter_number %d in GRIB file, no valid Dataset created", dataset_info['parameter_number']) break # Check if the parameter number in the GRIB message corresponds to the required key parameter_number = self._get_from_msg(gid, 'parameterNumber') if parameter_number == dataset_info['parameter_number']: self._res = dataset_id.resolution self._read_attributes(gid) # Read the missing value missing_value = self._get_from_msg(gid, 'missingValue') # Retrieve values and metadata from the GRIB message, masking the values equal to missing_value xarr = self._get_xarray_from_msg(gid) xarr.data = da.where(xarr.data == missing_value, np.nan, xarr.data) ec.codes_release(gid) # Combine all metadata into the dataset attributes and break out of the loop xarr.attrs.update(dataset_info) xarr.attrs.update(self._get_attributes()) message_found = True else: # The parameter number is not the correct one, release gid and skip to next message ec.codes_release(gid) return xarr
def cli(file_path): with open(file_path, 'rb') as f: handle = eccodes.codes_grib_new_from_file(f, headers_only=False) while handle is not None: date = eccodes.codes_get(handle, "dataDate") type_of_level = eccodes.codes_get(handle, "typeOfLevel") level = eccodes.codes_get(handle, "level") points = eccodes.codes_grib_find_nearest(handle, 39.92, 116.46, False, 1) point = points[0] print(date, type_of_level, level, " :", point.lat, point.lon, point.value, point.distance) eccodes.codes_release(handle) handle = eccodes.codes_grib_new_from_file(f, headers_only=False)
def get_ecc_gids(filename): f = open(filename) msg_count = ecc.codes_count_in_file(f) # TODO: this is only for grib files gid_list = [ecc.codes_grib_new_from_file(f) for i in range(msg_count)] #print(gid_list) f.close() return gid_list
def main(): #point = dict(lat = 49.014, lon = 8.404, name = 'Karlsruhe') point = dict(lat = 50.822, lon = 8.920, name = 'Kirchhain') #point = dict(lat = 65.661, lon =-18.718, name = 'Iceland') path = dict(base = '/lsdfos/kit/imk-tro/projects/MOD/Gruppe_Knippertz/nw5893/', data = 'forecast_archive/pamore/kirchhain/run_12Z/', grid = 'forecast_archive/icon-eu-eps/grid/', plots = 'plots/kirchhain/run_12Z/') data_rain_gsp_sum = np.empty((25, 40, 75948)) data_rain_con_sum = np.empty((25, 40, 75948)) hours = list(range(0, 12+1)) filenames_all = [] for hour in hours: filenames_all.append(['iefff0{:03d}0000.m0{:02d}'.format(hour, member) for member in range(1, 41)]) with ExitStack() as stack: files_all = [[stack.enter_context(open(path['base'] + path['data'] + filename,'rb'))\ for filename in filenames_of_one_hour] for filenames_of_one_hour in filenames_all] for i, files_of_one_hour in enumerate(files_all): for j, file in enumerate(files_of_one_hour): grib_id = eccodes.codes_grib_new_from_file(file) data_rain_gsp_sum[i - hours[0], j, :] = eccodes.codes_get_array(grib_id, 'values') grib_id = eccodes.codes_grib_new_from_file(file) data_rain_con_sum[i - hours[0], j, :] = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) data_rain_tot_sum = data_rain_gsp_sum + data_rain_con_sum del data_rain_gsp_sum, data_rain_con_sum, files_all print(data_rain_tot_sum.max(axis=1).max(axis=1)) print('data_rain_tot_sum: {:.0f}MB'.format(data_rain_tot_sum.nbytes / 1e6)) stat_processing = 'max' member = None plot_rain_around_point(path, data_rain_tot_sum, 0, 12, point, stat_processing, member) '''stat_processing = 'member_extract' for member in range(1,41): plot_rain_around_point(path, data_rain_tot_sum, 0, 24, point, stat_processing, member)''' return
def load_bytes_from_index(index: GribMessageIndex) -> typing.Optional[bytes]: with open(index.file_path, "rb") as f: f.seek(index.offset) message = eccodes.codes_grib_new_from_file(f) if message is None: return None raw_bytes = eccodes.codes_get_message(message) return raw_bytes
def cli(file_path): with open(file_path, 'rb') as f: handle = eccodes.codes_grib_new_from_file(f, headers_only=False) if handle is None: print("ERROR: unable to create handle from file " + file_path) sys.exit(-1) eccodes.codes_release(handle)
def keep_only_first_grib(fname: str): with open(fname, "r+b") as infd: gid = eccodes.codes_grib_new_from_file(infd) try: if eccodes.codes_get_message_offset(gid) != 0: raise RuntimeError( f"{fname}: first grib does not start at offset 0") infd.truncate(eccodes.codes_get_message_size(gid)) finally: eccodes.codes_release(gid)
def _index_grib_file(path, path_name=None): import eccodes with open(path, "rb") as f: h = eccodes.codes_grib_new_from_file(f) while h: try: field = dict() if isinstance(path_name, str): field["_path"] = path_name elif path_name is False: pass elif path_name is None: field["_path"] = path else: raise ValueError( f"Value of path_name cannot be '{path_name}.'") i = eccodes.codes_keys_iterator_new(h, "mars") try: while eccodes.codes_keys_iterator_next(i): name = eccodes.codes_keys_iterator_get_name(i) value = eccodes.codes_get_string(h, name) field[name] = value finally: eccodes.codes_keys_iterator_delete(i) field["_offset"] = eccodes.codes_get_long(h, "offset") field["_length"] = eccodes.codes_get_long(h, "totalLength") field["_param_id"] = eccodes.codes_get_string(h, "paramId") field["param"] = eccodes.codes_get_string(h, "shortName") yield field finally: eccodes.codes_release(h) h = eccodes.codes_grib_new_from_file(f)
def from_file(cls, file, offset=None, **kwargs): # type: (T.IO[bytes], T.Union[int, T.Tuple[int ,int], None], T.Any) -> Message field_in_message = 0 if isinstance(offset, tuple): offset, field_in_message = offset if offset is not None: file.seek(offset) codes_id = None if field_in_message == 0: codes_id = eccodes.codes_grib_new_from_file(file) else: # MULTI-FIELD is enabled only when accessing additional fields with multi_enabled(file): for _ in range(field_in_message + 1): codes_id = eccodes.codes_grib_new_from_file(file) if codes_id is None: raise EOFError("End of file: %r" % file) return cls(codes_id=codes_id, **kwargs)
def _read_global_attributes(self): """Read the global product attributes from the first message. Read the information about the date and time of the data product, the projection and area definition and the number of messages. """ with open(self.filename, 'rb') as fh: gid = ec.codes_grib_new_from_file(fh) if gid is None: # Could not obtain a valid message id: set attributes to None, number of messages to 0 logger.warning( "Could not obtain a valid message id in GRIB file") self._ssp_lon = None self._nrows = None self._ncols = None self._pdict, self._area_dict = None, None return # Read SSP and date/time self._ssp_lon = self._get_from_msg( gid, 'longitudeOfSubSatellitePointInDegrees') # Read number of points on the x and y axes self._nrows = self._get_from_msg(gid, 'Ny') self._ncols = self._get_from_msg(gid, 'Nx') # Creates the projection and area dictionaries self._pdict, self._area_dict = self._get_proj_area(gid) # Determine the number of messages in the product by iterating until an invalid id is obtained i = 1 ec.codes_release(gid) while True: gid = ec.codes_grib_new_from_file(fh) if gid is None: break ec.codes_release(gid) i = i + 1
def scan_for_gid(filename, short_name): filee = open(filename, "rb") for j in range(ec.codes_count_in_file(filee)): gid = ec.codes_grib_new_from_file(filee, headers_only=True) if ec.codes_get(gid, "shortName") == short_name: filee.close() return gid else: ec.codes_release(gid) filee.close() exit(1)
def scan_for_gid(filename, short_name, time_since_init, level): filee = open(filename, "rb") for j in np.arange(0, ec.codes_count_in_file(filee)): gid = ec.codes_grib_new_from_file(filee, headers_only = True) if ec.codes_get(gid, "shortName") == short_name and ec.codes_get(gid, "forecastTime") == time_since_init and ec.codes_get(gid, "level") == level: filee.close() return gid else: ec.codes_release(gid) filee.close() exit(1)
def _load_message_from_file_by_count(file_path, count): current_index = 0 with open(file_path, "rb") as f: while True: message_id = eccodes.codes_grib_new_from_file(f) if message_id is None: return None current_index += 1 if current_index == count: return message_id else: eccodes.codes_release(message_id)
def confirm_packing_type(gribfile, packing_type): """Confirm that gribfile contains only GRIBs with specified packingType.""" comparisons = [] with open(gribfile) as infile: while True: gid = codes_grib_new_from_file(infile) if gid is None: break encoded_type = codes_get(gid, "packingType") codes_release(gid) comparisons.append(encoded_type == packing_type) return comparisons
def load_bytes_from_file( file_path: str or Path, parameter: str or typing.Dict, level_type: str = None, level: int = None, ) -> bytes or None: """ Load one message from grib file and return message's original bytes. Parameters ---------- file_path parameter level_type level Returns ------- bytes or None Examples -------- Load bytes of 850hPa temperature from GRAPES GFS GMF and create GRIB message using `eccodes.codes_new_from_message`. >>> file_path = "/sstorage1/COMMONDATA/OPER/NWPC/GRAPES_GFS_GMF/Prod-grib/2020031721/ORIG/gmf.gra.2020031800105.grb2" >>> message_bytes = load_bytes_from_file( ... file_path=file_path, ... parameter="t", ... level_type="pl", ... level=850, ... ) >>> message = eccodes.codes_new_from_message(message_bytes) >>> values = eccodes.codes_get_double_array(message, "values") >>> print(len(values)) 1036800 """ offset = 0 fixed_level_type = fix_level_type(level_type) with open(file_path, "rb") as f: while True: message_id = eccodes.codes_grib_new_from_file(f) length = eccodes.codes_get(message_id, "totalLength") if message_id is None: return None if not _check_message(message_id, parameter, fixed_level_type, level): eccodes.codes_release(message_id) offset += length continue return eccodes.codes_get_message(message_id)
def from_file(cls, file, offset=None, **kwargs): # type: (T.IO[bytes], int, T.Any) -> Message field_in_message = 0 if isinstance(offset, tuple): offset, field_in_message = offset if offset is not None: file.seek(offset) codes_id = None # iterate over multi-fields in the message for _ in range(field_in_message + 1): codes_id = eccodes.codes_grib_new_from_file(file) if codes_id is None: raise EOFError("End of file: %r" % file) return cls(codes_id=codes_id, **kwargs)
def gribs_match(left, right): """Check if GRIBs in both input files store the same data.""" comparisons = [] with open(left) as a, open(right) as b: while True: a_gid = codes_grib_new_from_file(a) if a_gid is None: break b_gid = codes_grib_new_from_file(b) if b_gid is None: comparisons.append(False) info("GRIBs contain unequal number of messages.") continue packing_errors = [0] try: packing_errors.append(codes_get(a_gid, "packingError")) packing_errors.append(codes_get(b_gid, "packingError")) except CodesInternalError: pass tolerance = max(packing_errors) a_values = codes_get_values(a_gid) b_values = codes_get_values(b_gid) comparisons.append(np.allclose(a_values, b_values, atol=tolerance)) return comparisons
def get_grid_from_gribfile(filename, rotated=False): f = open(filename) # get grib message count and create gid_list, close filehandle msg_count = codes.codes_count_in_file(f) gid_list = [codes.codes_grib_new_from_file(f) for i in range(msg_count)] f.close() print("Working on grib-file: {0}".format(filename)) print("Message Count: {0}".format(msg_count)) # read grib grid details from given gid gid = gid_list[0] return get_grid_from_gid(gid, rotated=rotated)
def fetch_model_output(input_file, time_since_init, short_name, level): file = open(input_file, "rb") gid = ec.codes_grib_new_from_file(file) file.close() values = read_grib_array(input_file, short_name, time_since_init, level) lat = np.deg2rad(ec.codes_get_array(gid, "latitudes")) lon = np.deg2rad(ec.codes_get_array(gid, "longitudes")) no_of_columns = ec.codes_get_long(gid, "Ni") no_of_lines = ec.codes_get_long(gid, "Nj") ec.codes_release(gid) lat_vector = np.zeros([no_of_lines]) lon_vector = np.zeros([no_of_columns]) for i in range(no_of_lines): lat_vector[i] = lat[i*no_of_columns] for i in range(no_of_columns): lon_vector[i] = lon[i] return lat_vector, lon_vector, vector_2_array(values, no_of_lines, no_of_columns)
def index_grib_files(grib_files): logging.info("Indexing grib files") index = {} index_keys = get_index_keys() cnt = 0 for grib_file in grib_files[0]: with open(grib_file) as fp: message_no = 0 offset = 0 while True: gid = ecc.codes_grib_new_from_file(fp) if gid is None: break ref = index for k in index_keys: try: val = ecc.codes_get_long(gid, k) except gribapi.errors.KeyValueNotFoundError as e: val = None if val not in ref: ref[val] = {} ref = ref[val] else: ref = ref[val] length = ecc.codes_get_long(gid, 'totalLength') ref['file_name'] = grib_file ref['message_no'] = message_no ref['length'] = length ref['offset'] = offset # print(ref) message_no += 1 offset += length cnt += 1 logging.info(f"Indexed {cnt} messages from {len(grib_files[0])} file(s)") return index
def cli(file_path): with open(file_path, 'rb') as f: handle = eccodes.codes_grib_new_from_file(f, headers_only=False) date = eccodes.codes_get(handle, "dataDate") type_of_level = eccodes.codes_get(handle, "typeOfLevel") level = eccodes.codes_get(handle, "level") iter_id = eccodes.codes_grib_iterator_new(handle, 0) while 1: result = eccodes.codes_grib_iterator_next(iter_id) if not result: break [lat, lon, value] = result print(lat, lon, value) eccodes.codes_grib_iterator_delete(iter_id) eccodes.codes_release(handle)
def get_dataset(self, dataset_id, dataset_info): """Get dataset using the parameter_number key in dataset_info.""" logger.debug( 'Reading in file to get dataset with parameter number %d.', dataset_info['parameter_number']) xarr = None with open(self.filename, 'rb') as fh: # Iterate until a message containing the correct parameter number is found while True: gid = ec.codes_grib_new_from_file(fh) if gid is None: # Could not obtain a valid message ID, break out of the loop logger.warning( "Could not find parameter_number %d in GRIB file, no valid Dataset created", dataset_info['parameter_number']) break # Check if the parameter number in the GRIB message corresponds to the required key parameter_number = self._get_from_msg(gid, 'parameterNumber') if parameter_number != dataset_info['parameter_number']: # The parameter number is not the correct one, skip to next message ec.codes_release(gid) continue # Read the missing value missing_value = self._get_from_msg(gid, 'missingValue') # Retrieve values and metadata from the GRIB message, masking the values equal to missing_value xarr = self._get_xarray_from_msg(gid) xarr.where(xarr.data == missing_value, np.NaN) ec.codes_release(gid) # Combine all metadata into the dataset attributes and break out of the loop xarr.attrs.update(dataset_info) xarr.attrs.update(self._get_global_attributes()) break return xarr
def repack(input_file, outfile, packing_type): """Repack infile with packing_type, write result to outfile.""" with open(input_file) as infile: i = 1 while True: in_gid = codes_grib_new_from_file(infile) if in_gid is None: break info("Repacking GRIB #{}".format(i)) payload = codes_get_values(in_gid) clone_id = codes_clone(in_gid) codes_set(clone_id, "packingType", packing_type) codes_set_values(clone_id, payload) if i == 1: mode = "w" else: mode = "a" with open(outfile, mode) as output: codes_write(clone_id, output) codes_release(clone_id) codes_release(in_gid) i += 1 if not confirm_packing_type(outfile, packing_type): raise EncodingError("Reencoding silently failed.")
def plot_contourmap(path, date, fcst_hours, variable, stat_processing_methods): ##### generate subpath and filename ##### subpath = 'run_{}{:02}{:02}{:02}/{}/'.format(\ date['year'], date['month'], date['day'], date['hour'], variable) filename1 = 'icon-eu-eps_europe_icosahedral_single-level_{}{:02}{:02}{:02}_{:03}_{}.grib2'.format(\ date['year'], date['month'], date['day'], date['hour'], fcst_hours[0], variable) filename2 = 'icon-eu-eps_europe_icosahedral_single-level_{}{:02}{:02}{:02}_{:03}_{}.grib2'.format(\ date['year'], date['month'], date['day'], date['hour'], fcst_hours[1], variable) ######################################################################## ### read data ### ######################################################################## ##### create empty numpy arrays ##### ##### 2 fcst_hours, 40 members, 75948 eu gridpoints ##### data_raw = np.empty((2, 40, 75948)) data_members = np.empty((40, 75948)) ##### every time in loop open next grib msg from grib file ##### ##### grib messages in dwd file are sorted by increasing member number ##### with open(path['base'] + path['data'] + subpath + filename1, 'rb') as file: for member in range(1, 41): print('read data from member {}'.format(member)) grib_msg_id = eccodes.codes_grib_new_from_file(file) data_raw[0, member - 1, :] = eccodes.codes_get_array( grib_msg_id, 'values') eccodes.codes_release(grib_msg_id) with open(path['base'] + path['data'] + subpath + filename2, 'rb') as file: for member in range(1, 41): print('read data from member {}'.format(member)) grib_msg_id = eccodes.codes_grib_new_from_file(file) data_raw[1, member - 1, :] = eccodes.codes_get_array( grib_msg_id, 'values') eccodes.codes_release(grib_msg_id) ##### take the difference of the two accumulated total precipitation arrays ##### data_members = data_raw[1, :, :] - data_raw[0, :, :] del data_raw ##### open icon-eps grid file ##### icongrid_file = nc.Dataset( path['base'] + path['grid'] + 'icon_grid_0028_R02B07_N02.nc', 'r') vlat = icongrid_file.variables['clat_vertices'][:].data * 180. / np.pi vlon = icongrid_file.variables['clon_vertices'][:].data * 180. / np.pi clat = icongrid_file.variables['clat'][:].data * 180. / np.pi clon = icongrid_file.variables['clon'][:].data * 180. / np.pi icongrid_file.close() for stat_processing in stat_processing_methods: ######################################################################## ### statistically process data ### ######################################################################## if stat_processing == 'mean': data_processed = data_members.mean(axis=0) elif stat_processing == 'max': data_processed = data_members.max(axis=0) elif stat_processing == 'min': data_processed = data_members.min(axis=0) elif stat_processing == '90p': data_processed = np.percentile(data_members, 90, axis=0) elif stat_processing == '75p': data_processed = np.percentile(data_members, 75, axis=0) elif stat_processing == '50p': data_processed = np.percentile(data_members, 50, axis=0) elif stat_processing == '25p': data_processed = np.percentile(data_members, 25, axis=0) elif stat_processing == '10p': data_processed = np.percentile(data_members, 10, axis=0) #print('shape of data_members: {}'.format(np.shape(data_members))) #print('shape of data_processed: {}'.format(np.shape(data_processed))) ######################################################################## ### plot data on world map ### ######################################################################## ##### set domain due to center point and radius ##### center_point = dict(lat=48.5, lon=9.0) radius = 1800 # domain radius in km around center_point domain = dict( lat_min=center_point['lat'] - radius / 111.2, lat_max=center_point['lat'] + radius / 111.2, lon_min=center_point['lon'] - radius / (111.2 * np.cos(center_point['lat'] * np.pi / 180)), lon_max=center_point['lon'] + radius / (111.2 * np.cos(center_point['lat'] * np.pi / 180)), ) ##### or set domain manually in deg N/E ##### '''domain = dict( lat_min = 0.0, lat_max = 20.0, lon_min = 0.0, lon_max = 20.0, )''' ##### set image size (should be squared) ##### ##### plotting area in pyngl can not exceed squared area even if plotting on rectangular images ##### ##### for obtaining rectangular plots on has to cut manually afterwards e.g. with pillow package ##### x_resolution = 800 y_resolution = 800 wks_res = Ngl.Resources() wks_res.wkWidth = x_resolution wks_res.wkHeight = y_resolution plot_name = 'contourplot_icon-eu-eps_tot_prec_{:02d}-{:02d}h_{}'.format(\ fcst_hours[0], fcst_hours[1], stat_processing) wks_type = 'png' wks = Ngl.open_wks(wks_type, path['base'] + path['plots'] + plot_name, wks_res) resources = Ngl.Resources( ) # create resources object containing all the plot settings resources.mpProjection = 'Hammer' # projection type resources.mpCenterLonF = (domain['lon_max'] + domain['lon_min'] ) / 2 # projection center point resources.mpCenterLatF = (domain['lat_max'] + domain['lat_min']) / 2 resources.mpLimitMode = 'latlon' resources.mpMinLonF = domain['lon_min'] resources.mpMaxLonF = domain['lon_max'] resources.mpMinLatF = domain['lat_min'] resources.mpMaxLatF = domain['lat_max'] ##### set plot area ##### resources.nglMaximize = False resources.vpXF = 0.05 resources.vpYF = 0.9 resources.vpWidthF = 0.7 resources.vpHeightF = 0.7 ##### set all map plot settings ##### resources.mpFillOn = True # turn on filled map areas resources.mpFillColors = [ 'pink', 'blue', 'white', 'white' ] # set colors for [FillValue, Ocean, Land , InlandWater] resources.mpDataBaseVersion = 'MediumRes' # quality of national borders resources.mpDataSetName = 'Earth..4' resources.mpOutlineBoundarySets = 'national' #resources.mpDataBaseVersion = 'HighRes' #resources.mpDataResolution = 'Fine' resources.mpGeophysicalLineThicknessF = 7.0 * x_resolution / 1000 # keep borders thickness resolution-independent resources.mpNationalLineThicknessF = 7.0 * x_resolution / 1000 #resources.mpGridAndLimbDrawOrder = 'postdraw' resources.mpGridAndLimbOn = False # turn off geographic coordinates grid #resources.mpLimbLineColor = 'black' #resources.mpLimbLineThicknessF = 10 #resources.mpGridLineColor = 'black' #resources.mpGridLineThicknessF = 1.0 #resources.mpGridSpacingF = 1 resources.mpPerimOn = True # turn on perimeter around plot resources.mpPerimLineColor = 'black' resources.mpPerimLineThicknessF = 8.0 * x_resolution / 1000 # keep perimeter thickness resolution-independent resources.tmXBOn = False # turn off location ticks around plot resources.tmXTOn = False resources.tmYLOn = False resources.tmYROn = False resources.sfDataArray = data_processed # data input file to plot resources.sfXArray = clon # array with cell center locations resources.sfYArray = clat resources.sfXCellBounds = vlon # array with cell vertices locations resources.sfYCellBounds = vlat resources.sfMissingValueV = 9999 # in case you want to mask values resources.cnFillOn = True resources.cnFillMode = 'CellFill' #resources.cnCellFillEdgeColor = 'black' # uncomment this for plotting the cell edges resources.cnMissingValFillColor = 'black' resources.cnFillPalette = 'WhiteBlueGreenYellowRed' # color palette resources.cnLevelSelectionMode = 'ManualLevels' minlevel = 0.0 # min level of colorbar maxlevel = 50.0 # max level of colorbar numberoflevels = 250 # number of levels of colorbar, max. 250 with this color palette resources.cnMinLevelValF = minlevel resources.cnMaxLevelValF = maxlevel resources.cnLevelSpacingF = (maxlevel - minlevel) / numberoflevels resources.cnLinesOn = False # turn off contour lines resources.cnLineLabelsOn = False # turn off contour labels ##### set resources for a nice colorbar ##### resources.lbLabelBarOn = True resources.lbAutoManage = False resources.lbOrientation = 'vertical' resources.lbLabelOffsetF = 0.05 #resources.lbBoxMinorExtentF = 0.2 resources.lbLabelStride = 25 # print a tick every 25 levels resources.lbLabelFontHeightF = 0.016 resources.lbBoxSeparatorLinesOn = False resources.lbBoxLineThicknessF = 4.0 #resources.lbBoxEndCapStyle = 'TriangleBothEnds' resources.lbLabelAlignment = 'BoxCenters' resources.lbTitleString = 'mm' resources.lbTitleFontHeightF = 0.016 resources.lbTitlePosition = 'Right' resources.lbTitleDirection = 'Across' resources.lbTitleAngleF = 90.0 resources.lbTitleExtentF = 0.1 resources.lbTitleOffsetF = -0.15 resources.nglFrame = False # hold on frame because will plot text afterwards on same plot Ngl.contour_map(wks, data_processed, resources) # plot the actual plot ##### plot title text ##### text = '{:02d}-{:02d}h Total Precipitation {}, ICON-EPS run {:02}.{:02}.{} {:02}Z'.format(\ fcst_hours[0], fcst_hours[1], stat_processing,\ date['day'], date['month'], date['year'], date['hour']) x = 0.5 y = 0.95 text_res_1 = Ngl.Resources() text_res_1.txFontHeightF = 0.018 text_res_1.txJust = 'TopCenter' Ngl.text_ndc(wks, text, x, y, text_res_1) Ngl.frame(wks) # advance frame Ngl.destroy( wks ) # delete workspace to free memory, relevant if plotting various plots in one script print('plotted "{}.png"'.format(plot_name)) return
def load_message_from_file( file_path: str or Path, parameter: str or typing.Dict or None = None, level_type: str or typing.Dict or None = None, level: int or float or None = None, **kwargs, ) -> int or None: """ Load the **first** message from GRIB 2 file using eccodes-python library. Returned message is a copied one of original message and file is closed before return. And the returned message should be released by user using `eccodes.codes_release()`. Parameters ---------- file_path: str or Path GRIB 2 file path. parameter: str or typing.Dict short name of the field or a dictionary including some GRIB keys: - discipline - parameterCategory - parameterNumber level_type: str or typing.Dict level type. level: int or float level value. kwargs: dict ignored Returns ------- int or None GRIB handler (int) if found or None if not found. Examples -------- Load 850hPa temperature from GRAPES GFS and get values from GRIB message. >>> t = load_message_from_file( ... file_path="/g1/COMMONDATA/OPER/NWPC/GRAPES_GFS_GMF/Prod-grib/2020031721/ORIG/gmf.gra.2020031800105.grb2", ... parameter="t", ... level_type="isobaricInhPa", ... level=850, ... ) >>> data = eccodes.codes_get_double_array(t, "values") >>> data = data.reshape([720, 1440]) >>> data array([[249.19234375, 249.16234375, 249.16234375, ..., 249.15234375, 249.19234375, 249.14234375], [249.45234375, 249.45234375, 249.42234375, ..., 249.45234375, 249.44234375, 249.44234375], [249.69234375, 249.68234375, 249.68234375, ..., 249.70234375, 249.67234375, 249.68234375], ..., [235.33234375, 235.45234375, 235.62234375, ..., 235.47234375, 235.63234375, 235.48234375], [235.78234375, 235.91234375, 235.64234375, ..., 235.80234375, 235.72234375, 235.82234375], [235.66234375, 235.86234375, 235.82234375, ..., 235.85234375, 235.68234375, 235.70234375]]) """ fixed_level_type, _ = _fix_level(level_type, None) with open(file_path, "rb") as f: while True: message_id = eccodes.codes_grib_new_from_file(f) if message_id is None: return None if not _check_message(message_id, parameter, fixed_level_type, level): eccodes.codes_release(message_id) continue # clone message new_message_id = eccodes.codes_clone(message_id) eccodes.codes_release(message_id) return new_message_id return None
def load_messages_from_file( file_path: str or Path, parameter: str or typing.Dict, level_type: str or typing.Dict or typing.List or None = None, level: int or float or typing.List or None = None, **kwargs, ) -> typing.List or None: """ Load multiply messages from file. This function will scan all messages in GRIB 2 file and return all messages which fit conditions. Parameters ---------- file_path: str or Path parameter: str or typing.Dict see ``load_message_from_file``, required option. level_type: str or typing.List or None level type. - string, same as ``load_message_from_file`` - typing.List, level type should be in the list. - None, don't check level type. level: int or float or typing.Dict or typing.List or None level value. - string, same as ``load_message_from_file`` - typing.Dict, same as ``load_message_from_file`` - typing.List, level value should be in the list. - None, don't check level value. For example, load all messages of some typeOfLevel. kwargs: dict other parameters Returns ------- typing.List or None: a list of message number or None if no message is found. """ fixed_level_type, _ = _fix_level(level_type, None) messages = [] # print("count...") # with open(file_path, "rb") as f: # total_count = eccodes.codes_count_in_file(f) # print(total_count) # print("count..done") with open(file_path, "rb") as f: # pbar = tqdm(total=total_count) while True: message_id = eccodes.codes_grib_new_from_file(f) if message_id is None: break # pbar.update(1) if not _check_parameter(message_id, parameter): eccodes.codes_release(message_id) continue if not _check_level_type(message_id, fixed_level_type): eccodes.codes_release(message_id) continue if not _check_level_value(message_id, level): eccodes.codes_release(message_id) continue # clone message new_message_id = eccodes.codes_clone(message_id) eccodes.codes_release(message_id) messages.append(new_message_id) # pbar.close() if len(messages) == 0: return None return messages
def load_field_from_file( file_path: str or Path, parameter: str or typing.Dict or None = None, level_type: str or typing.Dict or None = None, level: int or float or typing.List or None = None, level_dim: str or None = None, show_progress: bool = False, ) -> xr.DataArray or None: """ Load **one** field from local GRIB2 file using eccodes-python. Or load multi levels into one field. Parameters ---------- file_path: str or Path parameter: str or typing.Dict parameter name. Use GRIB key `shortName` or a dict of filter conditions such as: { "discipline": 0, "parameterCategory": 2, "parameterNumber": 225, } level_type: str or typing.Dict or None level type. - Use "pl", "ml" or "sfc". They will be converted into dict. - Use GRIB key `typeOfLevel`, such as - "isobaricInhPa" - "isobaricInPa" - "surface" - "heightAboveGround" - ... See https://apps.ecmwf.int/codes/grib/format/edition-independent/3/ for more values. - If `typeOfLevel` is not available, use dict to specify filter conditions. For example, to get one filed from GRAPES GFS modelvar GRIB2 file, use: { "typeOfFirstFixedSurface": 131 } level: int or float or typing.List or None level value(s). - If use a scalar, level will be a non-dimension coordinate. - If your want to extract multi levels, use a list and level will be a dimension (level, lat, lon). - If use `None`, all levels of level_type will be packed in the result field. level_dim: str or None name of level dimension. If none, function will generate a name for level dim. If `level_type="pl"`, some values can be used: - `None` or `pl` or `isobaricInhPa`: level_dim is a float number with unit hPa. - `isobaricInPa`: level_dim is a float number with unit Pa. show_progress: bool show progress bar. Returns ------- DataArray or None: DataArray if found, or None if not. Examples -------- Load 850hPa temperature field from a GRIB2 file generated by GRAPES GFS. >>> load_field_from_file( ... file_path="/sstorage1/COMMONDATA/OPER/NWPC/GRAPES_GFS_GMF/Prod-grib/2020031721/ORIG/gmf.gra.2020031800105.grb2", ... parameter="t", ... level_type="isobaricInhPa", ... level=850, ... ) <xarray.DataArray 't' (latitude: 720, longitude: 1440)> array([[249.19234375, 249.16234375, 249.16234375, ..., 249.15234375, 249.19234375, 249.14234375], [249.45234375, 249.45234375, 249.42234375, ..., 249.45234375, 249.44234375, 249.44234375], [249.69234375, 249.68234375, 249.68234375, ..., 249.70234375, 249.67234375, 249.68234375], ..., [235.33234375, 235.45234375, 235.62234375, ..., 235.47234375, 235.63234375, 235.48234375], [235.78234375, 235.91234375, 235.64234375, ..., 235.80234375, 235.72234375, 235.82234375], [235.66234375, 235.86234375, 235.82234375, ..., 235.85234375, 235.68234375, 235.70234375]]) Coordinates: time datetime64[ns] 2020-03-18 step timedelta64[ns] 4 days 09:00:00 valid_time datetime64[ns] 2020-03-22T09:00:00 isobaricInhPa int64 850 * latitude (latitude) float64 89.88 89.62 89.38 ... -89.38 -89.62 -89.88 * longitude (longitude) float64 0.0 0.25 0.5 0.75 ... 359.2 359.5 359.8 Attributes: GRIB_edition: 2 GRIB_centre: babj GRIB_subCentre: 0 GRIB_tablesVersion: 4 GRIB_localTablesVersion: 1 GRIB_dataType: fc GRIB_dataDate: 20200318 GRIB_dataTime: 0 GRIB_validityDate: 20200322 GRIB_validityTime: 900 GRIB_step: 105 GRIB_stepType: instant GRIB_stepUnits: 1 GRIB_stepRange: 105 GRIB_endStep: 105 GRIB_name: Temperature GRIB_shortName: t GRIB_cfName: air_temperature GRIB_discipline: 0 GRIB_parameterCategory: 0 GRIB_parameterNumber: 0 GRIB_gridType: regular_ll GRIB_gridDefinitionDescription: Latitude/longitude GRIB_typeOfFirstFixedSurface: pl GRIB_typeOfLevel: isobaricInhPa GRIB_level: 850 GRIB_numberOfPoints: 1036800 GRIB_missingValue: 9999 GRIB_units: K long_name: Temperature units: K """ messages = [] fixed_level_type, fixed_level_dim = _fix_level(level_type, level_dim) if show_progress: with open(file_path, "rb") as f: total_count = eccodes.codes_count_in_file(f) with open(file_path, "rb") as f: if show_progress: pbar = tqdm( total=total_count, desc="Filtering", ) while True: message_id = eccodes.codes_grib_new_from_file(f) if message_id is None: break if show_progress: pbar.update(1) if not _check_message(message_id, parameter, fixed_level_type, level): eccodes.codes_release(message_id) continue messages.append(message_id) if isinstance(level, typing.List) or level is None: continue else: break if show_progress: pbar.close() if len(messages) == 0: return None if len(messages) == 1: message_id = messages[0] data = create_data_array_from_message(message_id, level_dim_name=fixed_level_dim) eccodes.codes_release(message_id) return data if len(messages) > 1: if show_progress: pbar = tqdm( total=len(messages), desc="Decoding", ) def creat_array(message): array = create_data_array_from_message(message, level_dim_name=fixed_level_dim) if show_progress: pbar.update(1) return array xarray_messages = [creat_array(message) for message in messages] for m in messages: eccodes.codes_release(m) if show_progress: pbar.close() if level_dim is None: if isinstance(level_type, str): level_dim_name = level_type elif isinstance(level_type, typing.Dict): level_dim_name = get_level_coordinate_name(xarray_messages[0]) else: raise ValueError(f"level_type is not supported: {level_type}") elif isinstance(level_dim, str): level_dim_name = level_dim else: raise ValueError(f"level_type is not supported: {level_type}") if show_progress: print("Packing...") data = xr.concat(xarray_messages, level_dim_name) return data return None
def read_grid_coordinates(model, grid): path = dict(base='/') path['grid'] = 'data/model_data/{}/grid/'.format(model) if model == 'icon-eu-eps': if grid == 'icosahedral': filename = 'icon_grid_0028_R02B07_N02.nc' elif grid == 'latlon_0.2': filename = 'icon-eu-eps_latlon_0.2_grid_coordinates.nc' elif model == 'icon-global-eps': filename = 'icon_grid_0024_R02B06_G.nc' elif model == 'icon-eu-det': filename_clat = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2' filename_clon = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2' elif model == 'icon-global-det': if grid == 'icosahedral': filename = 'icon_grid_0026_R03B07_G.nc' elif grid == 'latlon_0.25': filename = 'icon-global-det_latlon_0.25_grid_coordinates.nc' elif grid == 'latlon_0.1': filename = 'icon-global-det_latlon_0.1_grid_coordinates.nc' if grid == 'icosahedral': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['clat'].values * 180 / np.pi clon = ds['clon'].values * 180 / np.pi vlat = ds['clat_vertices'].values * 180 / np.pi vlon = ds['clon_vertices'].values * 180 / np.pi return clat, clon, vlat, vlon elif grid == 'latlon_0.0625': with open(path['base'] + path['grid'] + filename_clat, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clat = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) with open(path['base'] + path['grid'] + filename_clon, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clon = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) return clat.reshape((657, 1097)), clon.reshape((657, 1097)) elif grid == 'latlon_0.2': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['lat'].values clon = ds['lon'].values return clat, clon elif grid == 'latlon_0.25': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['lat'].values clon = ds['lon'].values clon = np.where(clon > 180, clon - 360, clon) clon_new = np.empty_like(clon) clon_new[719:] = clon[:721] clon_new[:719] = clon[721:] return clat, clon_new elif grid == 'latlon_0.1': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['lat'].values clon = ds['lon'].values clon = np.where(clon > 180, clon - 360, clon) clon_new = np.empty_like(clon) clon_new[1799:] = clon[:1801] clon_new[:1799] = clon[1801:] return clat, clon_new
def field(self, gribvar, time): if eccodes is None: raise Exception("eccodes not found. Needed for reading grib files") """ """ geography = ["bitmapPresent", "Nx", "Ny", "latitudeOfFirstGridPointInDegrees", "longitudeOfFirstGridPointInDegrees", "LoVInDegrees", "DxInMetres", "DyInMetres", "iScansNegatively", "jScansPositively", "jPointsAreConsecutive", "Latin1InDegrees", "LaDInDegrees", "Latin2InDegrees", "latitudeOfSouthernPoleInDegrees", "longitudeOfSouthernPoleInDegrees", "gridType" ] geo_out = None fh = open(self.fname) while 1: gid = eccodes.codes_grib_new_from_file(fh) if gid is None: print("\nCould not find key") gribvar.print_keys() fh.close() return None else: # print("\n Next key") # print_grib_id(gid) if gribvar.matches(gid): # print("Found key") # gribvar.print_keys() geo = {} for key in geography: try: geo.update({key: eccodes.codes_get(gid, key)}) except eccodes.CodesInternalError as err: print('Error with key="%s" : %s' % (key, err.msg)) # print('There are %d values, average is %f, min is %f, max is %f' % ( # codes_get_size(gid, 'values'), # codes_get(gid, 'average'), # codes_get(gid, 'min'), # codes_get(gid, 'max') # )) if geo["gridType"].lower() == "lambert": values = eccodes.codes_get_values(gid) print(values) nx = geo["Nx"] ny = geo["Ny"] lon0 = geo["LoVInDegrees"] lat0 = geo["LaDInDegrees"] ll_lon = geo["longitudeOfFirstGridPointInDegrees"] ll_lat = geo["latitudeOfFirstGridPointInDegrees"] dx = geo["DxInMetres"] dy = geo["DyInMetres"] # TODO Check time consistency print("Hopefullly valid for time ", time) earth = 6.37122e+6 proj4 = "+proj=lcc +lat_0=" + str(lat0) + " +lon_0=" + str(lon0) + " +lat_1=" + \ str(lat0) + " +lat_2=" + str(lat0) + " +units=m +no_defs +R=" + str(earth) proj = Proj(proj4) x0, y0 = proj(ll_lon, ll_lat) xc = x0 + 0.5 * (nx - 1) * dx yc = y0 + 0.5 * (ny - 1) * dy lonc, latc = proj(xc, yc, inverse=True) field = np.reshape(values, [nx, ny], order="F") if geo_out is None: domain = { "nam_conf_proj": { "xlon0": lon0, "xlat0": lat0 }, "nam_conf_proj_grid": { "xloncen": lonc, "xlatcen": latc, "nimax": nx, "njmax": ny, "xdx": dx, "xdy": dy, "ilone": 0, "ilate": 0 } } geo_out = surfex.geo.ConfProj(domain) else: raise NotImplementedError(geo["gridType"] + " not implemented yet!") eccodes.codes_release(gid) fh.close() # print lons # print lats if geo_out is None: raise Exception("No geometry is found in file") return field, geo_out eccodes.codes_release(gid)
def preprocess(self, inqueue: mp.Queue, consecutive: bool) -> None: """ Will read a request with rawfilepath and date from the queue. If consecutive is set to true then it checks whether the requested date is consecutive to the content of the netcdf file and puts the request back if it is not. If set to False, the eventual netcdf time axis can have missing days and be non-monotonic. This function then decodes the raw grib file, extracts hourly fields, does the operation (even if just a simple hhUTC timeslice) And will write to the netcdf. """ def extract_from_message( messageid: int) -> Tuple[Dict[str, np.ma.MaskedArray], str]: """ Eccodes tools to extract the field and do a sanity check on the name or cfVarName Returns a dictionary of the field, with endStep key, and additionally the units string EndStep is important in case the self.operation wants a certain timestep extracted. """ name = ec.codes_get(messageid, 'name').lower().split(' ') datestamp = str(ec.codes_get(messageid, 'dataDate')) assert ('_'.join(name) == self.encoding.loc['variable']) or (ec.codes_get( messageid, 'cfVarName') == self.encoding.name) assert pd.Timestamp( year=int(datestamp[:4]), month=int(datestamp[4:6]), day=int(datestamp[6:])) == date # Date comes from one level up # Extract the gridded values, reshape and mask. values = ec.codes_get_values(messageid) # One dimensional array lat_fastest_changing = (ec.codes_get(messageid, 'jPointsAreConsecutive') == 1) values = values.reshape( (ec.codes_get(messageid, 'Nj'), ec.codes_get(messageid, 'Ni')), order='F' if lat_fastest_changing else 'C') # order C means last index fastest changing if ec.codes_get( messageid, 'latitudeOfFirstGridPointInDegrees') > ec.codes_get( messageid, 'latitudeOfLastGridPointInDegrees'): values = values[:: -1, :] # In my eventual netcdf storage I want the latitudes to increase with index masked_values = np.ma.MaskedArray(data=values, mask=(values == ec.codes_get( messageid, 'missingValue'))) units = ec.codes_get(messageid, 'units') timeinfo = str( ec.codes_get(messageid, self.encoding.loc['timevariable'])) if timeinfo[-2:] == '00': timeinfo = timeinfo[: -2] # Remove the trailing zeros of the minutes if len(timeinfo) == 1: timeinfo = '0' + timeinfo[ 0] # Prepending to match the hhUTC codes return ({timeinfo: masked_values}, units) while True: # Handling the queued tuples and the termination command request = inqueue.get() if (request[0] == 'STOP'): logging.debug('terminating this preprocess subprocess') break date, rawfilepath = request # A valid request contains a pd.DatetimeIndex and a Path # Check file time content, and make sure that we are not requested to write an existing date with nc.Dataset(self.datapath, mode='a') as ds: times = ds[self.operation]['time'][:] if times.size == 0: # Empty timeaxis presentdates = [] lastdate = date - pd.Timedelta( 1, 'D') # Set because of possible consecitive criterium else: presentdates = nc.num2date( times, units=ds[self.operation]['time'].units, calendar=ds[self.operation]['time'].calendar) lastdate = pd.Timestamp(presentdates[-1]) writedate = date.to_pydatetime() assert not (writedate in presentdates) # Check whether the writedate does not follow the presentdates, given consecutive is set to True # Because then the request should be given back until the consecutive turns up if consecutive and (lastdate + pd.Timedelta(1, 'D') != date): inqueue.put(request) try: # Checking if we have one single cycling request, if not or if the comparison fails then log and set a new previous request if request != oldreq: logging.debug( f'Placed back request for {date} as it does not follow the files last day {pd.Timestamp(presentdates[-1])}' ) oldreq = request except NameError: oldreq = request logging.debug( f'Placed back request for {date} as it does not follow the files last day {pd.Timestamp(presentdates[-1])}' ) else: # Let ECcodes loop through as many messages as there are in the raw file of this date (maximum 24) content = OrderedDict() with open(rawfilepath, mode='rb') as f: while True: gid = ec.codes_grib_new_from_file(f) if gid is None: break stepfield, units = extract_from_message(gid) content.update(stepfield) ec.codes_release(gid) # Do the actual operations if self.operation.endswith('UTC'): try: dayfield = content[self.operation[:2]] except KeyError: logging.error( f'field to extract at: {self.operation} is not present in {rawfilepath}' ) else: if len(content) < 24: logging.warning( f'less than 24 hours present in {rawfilepath} for {self.operation}' ) stacked_array = np.ma.stack(content.values(), axis=0) operation_method = getattr(stacked_array, self.operation) dayfield = operation_method(axis=0) self.writer.append_one_day(writedate=writedate, dayfield=dayfield, index=len(presentdates), units=units)
def read_field(self): geography = [ "bitmapPresent", "Nx", "Ny", "latitudeOfFirstGridPointInDegrees", "longitudeOfFirstGridPointInDegrees", "LoVInDegrees", "DxInMetres", "DyInMetres", "iScansNegatively", "jScansPositively", "jPointsAreConsecutive", "Latin1InDegrees", "LaDInDegrees", "Latin2InDegrees", "latitudeOfSouthernPoleInDegrees", "longitudeOfSouthernPoleInDegrees", "gridType" ] if self.fname == None or not os.path.isfile(self.fname): print("The file " + str(self.fname) + " does not exist!") sys.exit(1) print("Reading file: " + self.fname) f = open(self.fname, "r") while 1: gid = ec.codes_grib_new_from_file(f) if gid is None: break par = ec.codes_get(gid, "indicatorOfParameter") lev = ec.codes_get(gid, "level") typ = ec.codes_get(gid, "indicatorOfTypeOfLevel") tri = ec.codes_get(gid, "timeRangeIndicator") #print("Search::", w_par, w_lev, w_typ, w_tri) if self.par == par and self.lev == lev and self.typ == typ and self.tri == tri: print("Found:", self.par, self.lev, self.typ, self.tri) geo = {} for key in geography: try: geo.update({key: ec.codes_get(gid, key)}) except ec.CodesInternalError as err: print('Error with key="%s" : %s' % (key, err.msg)) print( 'There are %d values, average is %f, min is %f, max is %f' % (ec.codes_get_size(gid, 'values'), ec.codes_get(gid, 'average'), ec.codes_get( gid, 'min'), ec.codes_get(gid, 'max'))) # Date/time d = ec.codes_get(gid, "validityDate") t = ec.codes_get(gid, "validityTime") h = int(t) / 100 m = t % h s = (h * 3600) + (m * 60) date = datetime.strptime(str(d), "%Y%m%d") time = timedelta(seconds=s) dt = date + time # Missing values mv = None try: mv = ec.codes_get(gid, "missingValue") except: print("Field does not contanin missing values") if geo["gridType"].lower() == "lambert": values = ec.codes_get_values(gid) nx = geo["Nx"] ny = geo["Ny"] lonCenter = geo["LoVInDegrees"] latCenter = geo["LaDInDegrees"] latRef = geo["Latin2InDegrees"] lon0 = geo["longitudeOfFirstGridPointInDegrees"] lat0 = geo["latitudeOfFirstGridPointInDegrees"] dx = geo["DxInMetres"] dy = geo["DyInMetres"] proj4_string = "+proj=lcc +lat_0=" + str( latCenter) + " +lon_0=" + str( lonCenter) + " +lat_1=" + str( latRef) + " +lat_2=" + str( latRef) + " +no_defs +units=m +R=6.371e+06" proj4 = Proj(proj4_string) x0, y0 = proj4(lon0, lat0) x0 = int(round(x0)) y0 = int(round(y0)) field = np.empty([nx, ny]) lons = np.empty([nx, ny]) lats = np.empty([nx, ny]) X = np.arange(x0, x0 + (nx * dx), dx) Y = np.arange(y0, y0 + (ny * dy), dy) ii = 0 for i in range(0, nx): for j in range(0, ny): field[i, j] = values[ii] lons[i, j], lats[i, j] = proj4(X[i], Y[j], inverse=True) # print i,j,lons[i, j], lats[i, j] ii = ii + 1 if mv is not None: field[field == mv] = np.nan ec.codes_release(gid) f.close() return (lons, lats, X, Y, dt, field) else: print(geo["gridType"] + " not implemented yet!") ec.codes_release(gid) f.close()
def load_grib(path): f = open(path) gid = codes_grib_new_from_file(f) yield gid codes_release(gid) f.close()
def get_point_index(model, point): # if known named point get grid point location # if not 'lat' in point: #print('pointname is known: {}'.format(point['name'])) point = which_grid_point(point['name'], model) path = dict(base='/') path['subdir'] = 'data/model_data/{}/grid/'.format(model) if model == 'icon-eu-eps': filename_clat = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clat.grib2' filename_clon = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clon.grib2' elif model == 'icon-global-eps': filename_clat = 'icon-eps_global_icosahedral_time-invariant_2019010800_clat.grib2' filename_clon = 'icon-eps_global_icosahedral_time-invariant_2019010800_clon.grib2' elif model == 'icon-eu-det': filename_clat = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2' filename_clon = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2' elif model == 'icon-global-det': filename_clat = 'icon_global_icosahedral_time-invariant_2020020700_CLAT.grib2' filename_clon = 'icon_global_icosahedral_time-invariant_2020020700_CLON.grib2' # get clat and clon 1D arrays # with open(path['base'] + path['subdir'] + filename_clat, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clat = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) with open(path['base'] + path['subdir'] + filename_clon, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clon = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) # read out index of nearest icosahedral point # filter_distance = get_latlon_filter_distance(model) lat_near = list(np.where(abs(clat - point['lat']) < filter_distance)[0]) lon_near = list(np.where(abs(clon - point['lon']) < filter_distance)[0]) id_near = list(set(lat_near).intersection(lon_near)) id_near.sort() distances = np.sqrt( np.square(abs(clat[id_near] - point['lat']) * 111.2) \ + np.square(abs(clon[id_near] - point['lon']) * 111.2 \ * np.cos(point['lat']*np.pi/180)) ) index_nearest = id_near[np.argmin(distances)] #print(id_near) #print(distances) #print(index_nearest) #print(clat[index_nearest], clon[index_nearest]) if model == 'icon-eu-det': # this model data is on regular lat-lon-grid # lat_index = int((clat[index_nearest] - 29.5) / 0.0625) lon_index = int((clon[index_nearest] + 23.5) / 0.0625) point_index = [lat_index, lon_index] else: # all other model data is on icosahedral grid with single index # point_index = [index_nearest] return point_index
] filePrefix = 'rmf.hgra.2019032100' fileSuffix = '.grb2' filenames = [] filePaths = [] for i in range(nfiles): filenames.append(filePrefix + '{:03d}'.format(i) + fileSuffix) filePaths.append(os.path.join(inDir, filenames[i])) f = open(filePaths[0], 'r') print(filePaths[0]) eccodes.codes_grib_multi_support_on() mcount = eccodes.codes_count_in_file(f) print(mcount) gids = [eccodes.codes_grib_new_from_file(f) for i in range(mcount)] f.close() varNames = [] levels = [] varParCat = [] varParNum = [] prodDefinTemNum = [] for i in range(mcount): gid = gids[i] varNames.append(eccodes.codes_get(gid, 'shortName')) levels.append(eccodes.codes_get(gid, 'level')) varParCat.append(eccodes.codes_get(gid, 'parameterCategory')) varParNum.append(eccodes.codes_get(gid, 'parameterNumber')) prodDefinTemNum.append( eccodes.codes_get(gid, 'productDefinitionTemplateNumber'))
def __enter__(self): self.fd = open(self.fname, "rb") self.gid = eccodes.codes_grib_new_from_file(self.fd) return self