def get_array(self, key): """Get all data from file for the given BUFR key.""" with open(self.filename, "rb") as fh: msgCount = 0 while True: bufr = ec.codes_bufr_new_from_file(fh) if bufr is None: break ec.codes_set(bufr, 'unpack', 1) # if is the first message initialise our final array if (msgCount == 0): arr = da.from_array(ec.codes_get_array(bufr, key, float), chunks=CHUNK_SIZE) else: tmpArr = da.from_array(ec.codes_get_array( bufr, key, float), chunks=CHUNK_SIZE) arr = da.concatenate((arr, tmpArr)) msgCount = msgCount + 1 ec.codes_release(bufr) if arr.size == 1: arr = arr[0] return arr
def get_point_index(path, point, model): # custom version of the function in container_information_point # get clat and clon 1D arrays filename_clat = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clat.grib2' filename_clon = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clon.grib2' with open(path['base'] + path['grid'] + filename_clat, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clat = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) with open(path['base'] + path['grid'] + filename_clon, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clon = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) # read out index of native point filter_distance = get_latlon_filter_distance(model) lat_near = list(np.where(abs(clat - point['lat']) < filter_distance)[0]) lon_near = list(np.where(abs(clon - point['lon']) < filter_distance)[0]) id_near = list(set(lat_near).intersection(lon_near)) id_near.sort() distances = np.sqrt( np.square(abs(clat[id_near] - point['lat']) * 111.2) \ + np.square(abs(clon[id_near] - point['lon']) * 111.2 \ * np.cos(point['lat']*np.pi/180)) ) index_nearest = id_near[np.argmin(distances)] #print(id_near) #print(distances) #print(index_nearest) #print(clat[index_nearest], clon[index_nearest]) return index_nearest
def test_grib_get_array(): gid = eccodes.codes_grib_new_from_samples("reduced_gg_pl_160_grib1") pl = eccodes.codes_get_array(gid, "pl") assert pl[0] == 18 pli = eccodes.codes_get_array(gid, "pl", int) assert np.array_equal(pl, pli) pls = eccodes.codes_get_array(gid, "centre", str) assert pls == ["ecmf"] eccodes.codes_release(gid)
def get_subset_values(self, subset_nr , autoget_cval=False): # #[ """ request an array of values containing the values for a given subset for this bufr message """ if (self.msg_loaded == -1): raise NoMsgLoadedError data = [] field_names = self._get_abbr_names(subset_nr) print('field_names = ', field_names) print('DEBUG: names = ',self.get_names(subset_nr)) for field in field_names: if field[0] in string.digits: print('cannot get data for field: ',field) continue print('trying field name: ', field) s = eccodes.codes_get_size(self.bufr_id,field) if s==1: value = eccodes.codes_get(self.bufr_id,field) data.append(value) else: values = eccodes.codes_get_array(self.bufr_id,field) data.append(values[subset_nr]) return data
def decode_bufr_array(msgid, key): '''Returns a numpy maksed array of values corresponding to the fieldname. None if there is an error. ''' if key == 'datetime': return bufr_typical_timestamp(msgid) try: arr = eccodes.codes_get_array(msgid, key) if len(arr) == 0: return None dtype = eccodes.codes_get_native_type(msgid, key) if dtype is str: arr = numpy.char.strip(arr) if not isinstance(arr, numpy.ndarray): values = numpy.asarray(arr, dtype=dtype) else: values = arr if dtype is int: result = numpy.ma.masked_values(values, eccodes.CODES_MISSING_LONG) elif dtype is float: result = numpy.ma.masked_values(values, eccodes.CODES_MISSING_DOUBLE) else: result = numpy.ma.masked_array(data=values) return result except eccodes.CodesInternalError: return None
def main(): #point = dict(lat = 49.014, lon = 8.404, name = 'Karlsruhe') point = dict(lat = 50.822, lon = 8.920, name = 'Kirchhain') #point = dict(lat = 65.661, lon =-18.718, name = 'Iceland') path = dict(base = '/lsdfos/kit/imk-tro/projects/MOD/Gruppe_Knippertz/nw5893/', data = 'forecast_archive/pamore/kirchhain/run_12Z/', grid = 'forecast_archive/icon-eu-eps/grid/', plots = 'plots/kirchhain/run_12Z/') data_rain_gsp_sum = np.empty((25, 40, 75948)) data_rain_con_sum = np.empty((25, 40, 75948)) hours = list(range(0, 12+1)) filenames_all = [] for hour in hours: filenames_all.append(['iefff0{:03d}0000.m0{:02d}'.format(hour, member) for member in range(1, 41)]) with ExitStack() as stack: files_all = [[stack.enter_context(open(path['base'] + path['data'] + filename,'rb'))\ for filename in filenames_of_one_hour] for filenames_of_one_hour in filenames_all] for i, files_of_one_hour in enumerate(files_all): for j, file in enumerate(files_of_one_hour): grib_id = eccodes.codes_grib_new_from_file(file) data_rain_gsp_sum[i - hours[0], j, :] = eccodes.codes_get_array(grib_id, 'values') grib_id = eccodes.codes_grib_new_from_file(file) data_rain_con_sum[i - hours[0], j, :] = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) data_rain_tot_sum = data_rain_gsp_sum + data_rain_con_sum del data_rain_gsp_sum, data_rain_con_sum, files_all print(data_rain_tot_sum.max(axis=1).max(axis=1)) print('data_rain_tot_sum: {:.0f}MB'.format(data_rain_tot_sum.nbytes / 1e6)) stat_processing = 'max' member = None plot_rain_around_point(path, data_rain_tot_sum, 0, 12, point, stat_processing, member) '''stat_processing = 'member_extract' for member in range(1,41): plot_rain_around_point(path, data_rain_tot_sum, 0, 24, point, stat_processing, member)''' return
def fetch_model_output(input_file, time_since_init, short_name, level): file = open(input_file, "rb") gid = ec.codes_grib_new_from_file(file) file.close() values = read_grib_array(input_file, short_name, time_since_init, level) lat = np.deg2rad(ec.codes_get_array(gid, "latitudes")) lon = np.deg2rad(ec.codes_get_array(gid, "longitudes")) no_of_columns = ec.codes_get_long(gid, "Ni") no_of_lines = ec.codes_get_long(gid, "Nj") ec.codes_release(gid) lat_vector = np.zeros([no_of_lines]) lon_vector = np.zeros([no_of_columns]) for i in range(no_of_lines): lat_vector[i] = lat[i*no_of_columns] for i in range(no_of_columns): lon_vector[i] = lon[i] return lat_vector, lon_vector, vector_2_array(values, no_of_lines, no_of_columns)
def get(self, key, ktype=None): """Get value of a given key as its native or specified type.""" # if self.missing(key): # raise KeyError("Value of key %s is MISSING." % key) if eccodes.codes_get_size(self.codes_id, key) > 1: ret = eccodes.codes_get_array(self.codes_id, key, ktype) else: ret = eccodes.codes_get(self.codes_id, key, ktype) return ret
def get(self, name): # LOG.warn(str(self) + str(name)) if name in CHEAT: return CHEAT[name] try: if name == "values": return eccodes.codes_get_values(self.handle) size = eccodes.codes_get_size(self.handle, name) LOG.debug(f"{name}:{size}") if size and size > 1: return eccodes.codes_get_array(self.handle, name) return eccodes.codes_get(self.handle, name) except eccodes.KeyValueNotFoundError: return None
def cli(file_path): with open(file_path, 'rb') as f: handle = eccodes.codes_grib_new_from_file(f, headers_only=False) while handle is not None: date = eccodes.codes_get(handle, "dataDate") type_of_level = eccodes.codes_get(handle, "typeOfLevel") level = eccodes.codes_get(handle, "level") values = eccodes.codes_get_array(handle, "values") value = values[-1] values_array = eccodes.codes_get_values(handle, "values") value_array = values[-1] print(date, type_of_level, level, value) eccodes.codes_release(handle) handle = eccodes.codes_grib_new_from_file(f, headers_only=False)
def message_get(self, item, key_type=None, default=_MARKER): # type: (str, type, T.Any) -> T.Any """Get value of a given key as its native or specified type.""" key = item try: values = eccodes.codes_get_array(self.codes_id, key, key_type) if values is None: values = ['unsupported_key_type'] except eccodes.KeyValueNotFoundError: if default is _MARKER: raise KeyError(item) else: return default if len(values) == 1: return values[0] return values
def _get_abbr_names(self, subset=1): # #[ request abbr. name of each descriptor for the given subset ''' internal method to get the key name needed to extract the data ''' if (self.msg_loaded == -1): raise NoMsgLoadedError # remove entries that are not expanded and numeric. # these typically are replication codes and not part # of the field list abbr_names = [n for n in eccodes.codes_get_array(self.bufr_id, 'expandedAbbreviations') if not n[0] in string.digits] return abbr_names
def get_unexp_descr_list(self): # #[ get unexpanded descfriptor list ''' wrapper around self.bufr_obj.py_unexp_descr_list ''' if (self.msg_loaded == -1): raise NoMsgLoadedError n = eccodes.codes_get(self.bufr_id, 'numberOfUnexpandedDescriptors') if n==1: list_of_unexp_descr = [eccodes.codes_get(self.bufr_id, 'unexpandedDescriptors'),] else: list_of_unexp_descr = eccodes.codes_get_array(self.bufr_id, 'unexpandedDescriptors') return list_of_unexp_descr
def get_bufr_data(self, key): """Get BUFR data by key.""" attr = np.array([]) with open(self.filename, 'rb') as fh: while True: # get handle for message bufr = ec.codes_bufr_new_from_file(fh) if bufr is None: break ec.codes_set(bufr, 'unpack', 1) tmp = ec.codes_get_array(bufr, key, float) if len(tmp) == 1: size = ec.codes_get(bufr, 'numberOfSubsets') tmp = np.resize(tmp, size) attr = np.append(attr, tmp) ec.codes_release(bufr) return attr
def message_get(self, item, key_type=None, default=_MARKER): # type: (str, T.Optional[type], T.Any) -> T.Any """Get value of a given key as its native or specified type.""" try: values = eccodes.codes_get_array(self.codes_id, item, key_type) if values is None: values = ["unsupported_key_type"] except eccodes.KeyValueNotFoundError: if default is _MARKER: raise KeyError(item) else: return default if len(values) == 1: if isinstance(values, np.ndarray): values = values.tolist() return values[0] return values
def get_initial_values(idx, keep_sample=False): '''Get the values of surface z, pv and number of levels ''' codes_index_select(idx, 'level', 1) codes_index_select(idx, 'step', 0) codes_index_select(idx, 'shortName', 'z') gid = codes_new_from_index(idx) values = {} # surface geopotential values['z'] = codes_get_values(gid) values['pv'] = codes_get_array(gid, 'pv') values['nlevels'] = codes_get(gid, 'NV', int) // 2 - 1 check_max_level(idx, values) if keep_sample: values['sample'] = gid else: codes_release(gid) return values
def get_num_elements(self): # #[ """ request the number of elements (descriptors) in the current subset """ if (self.msg_loaded == -1): raise NoMsgLoadedError fieldnames = eccodes.codes_get_array(self.bufr_id, 'expandedNames') # other possibilities are: # 'expandedAbbreviations' # 'expandedNames' # 'expandedUnits' # 'expandedOriginalScales' # 'expandedOriginalReferences' # 'expandedOriginalWidths' return len(fieldnames)
def extract_msg_date_extremes(self, bufr, date_min=None, date_max=None): """Extract the minimum and maximum dates from a single bufr message.""" ec.codes_set(bufr, 'unpack', 1) size = ec.codes_get(bufr, 'numberOfSubsets') years = np.resize(ec.codes_get_array(bufr, 'year'), size) months = np.resize(ec.codes_get_array(bufr, 'month'), size) days = np.resize(ec.codes_get_array(bufr, 'day'), size) hours = np.resize(ec.codes_get_array(bufr, 'hour'), size) minutes = np.resize(ec.codes_get_array(bufr, 'minute'), size) seconds = np.resize(ec.codes_get_array(bufr, 'second'), size) for year, month, day, hour, minute, second in zip(years, months, days, hours, minutes, seconds): time_stamp = datetime(year, month, day, hour, minute, second) date_min = time_stamp if not date_min else min(date_min, time_stamp) date_max = time_stamp if not date_max else max(date_max, time_stamp) return date_min, date_max
def get_values(self, descr_nr, autoget_cval=False): # #[ """ request an array of values containing the values for a given descriptor number for all subsets NOTE: this may not work for templates using delayed replication. """ if (self.msg_loaded == -1): raise NoMsgLoadedError list_of_names = self._get_abbr_names() keyname = list_of_names[descr_nr] print('keyname: ', keyname) s = eccodes.codes_get_size(self.bufr_id,keyname) t = eccodes.codes_get_native_type(self.bufr_id, keyname) print('key:', keyname, 'size = ', s, 'type = ', t) if s==1: # or t==str: # values = eccodes.codes_get_string(bufr_id, keyname) values = [eccodes.codes_get(self.bufr_id, keyname),] else: values = eccodes.codes_get_array(self.bufr_id, keyname) return values
i + 1 for i in range(len(varNames)) if (varParCat[i] == 1 and varParNum[i] == 23 and levels[i] in levsIncl) ]) #Ice mixing ratio gidICEMR = gidICEMR[::-1] gidSNOWMR = np.flipud([ i + 1 for i in range(len(varNames)) if (varParCat[i] == 1 and varParNum[i] == 25 and levels[i] in levsIncl) ]) #Snow mixing ratio gidSNOWMR = gidSNOWMR[::-1] gidGRPMR = np.flipud([ i + 1 for i in range(len(varNames)) if (varParCat[i] == 1 and varParNum[i] == 32 and levels[i] in levsIncl) ]) #Graupel mixing ratio gidGRPMR = gidGRPMR[::-1] lats = eccodes.codes_get_array(gidPRMSL, 'distinctLatitudes') lons = eccodes.codes_get_array(gidPRMSL, 'distinctLongitudes') Ni = eccodes.codes_get(gidPRMSL, 'Ni') Nj = eccodes.codes_get(gidPRMSL, 'Nj') for i in range(len(lats) - 1, -1, -1): if lats[i] >= latMinCP: iLatMinGRIB = i + 1 print "i", (i), lats[i], lats[i + 1] break for i in range(len(lats) - 1, -1, -1): if lats[i] > latMaxCP: iLatMaxGRIB = i break for i in range(len(lons) - 1): if lons[i + 1] >= lonMinCP:
def plot_contourmap(path, date, fcst_hours, variable, stat_processing_methods): ##### generate subpath and filename ##### subpath = 'run_{}{:02}{:02}{:02}/{}/'.format(\ date['year'], date['month'], date['day'], date['hour'], variable) filename1 = 'icon-eu-eps_europe_icosahedral_single-level_{}{:02}{:02}{:02}_{:03}_{}.grib2'.format(\ date['year'], date['month'], date['day'], date['hour'], fcst_hours[0], variable) filename2 = 'icon-eu-eps_europe_icosahedral_single-level_{}{:02}{:02}{:02}_{:03}_{}.grib2'.format(\ date['year'], date['month'], date['day'], date['hour'], fcst_hours[1], variable) ######################################################################## ### read data ### ######################################################################## ##### create empty numpy arrays ##### ##### 2 fcst_hours, 40 members, 75948 eu gridpoints ##### data_raw = np.empty((2, 40, 75948)) data_members = np.empty((40, 75948)) ##### every time in loop open next grib msg from grib file ##### ##### grib messages in dwd file are sorted by increasing member number ##### with open(path['base'] + path['data'] + subpath + filename1, 'rb') as file: for member in range(1, 41): print('read data from member {}'.format(member)) grib_msg_id = eccodes.codes_grib_new_from_file(file) data_raw[0, member - 1, :] = eccodes.codes_get_array( grib_msg_id, 'values') eccodes.codes_release(grib_msg_id) with open(path['base'] + path['data'] + subpath + filename2, 'rb') as file: for member in range(1, 41): print('read data from member {}'.format(member)) grib_msg_id = eccodes.codes_grib_new_from_file(file) data_raw[1, member - 1, :] = eccodes.codes_get_array( grib_msg_id, 'values') eccodes.codes_release(grib_msg_id) ##### take the difference of the two accumulated total precipitation arrays ##### data_members = data_raw[1, :, :] - data_raw[0, :, :] del data_raw ##### open icon-eps grid file ##### icongrid_file = nc.Dataset( path['base'] + path['grid'] + 'icon_grid_0028_R02B07_N02.nc', 'r') vlat = icongrid_file.variables['clat_vertices'][:].data * 180. / np.pi vlon = icongrid_file.variables['clon_vertices'][:].data * 180. / np.pi clat = icongrid_file.variables['clat'][:].data * 180. / np.pi clon = icongrid_file.variables['clon'][:].data * 180. / np.pi icongrid_file.close() for stat_processing in stat_processing_methods: ######################################################################## ### statistically process data ### ######################################################################## if stat_processing == 'mean': data_processed = data_members.mean(axis=0) elif stat_processing == 'max': data_processed = data_members.max(axis=0) elif stat_processing == 'min': data_processed = data_members.min(axis=0) elif stat_processing == '90p': data_processed = np.percentile(data_members, 90, axis=0) elif stat_processing == '75p': data_processed = np.percentile(data_members, 75, axis=0) elif stat_processing == '50p': data_processed = np.percentile(data_members, 50, axis=0) elif stat_processing == '25p': data_processed = np.percentile(data_members, 25, axis=0) elif stat_processing == '10p': data_processed = np.percentile(data_members, 10, axis=0) #print('shape of data_members: {}'.format(np.shape(data_members))) #print('shape of data_processed: {}'.format(np.shape(data_processed))) ######################################################################## ### plot data on world map ### ######################################################################## ##### set domain due to center point and radius ##### center_point = dict(lat=48.5, lon=9.0) radius = 1800 # domain radius in km around center_point domain = dict( lat_min=center_point['lat'] - radius / 111.2, lat_max=center_point['lat'] + radius / 111.2, lon_min=center_point['lon'] - radius / (111.2 * np.cos(center_point['lat'] * np.pi / 180)), lon_max=center_point['lon'] + radius / (111.2 * np.cos(center_point['lat'] * np.pi / 180)), ) ##### or set domain manually in deg N/E ##### '''domain = dict( lat_min = 0.0, lat_max = 20.0, lon_min = 0.0, lon_max = 20.0, )''' ##### set image size (should be squared) ##### ##### plotting area in pyngl can not exceed squared area even if plotting on rectangular images ##### ##### for obtaining rectangular plots on has to cut manually afterwards e.g. with pillow package ##### x_resolution = 800 y_resolution = 800 wks_res = Ngl.Resources() wks_res.wkWidth = x_resolution wks_res.wkHeight = y_resolution plot_name = 'contourplot_icon-eu-eps_tot_prec_{:02d}-{:02d}h_{}'.format(\ fcst_hours[0], fcst_hours[1], stat_processing) wks_type = 'png' wks = Ngl.open_wks(wks_type, path['base'] + path['plots'] + plot_name, wks_res) resources = Ngl.Resources( ) # create resources object containing all the plot settings resources.mpProjection = 'Hammer' # projection type resources.mpCenterLonF = (domain['lon_max'] + domain['lon_min'] ) / 2 # projection center point resources.mpCenterLatF = (domain['lat_max'] + domain['lat_min']) / 2 resources.mpLimitMode = 'latlon' resources.mpMinLonF = domain['lon_min'] resources.mpMaxLonF = domain['lon_max'] resources.mpMinLatF = domain['lat_min'] resources.mpMaxLatF = domain['lat_max'] ##### set plot area ##### resources.nglMaximize = False resources.vpXF = 0.05 resources.vpYF = 0.9 resources.vpWidthF = 0.7 resources.vpHeightF = 0.7 ##### set all map plot settings ##### resources.mpFillOn = True # turn on filled map areas resources.mpFillColors = [ 'pink', 'blue', 'white', 'white' ] # set colors for [FillValue, Ocean, Land , InlandWater] resources.mpDataBaseVersion = 'MediumRes' # quality of national borders resources.mpDataSetName = 'Earth..4' resources.mpOutlineBoundarySets = 'national' #resources.mpDataBaseVersion = 'HighRes' #resources.mpDataResolution = 'Fine' resources.mpGeophysicalLineThicknessF = 7.0 * x_resolution / 1000 # keep borders thickness resolution-independent resources.mpNationalLineThicknessF = 7.0 * x_resolution / 1000 #resources.mpGridAndLimbDrawOrder = 'postdraw' resources.mpGridAndLimbOn = False # turn off geographic coordinates grid #resources.mpLimbLineColor = 'black' #resources.mpLimbLineThicknessF = 10 #resources.mpGridLineColor = 'black' #resources.mpGridLineThicknessF = 1.0 #resources.mpGridSpacingF = 1 resources.mpPerimOn = True # turn on perimeter around plot resources.mpPerimLineColor = 'black' resources.mpPerimLineThicknessF = 8.0 * x_resolution / 1000 # keep perimeter thickness resolution-independent resources.tmXBOn = False # turn off location ticks around plot resources.tmXTOn = False resources.tmYLOn = False resources.tmYROn = False resources.sfDataArray = data_processed # data input file to plot resources.sfXArray = clon # array with cell center locations resources.sfYArray = clat resources.sfXCellBounds = vlon # array with cell vertices locations resources.sfYCellBounds = vlat resources.sfMissingValueV = 9999 # in case you want to mask values resources.cnFillOn = True resources.cnFillMode = 'CellFill' #resources.cnCellFillEdgeColor = 'black' # uncomment this for plotting the cell edges resources.cnMissingValFillColor = 'black' resources.cnFillPalette = 'WhiteBlueGreenYellowRed' # color palette resources.cnLevelSelectionMode = 'ManualLevels' minlevel = 0.0 # min level of colorbar maxlevel = 50.0 # max level of colorbar numberoflevels = 250 # number of levels of colorbar, max. 250 with this color palette resources.cnMinLevelValF = minlevel resources.cnMaxLevelValF = maxlevel resources.cnLevelSpacingF = (maxlevel - minlevel) / numberoflevels resources.cnLinesOn = False # turn off contour lines resources.cnLineLabelsOn = False # turn off contour labels ##### set resources for a nice colorbar ##### resources.lbLabelBarOn = True resources.lbAutoManage = False resources.lbOrientation = 'vertical' resources.lbLabelOffsetF = 0.05 #resources.lbBoxMinorExtentF = 0.2 resources.lbLabelStride = 25 # print a tick every 25 levels resources.lbLabelFontHeightF = 0.016 resources.lbBoxSeparatorLinesOn = False resources.lbBoxLineThicknessF = 4.0 #resources.lbBoxEndCapStyle = 'TriangleBothEnds' resources.lbLabelAlignment = 'BoxCenters' resources.lbTitleString = 'mm' resources.lbTitleFontHeightF = 0.016 resources.lbTitlePosition = 'Right' resources.lbTitleDirection = 'Across' resources.lbTitleAngleF = 90.0 resources.lbTitleExtentF = 0.1 resources.lbTitleOffsetF = -0.15 resources.nglFrame = False # hold on frame because will plot text afterwards on same plot Ngl.contour_map(wks, data_processed, resources) # plot the actual plot ##### plot title text ##### text = '{:02d}-{:02d}h Total Precipitation {}, ICON-EPS run {:02}.{:02}.{} {:02}Z'.format(\ fcst_hours[0], fcst_hours[1], stat_processing,\ date['day'], date['month'], date['year'], date['hour']) x = 0.5 y = 0.95 text_res_1 = Ngl.Resources() text_res_1.txFontHeightF = 0.018 text_res_1.txJust = 'TopCenter' Ngl.text_ndc(wks, text, x, y, text_res_1) Ngl.frame(wks) # advance frame Ngl.destroy( wks ) # delete workspace to free memory, relevant if plotting various plots in one script print('plotted "{}.png"'.format(plot_name)) return
def get_ecc_msg(gid, namespace=None, skipkeys=None): """Read data from one particular ecc message Parameters ---------- gid : ecc message id namespace : string namespace to be retrieved, defaults to None (means all) 'ls', 'parameter', 'time', 'geography', 'vertical', 'statistics', 'mars' skipkeys : list of strings keys to be skipped, defaults to None possible keys: 'computed', 'coded', 'edition', 'duplicates', 'read_only', 'function' Returns ------- data : dictionary of ecc message contents """ # get key iterator iterid = ecc.codes_keys_iterator_new(gid, namespace) # Different types of keys can be skipped if skipkeys: if 'computed' in skipkeys: ecc.codes_skip_computed(iterid) if 'coded' in skipkeys: ecc.codes_skip_coded(iterid) if 'edition' in skipkeys: ecc.codes_skip_edition_specific(iterid) if 'duplicates' in skipkeys: ecc.codes_skip_duplicates(iterid) if 'read_only' in skipkeys: ecc.codes_skip_read_only(iterid) if 'function' in skipkeys: ecc.codes_skip_function(iterid) data = OrderedDict() # iterate over message keys while ecc.codes_keys_iterator_next(iterid): keyname = ecc.codes_keys_iterator_get_name(iterid) #print(keyname) #print("Size:", ecc.codes_get_size(gid, keyname)) #print("Values:", ecc.codes_get_values(gid, keyname)) #print("Array:", ecc.codes_get_values(gid, keyname)) # try to get key values, # use get_array for sizes > 1 and get for sizes == 1 if ecc.codes_get_size(gid,keyname) > 1: #print("has array", type is str) #print(type is not <type 'str'>) if ecc.codes_get_native_type(iterid, keyname) is not str: keyval = ecc.codes_get_array(gid, keyname, None) else: keyval = ecc.codes_get(gid, keyname, None) #print("Arr:", keyval) else: # Todo: fix reading mybits if keyname not in ['mybits']: keyval = ecc.codes_get(gid, keyname, None) #print("Val:", keyval) else: keyval = 'err' # add keyname-keyvalue-pair to output dictionary data[keyname] = keyval #print('Message processed') # release iterator ecc.codes_keys_iterator_delete(iterid) return data
def bufr_decode(f, fn, archive, args, fakeTimes=True, fakeDisplacement=True, logFixup=True): ibufr = codes_bufr_new_from_file(f) if not ibufr: raise BufrUnreadableError("empty file", fn, archive) codes_set(ibufr, "unpack", 1) missingHdrKeys = 0 header = {} try: k = "extendedDelayedDescriptorReplicationFactor" num_samples = codes_get_array(ibufr, k)[0] except Exception as e: codes_release(ibufr) raise MissingKeyError(k, message=f"cant determine number of samples: {e}") # BAIL HERE if no num_samples ivals = [ "typicalYear", "typicalMonth", "typicalDay", "typicalHour", "typicalMinute", "typicalSecond", "blockNumber", "stationNumber", "radiosondeType", "height", "year", "month", "day", "hour", "minute", "second", "correctionAlgorithmsForHumidityMeasurements", "pressureSensorType", "temperatureSensorType", "humiditySensorType", "geopotentialHeightCalculation", "trackingTechniqueOrStatusOfSystem", "measuringEquipmentType", ] fvals = [ "radiosondeOperatingFrequency", "latitude", "longitude", "heightOfStationGroundAboveMeanSeaLevel", "heightOfBarometerAboveMeanSeaLevel", ] svals = [ "radiosondeSerialNumber", "typicalDate", "typicalTime", "text", "softwareVersionNumber", ] for k in ivals + fvals + svals: try: value = codes_get(ibufr, k) if k in ivals: if value != CODES_MISSING_LONG: header[k] = value elif k in fvals: if value != CODES_MISSING_DOUBLE: header[k] = value elif k in svals: header[k] = value else: pass except Exception as e: logging.debug(f"missing header key={k} e={e}") missingHdrKeys += 1 # special-case warts we do not really care about warts = ["shipOrMobileLandStationIdentifier"] for k in warts: try: header[k] = codes_get(ibufr, k) except Exception: missingHdrKeys += 1 fkeys = [ # 'extendedVerticalSoundingSignificance', "pressure", "nonCoordinateGeopotentialHeight", "latitudeDisplacement", "longitudeDisplacement", "airTemperature", "dewpointTemperature", "windDirection", "windSpeed", ] samples = [] invalidSamples = 0 missingValues = 0 fakeTimeperiod = 0 fixups = [] # report once only for i in range(1, num_samples + 1): sample = {} k = "timePeriod" timePeriod = codes_get(ibufr, f"#{i}#{k}") if timePeriod == CODES_MISSING_LONG: invalidSamples += 1 if not fakeTimes: continue else: timePeriod = fakeTimeperiod fakeTimeperiod += FAKE_TIME_STEPS if k not in fixups: logging.debug( f"FIXUP timePeriod fakeTimes:{fakeTimes} fakeTimeperiod={fakeTimeperiod}" ) fixups.append(k) sample[k] = timePeriod replaceable = ["latitudeDisplacement", "longitudeDisplacement"] sampleOK = True for k in fkeys: name = f"#{i}#{k}" try: value = codes_get(ibufr, name) if value != CODES_MISSING_DOUBLE: sample[k] = value else: if fakeDisplacement and k in replaceable: if k not in fixups: logging.debug(f"--FIXUP key {k}") fixups.append(k) sample[k] = 0 else: # logging.warning(f"--MISSING {i} key {k} ") sampleOK = False missingValues += 1 except Exception as e: sampleOK = False logging.debug(f"sample={i} key={k} e={e}, skipping") missingValues += 1 if sampleOK: samples.append(sample) logging.debug((f"samples used={len(samples)}, invalid samples=" f"{invalidSamples}, skipped header keys={missingHdrKeys}," f" missing values={missingValues}")) codes_release(ibufr) return header, samples
def main(): path = dict( base='/lsdfos/kit/imk-tro/projects/MOD/Gruppe_Knippertz/nw5893/', data='forecast_archive/pamore/heatwave_25.07.19/', grid='forecast_archive/icon-eu-eps/grid/', plots='plots/heatwave_25.07.19/point_plots/') run = dict(year=2019, month=7, day=21, hour=0) hours = [ 6, 12, 18, 100, 106, 112, 118, 200, 206, 212, 218, 300, 306, 312, 318, 400, 406, 412, 418, 500 ] data_tmax_6h = np.empty((len(hours), 40, 75948)) filenames_all = [] for hour in hours: filenames_all.append([ 'iefff0{:03d}0000.m0{:02d}'.format(hour, member) for member in range(1, 41) ]) path['data_subfolder'] = 'run_{:4d}{:02d}{:02d}{:02d}/'.format( run['year'], run['month'], run['day'], run['hour']) with ExitStack() as stack: files_all = [[stack.enter_context(open(path['base'] + path['data'] + path['data_subfolder'] + filename,'rb'))\ for filename in filenames_of_one_hour] for filenames_of_one_hour in filenames_all] for i, files_of_one_hour in enumerate(files_all): for j, file in enumerate(files_of_one_hour): grib_id = eccodes.codes_grib_new_from_file(file) data_tmax_6h[i, j, :] = eccodes.codes_get_array( grib_id, 'values') eccodes.codes_release(grib_id) del files_all, files_of_one_hour, file data_tmax_6h -= 273.15 data_tmax_24h = data_tmax_6h.reshape(5, 4, 40, 75948).max(axis=1) points = [] points.append( dict(lat=49.014, lon=8.350, name='Karlsruhe', measurements=[30.0, 30.0, 30.0, 30.0, 30.0])) points.append( dict(lat=48.860, lon=2.350, name='Paris', measurements=[30.0, 30.0, 30.0, 30.0, 30.0])) points.append( dict(lat=50.937, lon=6.954, name='Koeln', measurements=[30.0, 30.0, 30.0, 30.0, 30.0])) points.append( dict(lat=47.198, lon=-1.534, name='Nantes', measurements=[30.0, 30.0, 30.0, 30.0, 30.0])) points.append( dict(lat=52.519, lon=13.407, name='Berlin', measurements=[30.0, 30.0, 30.0, 30.0, 30.0])) points.append( dict(lat=48.240, lon=11.570, name='Munchen', measurements=[30.0, 30.0, 30.0, 30.0, 30.0])) points.append( dict(lat=52.489, lon=-3.462, name='Wales', measurements=[30.0, 30.0, 30.0, 30.0, 30.0])) for point in points: index_nearest = get_point_index(path, point, 'icon-eu-eps') ens_data = data_tmax_24h[:, :, index_nearest] print(point['name']) #print(ens_data) plot_tmax_uncertainty_shades(path, run, point, ens_data) return
def read_grid_coordinates(model, grid): path = dict(base='/') path['grid'] = 'data/model_data/{}/grid/'.format(model) if model == 'icon-eu-eps': if grid == 'icosahedral': filename = 'icon_grid_0028_R02B07_N02.nc' elif grid == 'latlon_0.2': filename = 'icon-eu-eps_latlon_0.2_grid_coordinates.nc' elif model == 'icon-global-eps': filename = 'icon_grid_0024_R02B06_G.nc' elif model == 'icon-eu-det': filename_clat = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2' filename_clon = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2' elif model == 'icon-global-det': if grid == 'icosahedral': filename = 'icon_grid_0026_R03B07_G.nc' elif grid == 'latlon_0.25': filename = 'icon-global-det_latlon_0.25_grid_coordinates.nc' elif grid == 'latlon_0.1': filename = 'icon-global-det_latlon_0.1_grid_coordinates.nc' if grid == 'icosahedral': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['clat'].values * 180 / np.pi clon = ds['clon'].values * 180 / np.pi vlat = ds['clat_vertices'].values * 180 / np.pi vlon = ds['clon_vertices'].values * 180 / np.pi return clat, clon, vlat, vlon elif grid == 'latlon_0.0625': with open(path['base'] + path['grid'] + filename_clat, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clat = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) with open(path['base'] + path['grid'] + filename_clon, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clon = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) return clat.reshape((657, 1097)), clon.reshape((657, 1097)) elif grid == 'latlon_0.2': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['lat'].values clon = ds['lon'].values return clat, clon elif grid == 'latlon_0.25': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['lat'].values clon = ds['lon'].values clon = np.where(clon > 180, clon - 360, clon) clon_new = np.empty_like(clon) clon_new[719:] = clon[:721] clon_new[:719] = clon[721:] return clat, clon_new elif grid == 'latlon_0.1': with xr.open_dataset(path['base'] + path['grid'] + filename) as ds: clat = ds['lat'].values clon = ds['lon'].values clon = np.where(clon > 180, clon - 360, clon) clon_new = np.empty_like(clon) clon_new[1799:] = clon[:1801] clon_new[:1799] = clon[1801:] return clat, clon_new
def get_point_index(model, point): # if known named point get grid point location # if not 'lat' in point: #print('pointname is known: {}'.format(point['name'])) point = which_grid_point(point['name'], model) path = dict(base='/') path['subdir'] = 'data/model_data/{}/grid/'.format(model) if model == 'icon-eu-eps': filename_clat = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clat.grib2' filename_clon = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121000_clon.grib2' elif model == 'icon-global-eps': filename_clat = 'icon-eps_global_icosahedral_time-invariant_2019010800_clat.grib2' filename_clon = 'icon-eps_global_icosahedral_time-invariant_2019010800_clon.grib2' elif model == 'icon-eu-det': filename_clat = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2' filename_clon = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2' elif model == 'icon-global-det': filename_clat = 'icon_global_icosahedral_time-invariant_2020020700_CLAT.grib2' filename_clon = 'icon_global_icosahedral_time-invariant_2020020700_CLON.grib2' # get clat and clon 1D arrays # with open(path['base'] + path['subdir'] + filename_clat, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clat = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) with open(path['base'] + path['subdir'] + filename_clon, 'rb') as file: grib_id = eccodes.codes_grib_new_from_file(file) clon = eccodes.codes_get_array(grib_id, 'values') eccodes.codes_release(grib_id) # read out index of nearest icosahedral point # filter_distance = get_latlon_filter_distance(model) lat_near = list(np.where(abs(clat - point['lat']) < filter_distance)[0]) lon_near = list(np.where(abs(clon - point['lon']) < filter_distance)[0]) id_near = list(set(lat_near).intersection(lon_near)) id_near.sort() distances = np.sqrt( np.square(abs(clat[id_near] - point['lat']) * 111.2) \ + np.square(abs(clon[id_near] - point['lon']) * 111.2 \ * np.cos(point['lat']*np.pi/180)) ) index_nearest = id_near[np.argmin(distances)] #print(id_near) #print(distances) #print(index_nearest) #print(clat[index_nearest], clon[index_nearest]) if model == 'icon-eu-det': # this model data is on regular lat-lon-grid # lat_index = int((clat[index_nearest] - 29.5) / 0.0625) lon_index = int((clon[index_nearest] + 23.5) / 0.0625) point_index = [lat_index, lon_index] else: # all other model data is on icosahedral grid with single index # point_index = [index_nearest] return point_index
def read_grib_array(filename, short_name, time_since_init, level): gid = scan_for_gid(filename, short_name, time_since_init, level) return_array = ec.codes_get_array(gid, "values") ec.codes_release(gid) return return_array
def select_messages(self, **kwargs): self._selected_grbs = self._get_gids(**kwargs) self._log(f'Selected {len(self._selected_grbs)} grib messages') if len(self._selected_grbs) > 0: self._gid_main_res = self._selected_grbs[0] grid = GribGridDetails(self._selected_grbs[0]) # some cumulated messages come with the message at step=0 as instant, to permit aggregation # cumulated rainfall rates could have the step zero instant message as kg/m^2, instead of kg/(m^2*s) if len(self._selected_grbs) > 1: unit = codes_get(self._selected_grbs[1], 'units') type_of_step = codes_get(self._selected_grbs[1], 'stepType') else: type_of_step = codes_get(self._selected_grbs[0], 'stepType') unit = codes_get(self._selected_grbs[0], 'units') type_of_level = codes_get(self._selected_grbs[0], 'levelType') missing_value = codes_get(self._selected_grbs[0], 'missingValue') data_date = codes_get(self._selected_grbs[0], 'dataDate') all_values = {} all_values_second_res = {} grid2 = None input_step = self._step_grib for g in self._selected_grbs: start_step = codes_get(g, 'startStep') end_step = codes_get(g, 'endStep') points_meridian = codes_get(g, 'Nj') level = codes_get(g, 'level') if f'{start_step}-{end_step}' == self._change_step_at: # second time resolution input_step = self._step_grib2 step_key = Step(start_step, end_step, points_meridian, input_step, level) if points_meridian != grid.num_points_along_meridian and grid.get_2nd_resolution( ) is None: # found second resolution messages grid2 = GribGridDetails(g) self._gid_ext_res = g values = codes_get_double_array(g, 'values') # Handling missing grib values. # If bitmap is present, array will be a masked_array # and array.mask will be used later # in interpolation and manipulation bitmap_present = codes_get(g, 'bitmapPresent') if bitmap_present: # Get the bitmap array which contains 0s and 1s bitmap = codes_get_array(g, 'bitmap', int) values = ma.masked_where(bitmap == 0, values, copy=False) if not grid2: all_values[step_key] = values elif points_meridian != grid.num_points_along_meridian: all_values_second_res[step_key] = values if grid2: key_2nd_spatial_res = min(all_values_second_res.keys()) grid.set_2nd_resolution(grid2, key_2nd_spatial_res) return Messages(all_values, missing_value, unit, type_of_level, type_of_step, grid, all_values_second_res, data_date=data_date) # no messages found else: raise ApplicationException.get_exc(NO_MESSAGES, details=f'using {kwargs}')
def read_grib_array(filename, short_name): gid = scan_for_gid(filename, short_name) return_array = ec.codes_get_array(gid, "values") ec.codes_release(gid) return return_array
def get_units(self, subset=1): # #[ request unit of each descriptor for the given subset if (self.msg_loaded == -1): raise NoMsgLoadedError return eccodes.codes_get_array(self.bufr_id, 'expandedUnits')
def read_grib_property(filename, short_name, key): gid = scan_for_gid(filename, short_name) value = ec.codes_get_array(gid, key) ec.codes_release(gid) return value[0]
def main(): path = dict( base='/lsdfos/kit/imk-tro/projects/MOD/Gruppe_Knippertz/nw5893/', data='forecast_archive/pamore/heatwave_25.07.19/', grid='forecast_archive/icon-eu-eps/grid/', plots='plots/heatwave_25.07.19/', colorpalette='additional_data/colorpalettes/') for lead_days in [4]: #list(range(5)): run = dict(year=2019, month=7, day=25 - lead_days, hour=0) hours = [6, 12, 18, 100] for i in range(len(hours)): hours[i] += lead_days * 100 data_tmax_6h = np.empty((4, 40, 75948)) filenames_all = [] for hour in hours: filenames_all.append([ 'iefff0{:03d}0000.m0{:02d}'.format(hour, member) for member in range(1, 41) ]) path['data_subfolder'] = 'run_{:4d}{:02d}{:02d}{:02d}/'.format( run['year'], run['month'], run['day'], run['hour']) with ExitStack() as stack: files_all = [[stack.enter_context(open(path['base'] + path['data'] + path['data_subfolder'] + filename,'rb'))\ for filename in filenames_of_one_hour] for filenames_of_one_hour in filenames_all] for i, files_of_one_hour in enumerate(files_all): for j, file in enumerate(files_of_one_hour): grib_id = eccodes.codes_grib_new_from_file(file) data_tmax_6h[i, j, :] = eccodes.codes_get_array( grib_id, 'values') eccodes.codes_release(grib_id) del files_all, files_of_one_hour, file data_tmax_6h -= 273.15 data_tmax_24h = data_tmax_6h.max(axis=0) ##### settings for plotting ##### #point = dict(lat = 49.01, lon = 8.40, name = 'Karlsruhe') #point = dict(lat = 50.82, lon = 8.92, name = 'Kirchhain') #domain = dict(method = 'deltalatlon', radius = 0, deltalat = 700, deltalon = 760,\ # lat = 48.7, lon = 5.4, name = 'france-germany') domain = dict(method = 'deltalatlon', radius = 0, deltalat = 850, deltalon = 680,\ lat = 48.4, lon = 5.0, name = 'france-germany') #stat_processing = dict(method = 'max') #stat_processing = dict(method = 'min') #stat_processing = dict(method = 'median') #stat_processing = dict(method = 'spread') #stat_processing = dict(method = 'member_extract', member = 1) thresholds = [] #thresholds.append(30.0) thresholds.append(36.0) #thresholds.append(38.0) #thresholds.append(40.0) ##### call plotting function ##### #for stat_processing['member'] in range(1,41): #plot_statistical_value_around_point(path, run, data_tmax_24h, point, stat_processing) for threshold in thresholds: plot_prob_of_exceedance_around_point(path, run, data_tmax_24h, domain, threshold) return