def get_field(fnam, req):
f = ecc.GribFile(fnam)
nfound = 0
msghit = None
for i in range(len(f)):
msg = ecc.GribMessage(f)
matchlist = []
for key in req:
if str(msg[key]) == str(req[key]):
matchlist.append(1)
continue
else:
break
if len(matchlist) == len(req):
if msghit:
print("Ambigous spesification! found more than one match")
print(req)
exit(1)
msghit = msg
nfound += 1
if nfound == 0:
print("No match!")
print(req)
nx = msghit['Nx']
ny = msghit['Ny']
name = msghit['parameterName']
val = ma.masked_values(np.flipud(msghit['values'].reshape((ny, nx))),
msghit['missingValue'])
return {'vals': val, 'name': name}
def _parse_grib(self):
"""Parser for GRIB data."""
with eccodes.GribFile(self._fname) as grib:
for msg in grib:
valid_date = msg["validityDate"]
year = valid_date // 10000
month = valid_date % 10000 // 100
day = valid_date % 10000 % 100
valid_time = msg["validityTime"]
hour = valid_time // 100
minute = valid_time % 100
datime = dt.datetime(year, month, day, hour, minute)
if self.latitudes is None:
self.latitudes = np.reshape(msg["latitudes"], (msg["Nj"], msg["Ni"]))
self.longitudes = np.reshape(msg["longitudes"], (msg["Nj"], msg["Ni"]))
if datime not in self.data:
self.data[datime] = dict()
if msg["level"] not in self.data[datime]:
self.data[datime][msg["level"]] = dict()
level = self.data[datime][msg["level"]]
level[msg["name"]] = dict()
data = np.reshape(msg["values"], (msg["Nj"], msg["Ni"]))
data[data == msg["missingValue"]] = np.nan
level[msg["name"]]["data"] = data
level[msg["name"]]["units"] = msg["units"]
def parseMetadata(path):
grbs = eccodes.GribFile(path)
# TODO: get bounds and timeframe
bounds = [0, 0, 0, 0]
hoursForecasted = None
startTime = None
rindex = {}
centre = ''
for r in grbs:
if r['name'] != '10 metre U wind component' and r[
'name'] != '10 metre V wind component':
continue
if 'centre' in r.keys():
centre = r['centre']
if 'forecastTime' in r.keys():
ft = r['forecastTime']
else:
ft = r['P1']
startTime = datetime.datetime(int(r['year']), int(r['month']),
int(r['day']), int(r['hour']),
int(r['minute']))
if hoursForecasted == None or hoursForecasted < int(ft):
hoursForecasted = int(ft)
grbs.close()
return MetaGrib(path,
path.split('/')[-1], centre, bounds, startTime,
hoursForecasted)
def read_gf(fnam):
gfin = ecc.GribFile(fnam)
msgs = []
for i in range(len(gfin)):
msg = ecc.GribMessage(gfin)
msgs.append(ecc.GribMessage(clone=msg))
gfin.close()
del gfin
return msgs
def compare_file(gf_their, gf_my):
with ecc.GribFile(gf_their) as tf, ecc.GribFile(gf_my) as mf:
if len(tf) != len(mf):
print("different length!")
raise Exception
for i in range(len(tf)):
tmsg = ecc.GribMessage(tf)
mmsg = ecc.GribMessage(mf)
nx = tmsg['Nx']
ny = tmsg['Ny']
tv = tmsg['values'].reshape((ny, nx))
mv = mmsg['values'].reshape((ny, nx))
if 0 in mv:
denom = 1
else:
denom = mv
plt.imshow((mv - tv) / denom)
plt.colorbar()
plt.title("%f - %f" % ((mv - tv).min(), (mv - tv).max()))
plt.savefig(gf_my + '.png')
plt.close()
def parse(path):
grbs = eccodes.GribFile(path)
# TODO: get bounds and timeframe
bounds = [0, 0, 0, 0]
hoursForecasted = None
startTime = None
rindex = {}
centre = ''
for r in grbs:
if r['name'] != '10 metre U wind component' and r[
'name'] != '10 metre V wind component':
continue
if 'centre' in r.keys():
centre = r['centre']
if 'forecastTime' in r.keys():
ft = r['forecastTime']
else:
ft = r['P1']
startTime = datetime.datetime(int(r['year']), int(r['month']),
int(r['day']), int(r['hour']),
int(r['minute']))
if hoursForecasted == None or hoursForecasted < int(ft):
hoursForecasted = int(ft)
# timeIndex = str(r['dataDate'])+str(r['dataTime'])
if r['name'] == '10 metre U wind component':
rindex[hoursForecasted] = {'u': r.gid}
elif r['name'] == '10 metre V wind component':
rindex[hoursForecasted]['v'] = r.gid
with eccodes.GribIndex(path, ['name', 'P1']) as idx:
idx.write(path + '.idx')
grbs.close()
return Grib(path,
path.split('/')[-1], centre, bounds, rindex, startTime,
hoursForecasted)
def __init__(self, filename, code):
'''
The class Data loads data from the .grib file and performs the desired
manipulations on it. A Data object is the used by the functions from the
plot module to produce plots.
Parameters
----------
filename : str
location and filename of the .grib file
code : str
short name of the dataset to filter the convenient messages
Returns
-------
None.
'''
i = 0
self.mean_time = 0
self.mean_space = 0
self.lat = []
self.lon = []
with ecc.GribFile(filename) as grib:
n = len(grib)//2
for j in range(len(grib)):
msg = ecc.GribMessage(grib)
if msg.get('shortName') == code:
if i == 0:
self.lat = msg.get('distinctLatitudes')
self.lon = msg.get('distinctLongitudes')
self.name = msg.get('name')+' / '+msg.get('units')
self.sname = msg.get('shortName')
self.values = np.empty(
(n, len(self.lat), len(self.lon)))
self.values[i,:,:] = np.reshape(msg.get('values'),
(len(self.lat), len(self.lon)))
i = i + 1
return
def parse_grib_file(self):
gribfile = self.source
self.type, self.date, self.hour, self.levtype, grib2 = os.path.basename(
gribfile).split('.')
params = []
levels = []
steps = []
with ecc.GribFile(gribfile) as gf:
nfields = len(gf)
for i in range(len(gf)):
msg = ecc.GribMessage(gf)
params.append(msg['param'])
levels.append(msg['level'])
steps.append(msg['step'])
if i == 1:
self.date = str(msg["dataDate"])
self.hour = "%04d" % int(msg["dataTime"])
if str(msg['suiteName']) == '1':
self.origin = "no-ar-ce"
elif str(msg['suiteName']) == '2':
self.origin = "no-ar-cw"
elif str(msg['suiteName']) == '3':
self.origin = "no-ar-pa"
else:
print("unknown origin/suiteName")
exit(1)
param = list(set(params))
param.sort()
self.param = param
levelist = list(set(levels))
levelist.sort()
levelist.reverse()
self.levelist = levelist
step = list(set(steps))
step.sort()
self.step = step
self.expect = nfields
def plot_all(fnam):
f = ecc.GribFile(fnam)
dirnam = fnam + "_plots"
if os.path.isdir(dirnam):
pass
else:
os.mkdir(dirnam)
for i in range(len(f)):
msg = ecc.GribMessage(f)
nx = msg['Nx']
ny = msg['Ny']
name = msg['parameterName']
val = ma.masked_values(np.flipud(msg['values'].reshape((ny, nx))),
msg['missingValue'])
plt.imshow(val, cmap="jet", interpolation="none")
plt.title("%s [%s] %sH%s" %
(name, msg['units'], msg['date'], msg['time']))
plt.colorbar()
plt.savefig("%s/%s_%s_lvl%s_stp%s.png" %
(dirnam, re.sub("( )+", "_", name), msg['levelType'],
msg['level'], msg['step']))
plt.close()
def read_eccodes(filename, keys=None, **kwargs):
""" Read a GRIB file into DataArray class
Expects: var x date x level x lat x lon
Args:
filename (str):
keys (dict):
Returns:
"""
import os
from time import time
import numpy as np
import xarray as xr
try:
import eccodes
except ImportError:
print("This function requires eccodes from ECMWF installed")
print("eccodes requires python 2, for python 3 use cfgrib")
raise ImportError()
from ..fun import message
if not os.path.isfile(filename):
raise IOError("File not found: %s" % filename)
data = {}
attrs = {}
lat = None
lon = None
start_time = time()
try:
with eccodes.GribFile(filename) as grib:
for msg in grib:
# shortName
ivar = msg.get('shortName')
if lat is None:
# latlon = msg.get('latLonValues') # same as grid Lat, Lon
lat = msg.get('distinctLatitudes') # longitudes
lon = msg.get('distinctLongitudes') # latitudes
# Nj,i
ni = msg.get('Ni') # number of Columns
nj = msg.get('Nj') # number of Rows ?
ilevel = msg.get('level') # vertical level ?
values = msg.get('values') # 1D array
idate = "%04d-%02d-%02dT%02d:%02d:%02d" % (
msg.get('year'), msg.get('month'), msg.get('day'),
msg.get('hour'), msg.get('minute'), msg.get('second'))
if ivar not in data.keys():
data[ivar] = {}
attrs[ivar] = {}
attrs[ivar]['units'] = msg.get('units')
attrs[ivar]['long_name'] = msg.get('name')
if idate not in data[ivar].keys():
data[ivar][idate] = {}
data[ivar][idate][ilevel] = values.reshape(
nj, ni) # how to figure out which comes first ?
message("%s [%s] [%s]" % (ivar, idate, str(ilevel)), **kwargs)
except:
raise
# latlon = latlon.reshape(nj, ni)
# lon = latlon[0, :]
# lat = latlon[:, 0]
# lon = lon.reshape(nj,ni)[0,:] # Lons
# lat = lat.reshape(nj,ni)[:,0] # Lats
for ivar in data.keys():
dates = data[ivar].keys()
levels = data[ivar][dates[0]].keys()
order = ('date', 'level', 'latitude', 'longitude')
tmp = []
dim_attrs = {
'level': {
'units': 'hPa'
},
'longitude': {
'units': 'degrees_east'
},
'latitude': {
'units': 'degrees_north'
}
}
dims = {
'date': np.array([np.datetime64(i) for i in dates]),
'level': np.array(levels).astype(float),
'longitude': lon,
'latitude': lat
}
for idate in dates:
tmp += [np.stack(data[ivar][idate].values())]
message("DataArray ", ivar, **kwargs)
data[ivar] = xr.DataArray(ivar,
np.array(tmp),
order,
dims,
attrs=attrs[ivar],
dim_attrs=dim_attrs,
axes=['T', 'Z', 'Y', 'X'])
data[ivar].sort(verbose=0) # funktioniert das?
message("--- %s seconds ---" % (time() - start_time), **kwargs)
# missing global attributes and history
return data # needs to be an xarray
infile_raw_snow = sys.argv[1]
outfile_snow_depth = sys.argv[2]
print("SWE/SR input file:", infile_raw_snow)
print("SD output file:", outfile_snow_depth)
ikey = 'indicatorOfParameter' # 13, 191
ikey = 'indicatorOfTypeOfLevel' # sfc
ikey = 'level' # 721,722,723
swe_per_level = []
sr_per_level = []
out_message = None
with ecc.GribFile(infile_raw_snow) as gfin_raw_snow:
num_messages = len(gfin_raw_snow)
for _ in range(num_messages):
grib_message = ecc.GribMessage(gfin_raw_snow)
parameter = grib_message['indicatorOfParameter']
level_type = grib_message['indicatorOfTypeOfLevel']
level = grib_message['level']
print(parameter, level_type, level)
if parameter == 13 and level_type == 'sfc' and level == 721:
out_message = ecc.GribMessage(clone=grib_message)
if parameter == 13 and level_type == 'sfc' and level in [
721, 722, 723
]:
print('Found SWE, layer {}'.format(level))
#infile_rad='/scratch/ms/no/fab0/carra_grib/fc_sfc_2014030203_001.grib1'
#infile_alb='/scratch/ms/no/fab0/carra_grib/fc_soil_2014030203_001.grib1alb'
#outfile='/scratch/ms/no/fab0/carra_grib/test.grib1alb'
infile_alb = sys.argv[2]
infile_rad = sys.argv[1]
outfile = sys.argv[3]
print("albedo file:",infile_alb)
print("radiation file:",infile_rad)
print("output file:",outfile)
gfin_alb = ecc.GribFile(infile_alb)
gfin_rad = ecc.GribFile(infile_rad)
ikey = 'indicatorOfParameter'
for i in range(len(gfin_alb)):
msg = ecc.GribMessage(gfin_alb)
if (msg[ikey] == 84):
msg2 = ecc.GribMessage(clone=msg)
break
for i in range(len(gfin_rad)):
msg = ecc.GribMessage(gfin_rad)
if (msg[ikey] == 111):
ssr = msg['values']
elif (msg[ikey] == 117):
def sendGribData(self,
*,
requestHandle: dreg.RequestHandle = None,
userDataReqs=None):
if not self.registerAll_ and (not requestHandle or not userDataReqs):
raise RuntimeError(
"If not all topics are registered, we need to pass a request handle and list of topics"
)
luserDataReqs = userDataReqs
# timestamp: {fieldname: []}
fieldsmetadata = {}
with ecc.GribFile(self.filename_) as grib:
# Warning do not use/print/etc len(grib), for strange reasons it will always return the same msg
for i in range(len(grib)):
msg = ecc.GribMessage(grib)
fieldname = parseGrib.getGribFieldname(
table2Version=msg["table2Version"],
indicatorOfParameter=msg["indicatorOfParameter"],
indicatorOfTypeOfLevel=msg["indicatorOfTypeOfLevel"],
typeOfLevel=msg["typeOfLevel"],
timeRangeIndicator=msg["timeRangeIndicator"])
# fieldname2 = self.getGribFieldname(msg)
if not fieldname:
print(
'WARNING: found a grib field with no match in table : ',
msg['cfVarName'], msg['table2Version'],
msg['indicatorOfParameter'],
msg['indicatorOfTypeOfLevel'])
continue
if fieldname in [x.name
for x in luserDataReqs] or self.registerAll_:
timestamp = self.getTimestamp(msg)
toplevel = msg["topLevel"]
levels = fieldsmetadata.setdefault(timestamp,
{}).setdefault(
fieldname, [])
bisect.insort(levels, toplevel)
with ecc.GribFile(self.filename_) as grib:
# Warning do not use/print/etc len(grib), for strange reasons it will always return the same msg
for i in range(len(grib)):
msg = ecc.GribMessage(grib)
fieldname = parseGrib.getGribFieldname(
table2Version=msg["table2Version"],
indicatorOfParameter=msg["indicatorOfParameter"],
indicatorOfTypeOfLevel=msg["indicatorOfTypeOfLevel"],
typeOfLevel=msg["typeOfLevel"],
timeRangeIndicator=msg["timeRangeIndicator"])
# fieldname2 = self.getGribFieldname(msg)
if not fieldname:
print(
'WARNING: found a grib field with no match in table : ',
msg['cfVarName'], msg['table2Version'],
msg['indicatorOfParameter'],
msg['indicatorOfTypeOfLevel'])
continue
if self.registerAll_:
# Only subscribe if the field was not registered yet
requestHandle = dreg.DataRegistry.subscribeIfNotExists(
self, fieldname)
assert requestHandle
if fieldname in [x.name
for x in luserDataReqs] or self.registerAll_:
timestamp = self.getTimestamp(msg)
levels = fieldsmetadata[timestamp][fieldname]
requestHandle.timestamp_ = timestamp
ni = msg['Ni']
nj = msg['Nj']
lord = 'F'
if not msg['jPointsAreConsecutive'] == 0:
lord = 'C'
arr = np.reshape(ecc.codes_get_values(msg.gid), (ni, nj),
order='F').astype(np.float32)
lev = msg["topLevel"]
level_index = levels.index(lev)
msgkey = data.MsgKey(1, fieldname, 1, 0,
timestamp, 0, 0, level_index, ni, nj,
len(levels), ni, nj,
msg["longitudeOfFirstGridPoint"],
msg["longitudeOfLastGridPoint"],
msg["latitudeOfFirstGridPoint"],
msg["latitudeOfLastGridPoint"])
self.insertDataPatch(
requestHandle, fieldname,
fieldop.SinglePatch(0, 0, ni, nj, level_index, arr),
msgkey)