def resample_msg(h5files,
gribfiles,
outdir='./'): #,accstep=1,acc = True, start = 0):
'''Finds SAF MSG data in a hdf5 file,
resamples it to fc grid and puts it in a grib message '''
if h5files == [] or gribfiles == []:
print 'resample_msg: Files not found'
return 0
import find_date, gribtools, config
# make dir to output files
config.mkdir_p(outdir)
# get necessary lats,lons ONCE:
glats, glons = gribtools.get_latlon(gribfiles[0])
rlats, rlons = init_MSG_h5data(h5files[0])
def tofile(gfile, date, data, param=71, level=0):
'''take grib file and hdf5 data, resample and insert into new grib file'''
resampled_data = resample_radar(data, rlats, rlons, glats, glons)
dtg = dt.datetime.strftime(date, '%Y%m%d%H')
newfile = os.path.join(outdir, 'msg_' + str(dtg) + '+000.grib')
gribtools.insert_grib(gfile,
newfile,
resampled_data,
parameter=param,
level=level,
force=True)
#gribtools.insert_grib(gfile, newfile, resampled_data,parameter = 0, level = level)
return newfile
msgfiles = set() #[]
for gfile in gribfiles:
alreadythere = 0 # check below if accumulated observations have been made already
dtg, ltg = find_date.datetest(gfile, pattern='fc')
gdate = dtg + ltg
print 'In resample_msg', os.path.split(gfile)[1], dtg, ltg, gdate
#check if corresponding file already exists!
resampled_files = glob.glob(os.path.join(outdir, 'msg*grib'))
for rf in resampled_files:
dtr, ltr = find_date.datetest(rf, pattern='fc')
if dtr == None:
print 'not a valid filename ', os.path.split(rf)[1]
alreadythere = 1
break
elif dtr + ltr == gdate:
print 'MSG file available', rf
alreadythere = 1
#radfiles.append(rf)
msgfiles.add(rf)
break
if not alreadythere: # do matching, resampling, write to file
for afile in h5files:
hdate = find_date.datetest(afile, pattern='hdf')[0]
if hdate == gdate: # unlike radar, just take instantaneous cloud mask
wa = get_h5data(afile, h5source={'model': 'MSG'})
# sometimes file is corrupt or something
if wa == None: break
msgfile = tofile(gfile, gdate, wa)
print msgfile
msgfiles.add(msgfile)
return msgfiles
def accumulate_radar(h5files,
gribfiles,
outdir='./',
accstep=1,
acc=True,
start=0):
'''Finds reflectivity(!) data in a hdf5 file,
resamples it to fc grid and puts it in a grib message '''
if h5files == [] or gribfiles == []:
print 'Accumulate radar: Files not found'
return 0
import find_date, gribtools, config
config.mkdir_p(outdir)
glats, glons = gribtools.get_latlon(gribfiles[0])
rlats, rlons = init_h5data(h5files[0])
def tofile(gfile, date, data, step, param=61, level=457):
resampled_data = resample_radar(data, rlats, rlons, glats, glons)
#print 'now resampled: ',resampled_data.max(), resampled_data.mean(), resampled_data.min()
dtg = dt.datetime.strftime(date, '%Y%m%d%H')
newfile = os.path.join(
outdir, 'rad_' + str(int(step)) + '_' + str(dtg) + '+000.grib')
gribtools.insert_grib(gfile,
newfile,
resampled_data,
parameter=param,
level=level)
#gribtools.insert_grib(gfile, newfile, resampled_data,parameter = 0, level = level)
return newfile
def accumulate(gfile, h5files):
w = None #?
for afile in h5files:
hdate = find_date.datetest(afile, pattern='hdf')[0]
print 'files:', os.path.split(gfile)[1], os.path.split(afile)[1]
print gdate, hdate, acctime.seconds / 3.6e3, gdate - acctime < hdate <= gdate
if gdate - acctime < hdate <= gdate: # obs time between fctime and an accumulation time before
wa = get_h5data(afile)
if acc:
try:
w = w + wa
nr += 1
except: # first
w = wa
nr = 1
print 'now: nr, max, mean, min: ', nr, w.max(), w.mean(
), w.min()
else: # for the RACfiles: 3h accumulation every hour available!
#print 'now tot: ',wa.max(), wa.min()
if gdate - dt.timedelta(hours=1) < hdate <= gdate:
accfile = tofile(gfile, gdate, wa,
acctime.seconds / 3.6e3)
#radfiles.append( accfile )
return accfile
else:
pass
elif hdate > gdate or hdate <= gdate - acctime: # time to write to file
if acc:
#try:
if w not in [
[], None
]: # check if data to write to file is not None
accfile = tofile(gfile, gdate, w / nr,
acctime.seconds / 3.6e3)
#reset: (obsolete?)
w = []
nr = 0
#radfiles.append( accfile )
return accfile
else: #except:
pass
#pass
else: # do nothing
pass
else: # date strange
print 'strange, obdate, fcdate: ', hdate, gdate
wa = get_h5data(afile)
accfile = tofile(gfile, gdate, wa, acctime.seconds / 3.6e3)
return accfile
#last bit, check...
if acc:
try:
if w not in [[], None]:
print 'eh ', afile, gfile, hdate, gdate
accfile = tofile(gfile, gdate, w / nr,
int(acctime.seconds / 3.6e3))
return accfile
except:
print 'Not in accumulated file: ', gfile
print('strange, obdate, fcdate: ', hdate, gdate)
return None #accfile #(Done?)
acctime = dt.timedelta(hours=accstep)
dtg0, lt0 = find_date.datetest(gribfiles[0], pattern='fc')
dtg1, lt1 = find_date.datetest(gribfiles[1], pattern='fc')
deltime = (dtg1 + lt1) - (dtg0 + lt0)
if deltime > acctime: acctime = deltime
#sdate = find_date.datetest(h5files[0],pattern='hdf')[0] # start at first file
#sdate = dt.datetime(1900,1,1,12,0) # arbitrary date in far history
radfiles = set() #[]
for gfile in sorted(gribfiles):
alreadythere = 0 # check below if accumulated observations have been made already
dtg, ltg = find_date.datetest(gfile, pattern='fc')
gdate = dtg + ltg
#print 'In accumulate radar',os.path.split(gfile)[1], dtg,ltg,gdate
#check if corresponding file already exists!
resampled_files = glob.glob(os.path.join(outdir, 'rad*grib'))
for rf in sorted(resampled_files):
dtr, ltr = find_date.datetest(rf, pattern='fc')
if dtr == None:
print 'not a valid filename ', os.path.split(rf)[1]
alreadythere = 1
break
elif dtr + ltr == gdate:
#print 'radar file available'
alreadythere = 1
radfiles.add(rf)
break
if not alreadythere:
accfile = accumulate(gfile, h5files)
print accfile
radfiles.add(accfile)
return radfiles
def accumulate(gfile, h5files):
w = None #?
for afile in h5files:
hdate = find_date.datetest(afile, pattern='hdf')[0]
print 'files:', os.path.split(gfile)[1], os.path.split(afile)[1]
print gdate, hdate, acctime.seconds / 3.6e3, gdate - acctime < hdate <= gdate
if gdate - acctime < hdate <= gdate: # obs time between fctime and an accumulation time before
wa = get_h5data(afile)
if acc:
try:
w = w + wa
nr += 1
except: # first
w = wa
nr = 1
print 'now: nr, max, mean, min: ', nr, w.max(), w.mean(
), w.min()
else: # for the RACfiles: 3h accumulation every hour available!
#print 'now tot: ',wa.max(), wa.min()
if gdate - dt.timedelta(hours=1) < hdate <= gdate:
accfile = tofile(gfile, gdate, wa,
acctime.seconds / 3.6e3)
#radfiles.append( accfile )
return accfile
else:
pass
elif hdate > gdate or hdate <= gdate - acctime: # time to write to file
if acc:
#try:
if w not in [
[], None
]: # check if data to write to file is not None
accfile = tofile(gfile, gdate, w / nr,
acctime.seconds / 3.6e3)
#reset: (obsolete?)
w = []
nr = 0
#radfiles.append( accfile )
return accfile
else: #except:
pass
#pass
else: # do nothing
pass
else: # date strange
print 'strange, obdate, fcdate: ', hdate, gdate
wa = get_h5data(afile)
accfile = tofile(gfile, gdate, wa, acctime.seconds / 3.6e3)
return accfile
#last bit, check...
if acc:
try:
if w not in [[], None]:
print 'eh ', afile, gfile, hdate, gdate
accfile = tofile(gfile, gdate, w / nr,
int(acctime.seconds / 3.6e3))
return accfile
except:
print 'Not in accumulated file: ', gfile
print('strange, obdate, fcdate: ', hdate, gdate)
return None #accfile #(Done?)
h5files = glob.glob('./RAD_NL23*h5')
print 'h5: ', h5files
gfiles = glob.glob('./acc20120307*grib')
print 'grib:', gfiles
radfiles = accumulate_radar(sorted(h5files),
sorted(gfiles),
outdir='./test',
acc=False,
accstep=3)
print radfiles
sys.exit(1)
h5file = './RAD_NL23_PCP_NA_201203071400.h5'
h5data = get_h5data(h5file)
print 'retrieve: in ', find_date.datetest(
h5file, pattern='hdf'), '\n max,min:', h5data.max(), h5data.min()
rlats, rlons = init_h5data(h5file)
grib_file = 'fc2012030700+014.grb'
grbs = pygrib.open(grib_file)
hlats, hlons = grbs[1].latlons()
grbs.close()
newdata = resample_radar(h5data, rlats, rlons, hlats, hlons)
print 'resample: in ', find_date.datetest(
grib_file, pattern='fc'), '\n max,min:', newdata.max(), newdata.min()
h5files = glob.glob('./RAD*h5')
gfiles = glob.glob('./fc*grb')
radfiles = accumulate_radar(sorted(h5files),
sorted(gfiles),