def read_ECMWF(date):
""" Script reading the ECMWF data.
Not a generator as this is synchronized with the 1h part slice.
The data are assumed valid over the 1h period that follows the timestamp.
This is quite OK for UDR as this quantity is defined as a mean/accumuation over
this one-hour period.
Cloud-cover is from analysis, therefore as an instantaneous map, but varies less rapidly
than the UDR. """
dat = ECMWF('STC',date)
dat._get_var('T')
dat.attr['dlo'] = (dat.attr['lons'][-1] - dat.attr['lons'][0]) / (dat.nlon-1)
dat.attr['dla'] = (dat.attr['lats'][-1] - dat.attr['lats'][0]) / (dat.nlat-1)
if dat.attr['Lo1']>dat.attr['Lo2']:
dat.attr['Lo1'] = dat.attr['Lo1']-360.
#@@ test
# print('LoLa ',dat.attr['Lo1'],dat.attr['La1'],dat.attr['dlo'],dat.attr['dla'],\
# dat.attr['levs'][0],len(dat.attr['levs']))
#@@ end test
dat._get_var('UDR')
#dat._get_var('CC')
dat._mkp()
dat._mkrho()
dat.var['UDR'] /= dat.var['RHO']
dat.close()
return dat
def read_ERA5(t0, dtRange, pre=False, vshift=0):
""" Generator reading the ERA5 data.
The loop is infinite; ERA5 data are called when required until the end of
the parcel loop.
The output contain a function that gives the pressure of the tropo^pause as
a function of lon and lat in the FullAMA domain"""
# initial time for internal loop
current_time = t0
while True:
try:
if (current_time in blacklist): raise BlacklistError()
# defining a new object is necessary to avoid messing dat if an error occurs
dat = ECMWF('FULL-EA', current_time)
dat._get_T()
dat._mkp()
dats = dat.shift2west(-179)
# extraction in a domain that encompasses FullAMA
datr0 = dats.extract(latRange=[-5, 55],
lonRange=[-15, 165],
varss=['P', 'T'])
del dat, dats
datr0._WMO()
datr0.fP = RegularGridInterpolator(
(np.arange(-5, 56), np.arange(-15, 166)),
datr0.d2d['pwmo'],
bounds_error=True)
print('get ERA5 tropopause for ', current_time)
try:
del datr
except:
pass # intended for the first pass when datr undefined
datr = datr0 # datr as a view of datr0
# We reproduce here the same sequence as for the satellite file
# although perhapsnot appropriate (the ERA5 data are instantaneous)
if pre:
datr.tf = current_time + dtRange
datr.ti = current_time
else:
datr.tf = current_time
datr.ti = current_time - dtRange
except BlacklistError:
print('blacklisted date for GridSat', current_time)
# extend the lease while keeping the old dat
datr.ti -= dtRange
except FileNotFoundError:
print('ERA5 file not found ', current_time)
# extend the lease while keeping the old dat
datr.ti -= dtRange
current_time -= dtRange
yield datr
def Pinterpol(date):
""" Read the ERA5 data and interpolate to the pressure levels. """
dat = ECMWF('FULL-EA', date, exp=['VOZ', 'QN'])
dat._get_T()
dat._get_var('O3')
dat._get_var('Q')
dat.close()
dat._mkp()
dat._mkz()
dat2 = dat.shift2west(-20)
dat3 = dat2.extract(lonRange=[-10, 160], latRange=[0, 50], varss='All')
dat3._WMO()
dat4 = dat3.interpolP(pressPa, varList=['Z', 'T', 'Q', 'O3'])
dat4.d2d = dat3.d2d
return (dat4)
i = len(dats)
print('dats length ', len(dats))
date = day1
pts = list(np.arange(420, 610, 5))
varList = ['T', 'VO', 'O3', 'PV', 'Z', 'U', 'V']
while date < day2:
print(date)
dat = ECMWF('FULL-EA', date, exp='VOZ')
dat._get_var('T')
dat._get_var('VO')
dat._get_var('O3')
dat._get_var('U')
dat._get_var('V')
dat._mkp()
dat._mkz()
dat._mkthet()
dat._mkpv()
if date >= datetime(2017, 10, 12, 0):
dats[i] = dat.interpolPT(pts,
varList=varList,
latRange=(5, 35),
lonRange=(100, 220))
elif date >= datetime(2017, 10, 3, 0):
datr = dat.shift2west(-180)
dats[i] = datr.interpolPT(pts,
varList=varList,
latRange=(20, 50),
lonRange=(-180, -60))
elif date >= datetime(2017, 9, 23, 0):
if args.month1 is not None: month1 = args.month1
if args.month2 is not None: month2 = args.month2
if args.day1 is not None: day1 = args.day1
if args.day2 is not None: day2 = args.day2
if args.hour1 is not None: hour1 = args.hour1
if args.hour2 is not None: hour2 = args.hour2
date = datetime(year, month1, day1, hour1)
while date < datetime(year, month2, day2, hour2):
print('processing ', date)
outfile = date.strftime('TPP%y%m%d%H.hdf5')
fullname = os.path.join(maindir, date.strftime('%Y/%m'), outfile)
fdd = ECMWF('FULL-EA', date)
fdd._get_T()
fdd._mkp()
fdd.close()
fde = fdd.shift2west(-20)
fdf = fde.extract(lonRange=[-10, 160], latRange=[0, 50], varss='All')
fdf._CPT()
fdf._WMO()
tpp = {}
tpp['Twmo'] = fdf.d2d['Twmo']
tpp['pwmo'] = fdf.d2d['pwmo']
tpp['Tcold'] = fdf.d2d['Tcold']
tpp['pcold'] = fdf.d2d['pcold']
tpp['nlon'] = fdf.nlon
tpp['nlat'] = fdf.nlat
tpp['lats'] = fdf.attr['lats']
tpp['lons'] = fdf.attr['lons']
tpp['date'] = fdd.date
part0['y'] = np.empty(0, dtype=float)
part0['t'] = np.empty(0, dtype=float)
part0['p'] = np.empty(0, dtype=float)
part0['idx_back'] = np.empty(0, dtype=int)
# It is assumed that both date_end and date_beg are valid dates for the EN files
date_f = date_end
date_p = date_f - ENint
date_current = date_end
# first read and first interpolation
#data = read_ECMWF(ENdir,date_f)
# data is read from ERA5 data in the global domain at 1° horizontal resolution
# and longitude origin at 0°
data = ECMWF('FULL-EA', date_f)
data._get_var('T')
data._mkp(
) # Calculate pressure from surface pressure and hybrid coefficients
data._mkthet() # Calculate the potential temperature
# ACHTUNG: TO DO : ADD HERE A SHIFT IN LONGITUDE
# data -> data.shit2west(-179)
if not quiet:
print('load first EN file ', date_f)
T_p, P_p = interp3d_thet(
data, theta_level) # First interpolation to the theta level
delta_pf = ENint.total_seconds()
numpart = 0
# loop on the EN time
while date_p >= date_beg:
T_f = T_p.copy()
P_f = P_p.copy()
if not quiet:
fullname = os.path.join(SAF_dir,
current_date.strftime('%Y/%m/%Y-%m-%d'),
FAFfile)
# First try to get the SAFNWC file
try:
with gzip.open(fullname, 'rb') as file:
[ptop, cloud_flag] = pickle.load(file)
except IOError:
print('date', current_date, ' file not found ', FAFfile)
print(current_date, 'is skipped')
current_date += timedelta(hours=1)
continue
# The get the ECMWF data
data = ECMWF('STC', current_date)
data._get_T()
data._mkp()
data.close()
# find time
ir_start = int((current_date - date_beg).total_seconds())
# process the cloud tops above cut_level
ptop_temp = np.ma.masked_greater(ptop, cut_level)
ct = (cloud_flag >> 24) & 0xFF
if cloud_type == 'high':
filt = (ct == 8) | (ct == 9) | ((ct >= 11) & (ct <= 14))
elif cloud_type == 'meanhigh':
filt = (ct == 8) | (ct == 9) | (ct == 12) | (ct == 13)
elif cloud_type == 'veryhigh':
filt = (ct == 9) | (ct == 13)
ptop_temp[~filt] = np.ma.masked
# apply same mask to other fields
x_temp = np.ma.array(data=xg, mask=ptop_temp.mask)
part0['x'] = np.empty(0, dtype=float)
part0['y'] = np.empty(0, dtype=float)
part0['t'] = np.empty(0, dtype=float)
part0['p'] = np.empty(0, dtype=float)
part0['idx_back'] = np.empty(0, dtype=int)
# It is assumed that both date_end and date_beg are valid dates for the EN files
# The scan starts from date_end and proceeds to date_beg backward in time
date_f = date_end
date_p = date_f - ENint
date_current = date_end
# first read and first interpolation
#data = read_ECMWF(ENdir,date_f)
data = ECMWF('FULL-EI', date_f)
data._get_var('T')
data._mkp() #This package was defined already in ECMWF_N.py file
data._mkthet() #This package was defined already in ECMWF_N.py file
data.close()
if not quiet:
print('load first EN file ', date_f)
T_p, P_p = interp3d_thet(data, theta_level)
delta_pf = ENint.total_seconds()
numpart = 0
# loop on the EN intervals
while date_p >= date_beg:
T_f = T_p.copy()
P_f = P_p.copy()
if not quiet:
print('load new EN file ', date_p)
data = ECMWF('FULL-EI', date_p)
date1 = datetime(mean_date.year,mean_date.month,mean_date.day,mean_date.hour - mean_date.hour%3)
date2 = date1 + timedelta(hours=3)
dtt = (mean_date-date1).total_seconds()/10800
# test whether the selected part of the orbit is totally in the interval
enter_date = ref_date + timedelta(seconds=data['time'][0])
exit_date = ref_date + timedelta(seconds=data['time'][-1])
if enter_date < date1:
print('beginning of orbit out by',date1-enter_date,dtt)
if exit_date > date2:
print('beginning of orbit out by',exit_date-date2,dtt)
# get ECMWF data for the orbit and generate a curtain over the retained segment
# read data for first bracketting date
dat1 = ECMWF('FULL-EI',date1)
dat1._get_T()
dat1._mkp()
dat1._mkz()
dat1.close()
# generate a curtain along the track for first date
sect1 = dat1.interpol_orbit(data['longitude'],data['latitude'],varList=['P','T','Z'])
del dat1
# read data for second bracketting date
dat2 = ECMWF('FULL-EI',date2)
dat2._get_T()
dat2._mkp()
dat2._mkz()
dat2.close()
# generate a curtain along the track for scond date
sect2 = dat2.interpol_orbit(data['longitude'],data['latitude'],varList=['P','T','Z'])
del dat2
sect = curtain()
part0['flag'] = np.empty(0,dtype=int)
part0['ir_start'] = np.empty(0,dtype=int)
part0['x'] = np.empty(0,dtype=float)
part0['y'] = np.empty(0,dtype=float)
part0['t'] = np.empty(0,dtype=float)
part0['p'] = np.empty(0,dtype=float)
part0['idx_back'] = np.empty(0,dtype=int)
numpart = 0
# Loop on the values of theta
for targetTheta in [theta-dTheta, theta, theta+dTheta]:
# get the first time
date1 = date + timedelta(hours=int(begSeq/3600))
predat = ECMWF('STC',date1)
predat._get_T()
predat._mkp()
predat._mkthet()
(fT1,fP1) = interp3d_thet(predat,targetTheta)
predat.close()
#for n in range(nInt):
# finds the boundary
id1 = np.where(utc == begSeq)[0][0]
id2 = np.where(utc == endSeq)[0][0]
# get lat, lon and release time
yy = data.var['Lat'][id1:id2+1]
xx = data.var['Long'][id1:id2+1]
ir_start = utc[id1:id2+1] - 86400
ir_start = ir_start.astype(np.int)
#weigthing factor
w2 = (utc[id1:id2+1] % 3600) / 3600