# c4glfiles = dict([(EXP,odirexperiments[EXP]+'/'+format(STN['ID'],'05d')+'.yaml') \
# for EXP in experiments.keys()])
dict_diag_station = {}
df_logic_morning = pd.DataFrame()
df_logic_afternoon = pd.DataFrame()
with open(fnout,'w') as fileout, \
open(fnout_afternoon,'w') as fileout_afternoon:
wy_strm = wyoming(PATH=args.path_input, STNM=STN.name)
wy_strm.set_STNM(int(STN.name))
# we consider all soundings from 1981 onwards
wy_strm.find_first(year=int(args.startyear))
#wy_strm.find(dt.datetime(2004,10,19,6))
c4gli = class4gl_input(debug_level=logging.INFO)
c4gli_afternoon = class4gl_input(debug_level=logging.INFO)
# so we continue as long as we can find a new sounding
while wy_strm.current is not None:
c4gli.clear()
c4gli.get_profile_wyoming(wy_strm)
#import pdb;pdb.set_trace()
#print(STN['ID'],c4gli.pars.datetime)
#c4gli.get_global_input(globaldata)
print(c4gli.pars.STNID, c4gli.pars.ldatetime)
logic = dict()
logic['morning'] = (c4gli.pars.ldatetime.hour <= 12.)
def bllast_parser(balloon_file, file_sounding, ldate, hour, c4gli=None):
#balloon_conv = replace_iter(balloon_file,"°","deg")
#readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
#air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
air_balloon_in = pd.read_csv(
balloon_file,
delimiter='\t',
)
#widths=[14]*19,
#skiprows=9,
#skipfooter=15,
#decimal='.',
#header=None,
#names = columns,
#na_values='-----')
air_balloon_in = air_balloon_in.rename(columns=lambda x: x.strip())
print(air_balloon_in.columns)
rowmatches = {
't':
lambda x: x['TaRad'] + 273.15,
#'tv': lambda x: x['Virt. Temp[C]']+273.15,
'p':
lambda x: x['Press'] * 100.,
'u':
lambda x: x['VHor'] * np.sin((90. - x['VDir']) / 180. * np.pi),
'v':
lambda x: x['VHor'] * np.cos((90. - x['VDir']) / 180. * np.pi),
'z':
lambda x: x['Altitude'] - 582.,
# from virtual temperature to absolute humidity
'q':
lambda x: qfrom_e_p(efrom_rh100_T(x['UCal'], x['TaRad'] + 273.15), x[
'Press'] * 100.),
}
air_balloon = pd.DataFrame()
for varname, lfunction in rowmatches.items():
air_balloon[varname] = lfunction(air_balloon_in)
rowmatches = {
'R': lambda x: (Rd * (1. - x.q) + Rv * x.q),
'theta': lambda x: (x['t']) * (x['p'][0] / x['p'])**(x['R'] / cp),
'thetav': lambda x: x.theta + 0.61 * x.theta * x.q
}
for varname, lfunction in rowmatches.items():
air_balloon[varname] = lfunction(air_balloon)
dpars = {}
dpars['longitude'] = current_station['longitude']
dpars['latitude'] = current_station['latitude']
dpars['STNID'] = current_station.name
is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
valid_indices = air_balloon.index[is_valid].values
i = 1
while (air_balloon.thetav.iloc[valid_indices[0]] - \
air_balloon.thetav.iloc[valid_indices[i]] ) > 0.5:
#diff = (air_balloon.theta.iloc[valid_indices[i]] -air_balloon.theta.iloc[valid_indices[i+1]])- 0.5
air_balloon.thetav.iloc[valid_indices[0:i]] = \
air_balloon.thetav.iloc[valid_indices[i]] + 0.5
i += 1
air_ap_mode = 'b'
if len(valid_indices) > 0:
dpars['h'],dpars['h_u'],dpars['h_l'] =\
blh(air_balloon.z,air_balloon.thetav,air_balloon_in['VHor'])
dpars['h_b'] = np.max((dpars['h'], 10.))
dpars['h_u'] = np.max(
(dpars['h_u'], 10.)) #upper limit of mixed layer height
dpars['h_l'] = np.max(
(dpars['h_l'], 10.)) #low limit of mixed layer height
dpars['h_e'] = np.abs(dpars['h_u'] -
dpars['h_l']) # error of mixed-layer height
dpars['h'] = np.round(dpars['h_' + air_ap_mode], 1)
else:
dpars['h_u'] = np.nan
dpars['h_l'] = np.nan
dpars['h_e'] = np.nan
dpars['h'] = np.nan
if ~np.isnan(dpars['h']):
dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
else:
dpars['Ps'] = np.nan
if ~np.isnan(dpars['h']):
# determine mixed-layer properties (moisture, potential temperature...) from profile
# ... and those of the mixed layer
is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h'])
valid_indices_below_h = air_balloon.iloc[valid_indices].index[
is_valid_below_h].values
if len(valid_indices) > 1:
if len(valid_indices_below_h) >= 3.:
ml_mean = air_balloon.iloc[valid_indices][
is_valid_below_h].mean()
else:
ml_mean = air_balloon.iloc[
valid_indices[0]:valid_indices[1]].mean()
elif len(valid_indices) == 1:
ml_mean = (air_balloon.iloc[0:1]).mean()
else:
temp = pd.DataFrame(air_balloon)
temp.iloc[0] = np.nan
ml_mean = temp
dpars['theta'] = ml_mean.theta
dpars['q'] = ml_mean.q
dpars['u'] = ml_mean.u
dpars['v'] = ml_mean.v
else:
dpars['theta'] = np.nan
dpars['q'] = np.nan
dpars['u'] = np.nan
dpars['v'] = np.nan
air_ap_head = air_balloon[0:
0] #pd.DataFrame(columns = air_balloon.columns)
# All other data points above the mixed-layer fit
air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
air_ap_head.z = pd.Series(np.array([2., dpars['h'], dpars['h']]))
jump = air_ap_head.iloc[0] * np.nan
if air_ap_tail.shape[0] > 1:
# we originally used THTA, but that has another definition than the
# variable theta that we need which should be the temperature that
# one would have if brought to surface (NOT reference) pressure.
for column in ['theta', 'q', 'u', 'v']:
# initialize the profile head with the mixed-layer values
air_ap_head[column] = ml_mean[column]
# calculate jump values at mixed-layer height, which will be
# added to the third datapoint of the profile head
jump[column] = (air_ap_tail[column].iloc[1]\
-\
air_ap_tail[column].iloc[0])\
/\
(air_ap_tail.z.iloc[1]\
- air_ap_tail.z.iloc[0])\
*\
(dpars['h']- air_ap_tail.z.iloc[0])\
+\
air_ap_tail[column].iloc[0]\
-\
ml_mean[column]
if column == 'theta':
# for potential temperature, we need to set a lower limit to
# avoid the model to crash
jump.theta = np.max((0.1, jump.theta))
air_ap_head[column][2] += jump[column]
air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 + air_ap_head.v**2)
# filter data so that potential temperature always increases with
# height
cols = []
for column in air_ap_tail.columns:
#if column != 'z':
cols.append(column)
# only select samples monotonically increasing with height
air_ap_tail_orig = pd.DataFrame(air_ap_tail)
air_ap_tail = pd.DataFrame()
air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],
ignore_index=True)
for ibottom in range(1, len(air_ap_tail_orig)):
if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z + 10.:
air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],
ignore_index=True)
# make theta increase strong enough to avoid numerical
# instability
air_ap_tail_orig = pd.DataFrame(air_ap_tail)
air_ap_tail = pd.DataFrame()
#air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],
ignore_index=True)
theta_low = air_ap_head['theta'].iloc[2]
z_low = air_ap_head['z'].iloc[2]
ibottom = 0
for itop in range(0, len(air_ap_tail_orig)):
theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop + 1)].mean()
z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop + 1)].mean()
if (
#(z_mean > z_low) and \
(z_mean > (z_low+10.)) and \
#(theta_mean > (theta_low+0.2) ) and \
#(theta_mean > (theta_low+0.2) ) and \
(((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)):
air_ap_tail = air_ap_tail.append(
air_ap_tail_orig.iloc[ibottom:(itop + 1)].mean(),
ignore_index=True)
ibottom = itop + 1
theta_low = air_ap_tail.theta.iloc[-1]
z_low = air_ap_tail.z.iloc[-1]
# elif (itop > len(air_ap_tail_orig)-10):
# air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
air_ap = \
pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
# # make theta increase strong enough to avoid numerical
# # instability
# air_ap_tail_orig = pd.DataFrame(air_ap_tail)
# air_ap_tail = pd.DataFrame()
# #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
# air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
# theta_low = air_ap_head['theta'].iloc[2]
# z_low = air_ap_head['z'].iloc[2]
# ibottom = 0
# for itop in range(0,len(air_ap_tail_orig)):
# theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
# z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
# if ((theta_mean > (theta_low+0.2) ) and \
# (((theta_mean - theta_low)/(z_mean - z_low)) > 0.001)):
# air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
# ibottom = itop+1
# theta_low = air_ap_tail.theta.iloc[-1]
# z_low = air_ap_tail.z.iloc[-1]
# # elif (itop > len(air_ap_tail_orig)-10):
# # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
# air_ap = \
# pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
#
# # we copy the pressure at ground level from balloon sounding. The
# # pressure at mixed-layer height will be determined internally by class
rho = 1.2 # density of air [kg m-3]
g = 9.81 # gravity acceleration [m s-2]
air_ap['p'].iloc[0] = dpars['Ps']
air_ap['p'].iloc[1] = (dpars['Ps'] - rho * g * dpars['h'])
air_ap['p'].iloc[2] = (dpars['Ps'] - rho * g * dpars['h'] - 0.1)
dpars['lat'] = dpars['latitude']
# this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
dpars['lon'] = 0.
# this is the real longitude that will be used to extract ground data
dpars['ldatetime'] = ldate + dt.timedelta(hours=hour)
dpars['datetime'] = ldate + dt.timedelta(hours=hour)
dpars['doy'] = dpars['datetime'].timetuple().tm_yday
dpars['SolarAltitude'] = \
Pysolar.GetAltitude(\
dpars['latitude'],\
dpars['longitude'],\
dpars['datetime']\
)
dpars['SolarAzimuth'] = Pysolar.GetAzimuth(\
dpars['latitude'],\
dpars['longitude'],\
dpars['datetime']\
)
dpars['lSunrise'], dpars['lSunset'] \
= Pysolar.util.GetSunriseSunset(dpars['latitude'],
0.,
dpars['ldatetime'],0.)
# Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually
# write local solar time, we need to assign the timezone to UTC (which
# is WRONG!!!). Otherwise ruby cannot understand it (it always converts
# tolocal computer time :( ).
dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
# This is the nearest datetime when the sun is up (for class)
dpars['ldatetime_daylight'] = \
np.min(\
(np.max(\
(dpars['ldatetime'],\
dpars['lSunrise']+dt.timedelta(hours=2))\
),\
dpars['lSunset']\
)\
)
# apply the same time shift for UTC datetime
dpars['datetime_daylight'] = dpars['datetime'] \
+\
(dpars['ldatetime_daylight']\
-\
dpars['ldatetime'])
print('ldatetime_daylight', dpars['ldatetime_daylight'])
print('ldatetime', dpars['ldatetime'])
print('lSunrise', dpars['lSunrise'])
dpars['day'] = dpars['ldatetime'].day
# We set the starting time to the local sun time, since the model
# thinks we are always at the meridian (lon=0). This way the solar
# radiation is calculated correctly.
dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+ \
dpars['ldatetime_daylight'].minute/60.\
+ \
dpars['ldatetime_daylight'].second/3600.
print('tstart', dpars['tstart'])
dpars['sw_lit'] = False
# convert numpy types to native python data types. This provides
# cleaner data IO with yaml:
for key, value in dpars.items():
if type(value).__module__ == 'numpy':
dpars[key] = dpars[key].item()
decimals = {
'p': 0,
't': 2,
'theta': 4,
'z': 2,
'q': 5,
'u': 4,
'v': 4
}
#
for column, decimal in decimals.items():
air_balloon[column] = air_balloon[column].round(decimal)
air_ap[column] = air_ap[column].round(decimal)
updateglobal = False
if c4gli is None:
c4gli = class4gl_input()
updateglobal = True
print('updating...')
print(column)
c4gli.update(source='bllast',\
# pars=pars,
pars=dpars,\
air_balloon=air_balloon,\
air_ap=air_ap)
if updateglobal:
c4gli.get_global_input(globaldata)
# if profile_ini:
# c4gli.runtime = 10 * 3600
c4gli.dump(file_sounding)
# if profile_ini:
# c4gl = class4gl(c4gli)
# c4gl.run()
# c4gl.dump(file_model,\
# include_input=True,\
# timeseries_only=timeseries_only)
#
# # This will cash the observations and model tables per station for
# # the interface
#
# if profile_ini:
# profile_ini=False
# else:
# profile_ini=True
return c4gli
#run_station_chunk = int(args.global_chunk_number)
# ===============================
print('start looping over chunk')
# ===============================
os.system('mkdir -p '+args.path_experiments)
fn_ini = args.path_experiments+'/'+format(run_station.name,'05d')+'_'+\
str(int(run_station_chunk))+'_'+args.subset_experiments+'.yaml'
file_ini = open(fn_ini,'w');print('Writing to: ',fn_ini)
for iDT,DT in enumerate(DTS_chunk):
print(iDT,DT)
c4gli = class4gl_input(debug_level=logging.INFO)
c4gli.update(source='STNID'+format(STNID,'05d'),\
pars=dict(latitude = float(run_station.latitude), \
longitude = float(run_station.longitude),\
lat = float(run_station.latitude), \
# Note the difference between longitude and lon. The
# lon variable should always be zero because we are
# always working in solar time for running CLASS
lon = 0.,\
STNID = int(STNID)))
lSunrise, lSunset = GetSunriseSunset(c4gli.pars.latitude,0.,DT)
#start simulation at sunrise and stop at one hour before sunset
runtime = (lSunset - lSunrise).total_seconds() - 3600.*1.
ldatetime = lSunrise
def humppa_parser(balloon_file, file_sounding, ldate, lhour, c4gli=None):
print(balloon_file)
xrin = balloon_file
air_balloon = pd.DataFrame()
air_balloon['t'] = xrin.tdry.values + 273.15
air_balloon['p'] = xrin.pres.values * 100.
air_balloon['u'] = xrin.u_wind.values
air_balloon['v'] = xrin.v_wind.values
air_balloon['WSPD'] = xrin['wspd'].values
print(xrin.rh.values.shape)
air_balloon['q'] = qfrom_e_p(
efrom_rh100_T(xrin.rh.values, air_balloon['t'].values),
air_balloon.p.values)
#balloon_conv = replace_iter(balloon_file,"°","deg")
#readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()]
#air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15)
# air_balloon_in = pd.read_fwf(balloon_file,
# widths=[14]*19,
# skiprows=9,
# skipfooter=15,
# decimal=',',
# header=None,
# names = columns,
# na_values='-----')
print(air_balloon['p'][0])
varcalc = {
'R': lambda x: (Rd * (1. - x.q) + Rv * x.q),
'pp': lambda x: (x['p'][0] / x['p'])**(x['R'] / cp),
'theta': lambda x: (x['t']) * (x['p'][0] / x['p'])**(x['R'] / cp),
'thetav': lambda x: x.theta + 0.61 * x.theta * x.q,
'rho': lambda x: x.p / x.t / x.R,
}
for varname, lfunction in varcalc.items():
air_balloon[varname] = lfunction(air_balloon)
print('alt in xrin?:', 'alt' in xrin)
if 'alt' in xrin:
air_balloon['z'] = xrin.alt.values
else:
g = 9.81 # gravity acceleration [m s-2]
air_balloon['z'] = 0.
for irow, row in air_balloon.iloc[1:].iterrows():
air_balloon['z'].iloc[irow] = air_balloon['z'].iloc[irow-1] - \
2./(air_balloon['rho'].iloc[irow-1]+air_balloon['rho'].iloc[irow])/g * \
(air_balloon['p'].iloc[irow] - air_balloon['p'].iloc[irow-1])
# for varname,lfunction in varcakc.items():
# air_balloon[varname] = lfunction(air_balloon)
dpars = {}
dpars['longitude'] = current_station['longitude']
dpars['latitude'] = current_station['latitude']
dpars['STNID'] = current_station.name
# # there are issues with the lower measurements in the HUMPPA campaign,
# # for which a steady decrease of potential temperature is found, which
# # is unrealistic. Here I filter them away
# ifirst = 0
# while (air_balloon.theta.iloc[ifirst+1] < air_balloon.theta.iloc[ifirst]):
# ifirst = ifirst+1
# print ('ifirst:',ifirst)
# air_balloon = air_balloon.iloc[ifirst:].reset_index().drop(['index'],axis=1)
air_balloon = air_balloon.iloc[:].reset_index().drop(['index'], axis=1)
# if air_balloon.z.max() > 100000.:
# air_balloon.z = air_balloon.z/10.
is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0)
valid_indices = air_balloon.index[is_valid].values
air_ap_mode = 'b'
while ((len(valid_indices) > 10)
and ((air_balloon.theta.iloc[valid_indices[0]] -
air_balloon.theta.iloc[valid_indices[1]]) > 0.5)):
valid_indices = valid_indices[1:]
#theta_vs_first_inconsistent = True
# while theta_vs_first_inconsistent:
if len(valid_indices) > 0:
air_balloon_temp = air_balloon.iloc[valid_indices]
print(air_balloon_temp)
print(
air_balloon_temp.z.shape,
air_balloon_temp.thetav.shape,
)
dpars['h'],dpars['h_u'],dpars['h_l'] =\
blh(air_balloon_temp.z.values,air_balloon_temp.thetav.values,air_balloon_temp.WSPD.values)
dpars['h_b'] = np.max((dpars['h'], 10.))
dpars['h_u'] = np.max(
(dpars['h_u'], 10.)) #upper limit of mixed layer height
dpars['h_l'] = np.max(
(dpars['h_l'], 10.)) #low limit of mixed layer height
dpars['h_e'] = np.abs(dpars['h_u'] -
dpars['h_l']) # error of mixed-layer height
dpars['h'] = np.round(dpars['h_' + air_ap_mode], 1)
else:
dpars['h_u'] = np.nan
dpars['h_l'] = np.nan
dpars['h_e'] = np.nan
dpars['h'] = np.nan
if ~np.isnan(dpars['h']):
dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]]
else:
dpars['Ps'] = np.nan
if ~np.isnan(dpars['h']):
# determine mixed-layer properties (moisture, potential temperature...) from profile
# ... and those of the mixed layer
is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h'])
valid_indices_below_h = air_balloon.iloc[valid_indices].index[
is_valid_below_h].values
if len(valid_indices) > 1:
if len(valid_indices_below_h) >= 3.:
ml_mean = air_balloon.iloc[valid_indices][
is_valid_below_h].mean()
else:
ml_mean = air_balloon.iloc[
valid_indices[0]:valid_indices[1]].mean()
elif len(valid_indices) == 1:
ml_mean = (air_balloon.iloc[0:1]).mean()
else:
temp = pd.DataFrame(air_balloon)
temp.iloc[0] = np.nan
ml_mean = temp
dpars['theta'] = ml_mean.theta
dpars['q'] = ml_mean.q
dpars['u'] = ml_mean.u
dpars['v'] = ml_mean.v
# theta_vs_first_inconsistent = \
# ((air_balloon.theta.iloc[valid_indices[0]] - air_balloon.theta.iloc[valid_indices[1]]) > 0.2)
# theta_vs_first_inconsistent = \
# ((air_balloon.theta.iloc[valid_indices[0]] - dpars['theta']) > 0.1)
# if theta_vs_first_inconsistent:
# valid_indices = valid_indices[1:]
# print("warning! too large difference between near surface value and abl value of theta. I'm taking the next one as near surface vlue")
else:
dpars['theta'] = np.nan
dpars['q'] = np.nan
dpars['u'] = np.nan
dpars['v'] = np.nan
# theta_bl_inconsistent = False
air_ap_head = air_balloon[0:
0] #pd.DataFrame(columns = air_balloon.columns)
# All other data points above the mixed-layer fit
air_ap_tail = air_balloon[air_balloon.z > dpars['h']]
air_ap_head.z = pd.Series(np.array([2., dpars['h'], dpars['h']]))
jump = air_ap_head.iloc[0] * np.nan
if air_ap_tail.shape[0] > 1:
# we originally used THTA, but that has another definition than the
# variable theta that we need which should be the temperature that
# one would have if brought to surface (NOT reference) pressure.
for column in ['theta', 'q', 'u', 'v']:
# initialize the profile head with the mixed-layer values
air_ap_head[column] = ml_mean[column]
# calculate jump values at mixed-layer height, which will be
# added to the third datapoint of the profile head
jump[column] = (air_ap_tail[column].iloc[1]\
-\
air_ap_tail[column].iloc[0])\
/\
(air_ap_tail.z.iloc[1]\
- air_ap_tail.z.iloc[0])\
*\
(dpars['h']- air_ap_tail.z.iloc[0])\
+\
air_ap_tail[column].iloc[0]\
-\
ml_mean[column]
if column == 'theta':
# for potential temperature, we need to set a lower limit to
# avoid the model to crash
jump.theta = np.max((0.1, jump.theta))
air_ap_head[column][2] += jump[column]
air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 + air_ap_head.v**2)
# only select samples monotonically increasing with height
air_ap_tail_orig = pd.DataFrame(air_ap_tail)
air_ap_tail = pd.DataFrame()
print(air_ap_tail_orig)
air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],
ignore_index=True)
for ibottom in range(1, len(air_ap_tail_orig)):
if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z + 10.:
air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom],
ignore_index=True)
# make theta increase strong enough to avoid numerical
# instability
air_ap_tail_orig = pd.DataFrame(air_ap_tail)
# air_ap_tail = pd.DataFrame()
# #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
# air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True)
# theta_low = air_ap_head['theta'].iloc[2]
# z_low = air_ap_head['z'].iloc[2]
# ibottom = 0
# for itop in range(0,len(air_ap_tail_orig)):
# theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean()
# z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean()
# if (
# #(z_mean > z_low) and \
# (z_mean > (z_low+10.)) and \
# #(theta_mean > (theta_low+0.2) ) and \
# #(theta_mean > (theta_low+0.2) ) and \
# (((theta_mean - theta_low)/(z_mean - z_low)) > 0.00001)):
# air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True)
# ibottom = itop+1
# theta_low = air_ap_tail.theta.iloc[-1]
# z_low = air_ap_tail.z.iloc[-1]
# # elif (itop > len(air_ap_tail_orig)-10):
# # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True)
#
air_ap = \
pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1)
# we copy the pressure at ground level from balloon sounding. The
# pressure at mixed-layer height will be determined internally by class
rho = 1.2 # density of air [kg m-3]
g = 9.81 # gravity acceleration [m s-2]
air_ap['p'].iloc[0] = dpars['Ps']
air_ap['p'].iloc[1] = (dpars['Ps'] - rho * g * dpars['h'])
air_ap['p'].iloc[2] = (dpars['Ps'] - rho * g * dpars['h'] - 0.1)
dpars['lat'] = dpars['latitude']
# this is set to zero because we use local (sun) time as input (as if we were in Greenwhich)
dpars['lon'] = 0.
# this is the real longitude that will be used to extract ground data
# dpars['ldatetime'] = ldate+dt.timedelta(hours=lhour)
# dpars['datetime'] = dpars['ldatetime'] + dt.timedelta(hours=+4)
dpars['datetime'] = ldate + dt.timedelta(hours=lhour)
dpars['ldatetime'] = dpars['datetime'] + dt.timedelta(hours=-4)
dpars['doy'] = dpars['datetime'].timetuple().tm_yday
dpars['SolarAltitude'] = \
Pysolar.GetAltitude(\
dpars['latitude'],\
dpars['longitude'],\
dpars['datetime']\
)
dpars['SolarAzimuth'] = Pysolar.GetAzimuth(\
dpars['latitude'],\
dpars['longitude'],\
dpars['datetime']\
)
dpars['lSunrise'], dpars['lSunset'] \
= Pysolar.util.GetSunriseSunset(dpars['latitude'],
0.,
dpars['ldatetime'],0.)
# Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually write local solar time, we need to assign the timezone to UTC (which is WRONG!!!). Otherwise ruby cannot understand it (it always converts tolocal computer time :( ).
dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise'])
dpars['lSunset'] = pytz.utc.localize(dpars['lSunset'])
# This is the nearest datetime when the sun is up (for class)
dpars['ldatetime_daylight'] = \
np.min(\
(np.max(\
(dpars['ldatetime'],\
dpars['lSunrise'])\
),\
dpars['lSunset']\
)\
)
# apply the same time shift for UTC datetime
dpars['datetime_daylight'] = dpars['datetime'] \
+\
(dpars['ldatetime_daylight']\
-\
dpars['ldatetime'])
# We set the starting time to the local sun time, since the model
# thinks we are always at the meridian (lon=0). This way the solar
# radiation is calculated correctly.
dpars['tstart'] = dpars['ldatetime_daylight'].hour \
+ \
dpars['ldatetime_daylight'].minute/60.\
+ \
dpars['ldatetime_daylight'].second/3600.
dpars['sw_lit'] = False
# convert numpy types to native python data types. This provides
# cleaner data IO with yaml:
for key, value in dpars.items():
if type(value).__module__ == 'numpy':
dpars[key] = dpars[key].item()
decimals = {
'p': 0,
't': 2,
'theta': 4,
'z': 2,
'q': 5,
'u': 4,
'v': 4
}
#
for column, decimal in decimals.items():
air_balloon[column] = air_balloon[column].round(decimal)
air_ap[column] = air_ap[column].round(decimal)
dpars['gammatheta_lower_limit'] = 0.0001
updateglobal = False
if c4gli is None:
c4gli = class4gl_input()
updateglobal = True
print('updating...')
print(column)
c4gli.update(source='goamazon',\
# pars=pars,
pars=dpars,\
air_balloon=air_balloon,\
air_ap=air_ap)
if updateglobal:
c4gli.get_global_input(globaldata)
# if profile_ini:
# c4gli.runtime = 10 * 3600
if not ((dpars['ldatetime'].hour <=12) or\
((dpars['lSunset'].hour - dpars['ldatetime'].hour) >= (2.))):
c4gli = None
# if profile_ini:
# c4gl = class4gl(c4gli)
# c4gl.run()
# c4gl.dump(file_model,\
# include_input=True,\
# timeseries_only=timeseries_only)
#
# # This will cash the observations and model tables per station for
# # the interface
#
# if profile_ini:
# profile_ini=False
# else:
# profile_ini=True
return c4gli
def get_record_yaml(yaml_file, index_start, index_end, mode='model_output'):
filename = yaml_file.name
#filename = path_yaml+'/'+format(current_station.name,'05d')+suffix
#yaml_file = open(filename)
shortfn = filename.split('/')[-1]
#print('going to next observation',filename)
yaml_file.seek(index_start)
print('index_start', index_start)
print('index_end', index_end)
buf = yaml_file.read(index_end - index_start).replace(
'inf', '9e19').replace('nan', '9e19').replace('---', '')
# # Ruby way -> fast
# os.system('mkdir -p '+TEMPDIR)
# filebuffer = open(TEMPDIR+'/'+shortfn+'.buffer.yaml.'+str(index_start),'w')
# filebuffer.write(buf)
# filebuffer.close()
# # print("HHHEEELOOOO",filename+'.buffer.yaml'+str(index_start))
#
# if which('ruby') is None:
# raise RuntimeError ('ruby is not found. Aborting...')
# command = 'ruby -rjson -ryaml -e "'+"puts YAML.load_file('"+TEMPDIR+'/'+shortfn+".buffer.yaml."+str(index_start)+"').to_json"+'" > '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start)+' '
# #command = '/apps/gent/CO7/sandybridge/software/Ruby/2.4.2-foss-2017b/bin/ruby -rjson -ryaml -e "'+"puts YAML.load(ARGF.read()).to_json"+'"'
# print(command)
# os.system(command)
# jsonstream = open(TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start))
# record_dict = json.load(jsonstream)
# jsonstream.close()
# os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.yaml.'+str(index_start))
record_dict = yaml.load(buf, Loader=CLoader)
if mode == 'model_output':
modelout = class4gl()
modelout.load_yaml_dict(record_dict)
# # needed in case of Ruby
# os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start))
print('hello')
return modelout
elif mode == 'model_input':
# datetimes are incorrectly converted to strings. We need to convert them
# again to datetimes
for key, value in record_dict['pars'].items():
# # needed in case of ruby
# we don't want the key with columns that have none values
# if value is not None:
# if key in ['lSunrise','lSunset','datetime','ldatetime','ldatetime_daylight','datetime_daylight',]:#(type(value) == str):
# # elif (type(value) == str):
# record_dict['pars'][key] = dt.datetime.strptime(value,"%Y-%m-%d %H:%M:%S %z")
if (value == 0.9e19) or (value == '.9e19'):
record_dict['pars'][key] = np.nan
for key in record_dict.keys():
#print(key)
if key in [
'air_ap',
'air_balloon',
]:
#NNprint('check')
for datakey, datavalue in record_dict[key].items():
record_dict[key][datakey] = [
np.nan if (x == '.9e19') else x
for x in record_dict[key][datakey]
]
c4gli = class4gl_input()
#print(c4gli.logger,'hello')
c4gli.load_yaml_dict(record_dict)
# # needed in case of ruby
# os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start))
return c4gli
else:
print('Warning. Mode ' + mode + ' not recorgnized. Returning None')
return None
