# c4glfiles = dict([(EXP,odirexperiments[EXP]+'/'+format(STN['ID'],'05d')+'.yaml') \ # for EXP in experiments.keys()]) dict_diag_station = {} df_logic_morning = pd.DataFrame() df_logic_afternoon = pd.DataFrame() with open(fnout,'w') as fileout, \ open(fnout_afternoon,'w') as fileout_afternoon: wy_strm = wyoming(PATH=args.path_input, STNM=STN.name) wy_strm.set_STNM(int(STN.name)) # we consider all soundings from 1981 onwards wy_strm.find_first(year=int(args.startyear)) #wy_strm.find(dt.datetime(2004,10,19,6)) c4gli = class4gl_input(debug_level=logging.INFO) c4gli_afternoon = class4gl_input(debug_level=logging.INFO) # so we continue as long as we can find a new sounding while wy_strm.current is not None: c4gli.clear() c4gli.get_profile_wyoming(wy_strm) #import pdb;pdb.set_trace() #print(STN['ID'],c4gli.pars.datetime) #c4gli.get_global_input(globaldata) print(c4gli.pars.STNID, c4gli.pars.ldatetime) logic = dict() logic['morning'] = (c4gli.pars.ldatetime.hour <= 12.)
def bllast_parser(balloon_file, file_sounding, ldate, hour, c4gli=None): #balloon_conv = replace_iter(balloon_file,"°","deg") #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()] #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15) air_balloon_in = pd.read_csv( balloon_file, delimiter='\t', ) #widths=[14]*19, #skiprows=9, #skipfooter=15, #decimal='.', #header=None, #names = columns, #na_values='-----') air_balloon_in = air_balloon_in.rename(columns=lambda x: x.strip()) print(air_balloon_in.columns) rowmatches = { 't': lambda x: x['TaRad'] + 273.15, #'tv': lambda x: x['Virt. Temp[C]']+273.15, 'p': lambda x: x['Press'] * 100., 'u': lambda x: x['VHor'] * np.sin((90. - x['VDir']) / 180. * np.pi), 'v': lambda x: x['VHor'] * np.cos((90. - x['VDir']) / 180. * np.pi), 'z': lambda x: x['Altitude'] - 582., # from virtual temperature to absolute humidity 'q': lambda x: qfrom_e_p(efrom_rh100_T(x['UCal'], x['TaRad'] + 273.15), x[ 'Press'] * 100.), } air_balloon = pd.DataFrame() for varname, lfunction in rowmatches.items(): air_balloon[varname] = lfunction(air_balloon_in) rowmatches = { 'R': lambda x: (Rd * (1. - x.q) + Rv * x.q), 'theta': lambda x: (x['t']) * (x['p'][0] / x['p'])**(x['R'] / cp), 'thetav': lambda x: x.theta + 0.61 * x.theta * x.q } for varname, lfunction in rowmatches.items(): air_balloon[varname] = lfunction(air_balloon) dpars = {} dpars['longitude'] = current_station['longitude'] dpars['latitude'] = current_station['latitude'] dpars['STNID'] = current_station.name is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0) valid_indices = air_balloon.index[is_valid].values i = 1 while (air_balloon.thetav.iloc[valid_indices[0]] - \ air_balloon.thetav.iloc[valid_indices[i]] ) > 0.5: #diff = (air_balloon.theta.iloc[valid_indices[i]] -air_balloon.theta.iloc[valid_indices[i+1]])- 0.5 air_balloon.thetav.iloc[valid_indices[0:i]] = \ air_balloon.thetav.iloc[valid_indices[i]] + 0.5 i += 1 air_ap_mode = 'b' if len(valid_indices) > 0: dpars['h'],dpars['h_u'],dpars['h_l'] =\ blh(air_balloon.z,air_balloon.thetav,air_balloon_in['VHor']) dpars['h_b'] = np.max((dpars['h'], 10.)) dpars['h_u'] = np.max( (dpars['h_u'], 10.)) #upper limit of mixed layer height dpars['h_l'] = np.max( (dpars['h_l'], 10.)) #low limit of mixed layer height dpars['h_e'] = np.abs(dpars['h_u'] - dpars['h_l']) # error of mixed-layer height dpars['h'] = np.round(dpars['h_' + air_ap_mode], 1) else: dpars['h_u'] = np.nan dpars['h_l'] = np.nan dpars['h_e'] = np.nan dpars['h'] = np.nan if ~np.isnan(dpars['h']): dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]] else: dpars['Ps'] = np.nan if ~np.isnan(dpars['h']): # determine mixed-layer properties (moisture, potential temperature...) from profile # ... and those of the mixed layer is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h']) valid_indices_below_h = air_balloon.iloc[valid_indices].index[ is_valid_below_h].values if len(valid_indices) > 1: if len(valid_indices_below_h) >= 3.: ml_mean = air_balloon.iloc[valid_indices][ is_valid_below_h].mean() else: ml_mean = air_balloon.iloc[ valid_indices[0]:valid_indices[1]].mean() elif len(valid_indices) == 1: ml_mean = (air_balloon.iloc[0:1]).mean() else: temp = pd.DataFrame(air_balloon) temp.iloc[0] = np.nan ml_mean = temp dpars['theta'] = ml_mean.theta dpars['q'] = ml_mean.q dpars['u'] = ml_mean.u dpars['v'] = ml_mean.v else: dpars['theta'] = np.nan dpars['q'] = np.nan dpars['u'] = np.nan dpars['v'] = np.nan air_ap_head = air_balloon[0: 0] #pd.DataFrame(columns = air_balloon.columns) # All other data points above the mixed-layer fit air_ap_tail = air_balloon[air_balloon.z > dpars['h']] air_ap_head.z = pd.Series(np.array([2., dpars['h'], dpars['h']])) jump = air_ap_head.iloc[0] * np.nan if air_ap_tail.shape[0] > 1: # we originally used THTA, but that has another definition than the # variable theta that we need which should be the temperature that # one would have if brought to surface (NOT reference) pressure. for column in ['theta', 'q', 'u', 'v']: # initialize the profile head with the mixed-layer values air_ap_head[column] = ml_mean[column] # calculate jump values at mixed-layer height, which will be # added to the third datapoint of the profile head jump[column] = (air_ap_tail[column].iloc[1]\ -\ air_ap_tail[column].iloc[0])\ /\ (air_ap_tail.z.iloc[1]\ - air_ap_tail.z.iloc[0])\ *\ (dpars['h']- air_ap_tail.z.iloc[0])\ +\ air_ap_tail[column].iloc[0]\ -\ ml_mean[column] if column == 'theta': # for potential temperature, we need to set a lower limit to # avoid the model to crash jump.theta = np.max((0.1, jump.theta)) air_ap_head[column][2] += jump[column] air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 + air_ap_head.v**2) # filter data so that potential temperature always increases with # height cols = [] for column in air_ap_tail.columns: #if column != 'z': cols.append(column) # only select samples monotonically increasing with height air_ap_tail_orig = pd.DataFrame(air_ap_tail) air_ap_tail = pd.DataFrame() air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0], ignore_index=True) for ibottom in range(1, len(air_ap_tail_orig)): if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z + 10.: air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom], ignore_index=True) # make theta increase strong enough to avoid numerical # instability air_ap_tail_orig = pd.DataFrame(air_ap_tail) air_ap_tail = pd.DataFrame() #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0], ignore_index=True) theta_low = air_ap_head['theta'].iloc[2] z_low = air_ap_head['z'].iloc[2] ibottom = 0 for itop in range(0, len(air_ap_tail_orig)): theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop + 1)].mean() z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop + 1)].mean() if ( #(z_mean > z_low) and \ (z_mean > (z_low+10.)) and \ #(theta_mean > (theta_low+0.2) ) and \ #(theta_mean > (theta_low+0.2) ) and \ (((theta_mean - theta_low)/(z_mean - z_low)) > 0.0001)): air_ap_tail = air_ap_tail.append( air_ap_tail_orig.iloc[ibottom:(itop + 1)].mean(), ignore_index=True) ibottom = itop + 1 theta_low = air_ap_tail.theta.iloc[-1] z_low = air_ap_tail.z.iloc[-1] # elif (itop > len(air_ap_tail_orig)-10): # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True) air_ap = \ pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1) # # make theta increase strong enough to avoid numerical # # instability # air_ap_tail_orig = pd.DataFrame(air_ap_tail) # air_ap_tail = pd.DataFrame() # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) # theta_low = air_ap_head['theta'].iloc[2] # z_low = air_ap_head['z'].iloc[2] # ibottom = 0 # for itop in range(0,len(air_ap_tail_orig)): # theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean() # z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean() # if ((theta_mean > (theta_low+0.2) ) and \ # (((theta_mean - theta_low)/(z_mean - z_low)) > 0.001)): # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True) # ibottom = itop+1 # theta_low = air_ap_tail.theta.iloc[-1] # z_low = air_ap_tail.z.iloc[-1] # # elif (itop > len(air_ap_tail_orig)-10): # # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True) # air_ap = \ # pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1) # # # we copy the pressure at ground level from balloon sounding. The # # pressure at mixed-layer height will be determined internally by class rho = 1.2 # density of air [kg m-3] g = 9.81 # gravity acceleration [m s-2] air_ap['p'].iloc[0] = dpars['Ps'] air_ap['p'].iloc[1] = (dpars['Ps'] - rho * g * dpars['h']) air_ap['p'].iloc[2] = (dpars['Ps'] - rho * g * dpars['h'] - 0.1) dpars['lat'] = dpars['latitude'] # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich) dpars['lon'] = 0. # this is the real longitude that will be used to extract ground data dpars['ldatetime'] = ldate + dt.timedelta(hours=hour) dpars['datetime'] = ldate + dt.timedelta(hours=hour) dpars['doy'] = dpars['datetime'].timetuple().tm_yday dpars['SolarAltitude'] = \ Pysolar.GetAltitude(\ dpars['latitude'],\ dpars['longitude'],\ dpars['datetime']\ ) dpars['SolarAzimuth'] = Pysolar.GetAzimuth(\ dpars['latitude'],\ dpars['longitude'],\ dpars['datetime']\ ) dpars['lSunrise'], dpars['lSunset'] \ = Pysolar.util.GetSunriseSunset(dpars['latitude'], 0., dpars['ldatetime'],0.) # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually # write local solar time, we need to assign the timezone to UTC (which # is WRONG!!!). Otherwise ruby cannot understand it (it always converts # tolocal computer time :( ). dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise']) dpars['lSunset'] = pytz.utc.localize(dpars['lSunset']) # This is the nearest datetime when the sun is up (for class) dpars['ldatetime_daylight'] = \ np.min(\ (np.max(\ (dpars['ldatetime'],\ dpars['lSunrise']+dt.timedelta(hours=2))\ ),\ dpars['lSunset']\ )\ ) # apply the same time shift for UTC datetime dpars['datetime_daylight'] = dpars['datetime'] \ +\ (dpars['ldatetime_daylight']\ -\ dpars['ldatetime']) print('ldatetime_daylight', dpars['ldatetime_daylight']) print('ldatetime', dpars['ldatetime']) print('lSunrise', dpars['lSunrise']) dpars['day'] = dpars['ldatetime'].day # We set the starting time to the local sun time, since the model # thinks we are always at the meridian (lon=0). This way the solar # radiation is calculated correctly. dpars['tstart'] = dpars['ldatetime_daylight'].hour \ + \ dpars['ldatetime_daylight'].minute/60.\ + \ dpars['ldatetime_daylight'].second/3600. print('tstart', dpars['tstart']) dpars['sw_lit'] = False # convert numpy types to native python data types. This provides # cleaner data IO with yaml: for key, value in dpars.items(): if type(value).__module__ == 'numpy': dpars[key] = dpars[key].item() decimals = { 'p': 0, 't': 2, 'theta': 4, 'z': 2, 'q': 5, 'u': 4, 'v': 4 } # for column, decimal in decimals.items(): air_balloon[column] = air_balloon[column].round(decimal) air_ap[column] = air_ap[column].round(decimal) updateglobal = False if c4gli is None: c4gli = class4gl_input() updateglobal = True print('updating...') print(column) c4gli.update(source='bllast',\ # pars=pars, pars=dpars,\ air_balloon=air_balloon,\ air_ap=air_ap) if updateglobal: c4gli.get_global_input(globaldata) # if profile_ini: # c4gli.runtime = 10 * 3600 c4gli.dump(file_sounding) # if profile_ini: # c4gl = class4gl(c4gli) # c4gl.run() # c4gl.dump(file_model,\ # include_input=True,\ # timeseries_only=timeseries_only) # # # This will cash the observations and model tables per station for # # the interface # # if profile_ini: # profile_ini=False # else: # profile_ini=True return c4gli
#run_station_chunk = int(args.global_chunk_number) # =============================== print('start looping over chunk') # =============================== os.system('mkdir -p '+args.path_experiments) fn_ini = args.path_experiments+'/'+format(run_station.name,'05d')+'_'+\ str(int(run_station_chunk))+'_'+args.subset_experiments+'.yaml' file_ini = open(fn_ini,'w');print('Writing to: ',fn_ini) for iDT,DT in enumerate(DTS_chunk): print(iDT,DT) c4gli = class4gl_input(debug_level=logging.INFO) c4gli.update(source='STNID'+format(STNID,'05d'),\ pars=dict(latitude = float(run_station.latitude), \ longitude = float(run_station.longitude),\ lat = float(run_station.latitude), \ # Note the difference between longitude and lon. The # lon variable should always be zero because we are # always working in solar time for running CLASS lon = 0.,\ STNID = int(STNID))) lSunrise, lSunset = GetSunriseSunset(c4gli.pars.latitude,0.,DT) #start simulation at sunrise and stop at one hour before sunset runtime = (lSunset - lSunrise).total_seconds() - 3600.*1. ldatetime = lSunrise
def humppa_parser(balloon_file, file_sounding, ldate, lhour, c4gli=None): print(balloon_file) xrin = balloon_file air_balloon = pd.DataFrame() air_balloon['t'] = xrin.tdry.values + 273.15 air_balloon['p'] = xrin.pres.values * 100. air_balloon['u'] = xrin.u_wind.values air_balloon['v'] = xrin.v_wind.values air_balloon['WSPD'] = xrin['wspd'].values print(xrin.rh.values.shape) air_balloon['q'] = qfrom_e_p( efrom_rh100_T(xrin.rh.values, air_balloon['t'].values), air_balloon.p.values) #balloon_conv = replace_iter(balloon_file,"°","deg") #readlines = [ str(line).replace('°','deg') for line in balloon_file.readlines()] #air_balloon = pd.read_fwf( io.StringIO(''.join(readlines)),skiprows=8,skipfooter=15) # air_balloon_in = pd.read_fwf(balloon_file, # widths=[14]*19, # skiprows=9, # skipfooter=15, # decimal=',', # header=None, # names = columns, # na_values='-----') print(air_balloon['p'][0]) varcalc = { 'R': lambda x: (Rd * (1. - x.q) + Rv * x.q), 'pp': lambda x: (x['p'][0] / x['p'])**(x['R'] / cp), 'theta': lambda x: (x['t']) * (x['p'][0] / x['p'])**(x['R'] / cp), 'thetav': lambda x: x.theta + 0.61 * x.theta * x.q, 'rho': lambda x: x.p / x.t / x.R, } for varname, lfunction in varcalc.items(): air_balloon[varname] = lfunction(air_balloon) print('alt in xrin?:', 'alt' in xrin) if 'alt' in xrin: air_balloon['z'] = xrin.alt.values else: g = 9.81 # gravity acceleration [m s-2] air_balloon['z'] = 0. for irow, row in air_balloon.iloc[1:].iterrows(): air_balloon['z'].iloc[irow] = air_balloon['z'].iloc[irow-1] - \ 2./(air_balloon['rho'].iloc[irow-1]+air_balloon['rho'].iloc[irow])/g * \ (air_balloon['p'].iloc[irow] - air_balloon['p'].iloc[irow-1]) # for varname,lfunction in varcakc.items(): # air_balloon[varname] = lfunction(air_balloon) dpars = {} dpars['longitude'] = current_station['longitude'] dpars['latitude'] = current_station['latitude'] dpars['STNID'] = current_station.name # # there are issues with the lower measurements in the HUMPPA campaign, # # for which a steady decrease of potential temperature is found, which # # is unrealistic. Here I filter them away # ifirst = 0 # while (air_balloon.theta.iloc[ifirst+1] < air_balloon.theta.iloc[ifirst]): # ifirst = ifirst+1 # print ('ifirst:',ifirst) # air_balloon = air_balloon.iloc[ifirst:].reset_index().drop(['index'],axis=1) air_balloon = air_balloon.iloc[:].reset_index().drop(['index'], axis=1) # if air_balloon.z.max() > 100000.: # air_balloon.z = air_balloon.z/10. is_valid = ~np.isnan(air_balloon).any(axis=1) & (air_balloon.z >= 0) valid_indices = air_balloon.index[is_valid].values air_ap_mode = 'b' while ((len(valid_indices) > 10) and ((air_balloon.theta.iloc[valid_indices[0]] - air_balloon.theta.iloc[valid_indices[1]]) > 0.5)): valid_indices = valid_indices[1:] #theta_vs_first_inconsistent = True # while theta_vs_first_inconsistent: if len(valid_indices) > 0: air_balloon_temp = air_balloon.iloc[valid_indices] print(air_balloon_temp) print( air_balloon_temp.z.shape, air_balloon_temp.thetav.shape, ) dpars['h'],dpars['h_u'],dpars['h_l'] =\ blh(air_balloon_temp.z.values,air_balloon_temp.thetav.values,air_balloon_temp.WSPD.values) dpars['h_b'] = np.max((dpars['h'], 10.)) dpars['h_u'] = np.max( (dpars['h_u'], 10.)) #upper limit of mixed layer height dpars['h_l'] = np.max( (dpars['h_l'], 10.)) #low limit of mixed layer height dpars['h_e'] = np.abs(dpars['h_u'] - dpars['h_l']) # error of mixed-layer height dpars['h'] = np.round(dpars['h_' + air_ap_mode], 1) else: dpars['h_u'] = np.nan dpars['h_l'] = np.nan dpars['h_e'] = np.nan dpars['h'] = np.nan if ~np.isnan(dpars['h']): dpars['Ps'] = air_balloon.p.iloc[valid_indices[0]] else: dpars['Ps'] = np.nan if ~np.isnan(dpars['h']): # determine mixed-layer properties (moisture, potential temperature...) from profile # ... and those of the mixed layer is_valid_below_h = (air_balloon.iloc[valid_indices].z < dpars['h']) valid_indices_below_h = air_balloon.iloc[valid_indices].index[ is_valid_below_h].values if len(valid_indices) > 1: if len(valid_indices_below_h) >= 3.: ml_mean = air_balloon.iloc[valid_indices][ is_valid_below_h].mean() else: ml_mean = air_balloon.iloc[ valid_indices[0]:valid_indices[1]].mean() elif len(valid_indices) == 1: ml_mean = (air_balloon.iloc[0:1]).mean() else: temp = pd.DataFrame(air_balloon) temp.iloc[0] = np.nan ml_mean = temp dpars['theta'] = ml_mean.theta dpars['q'] = ml_mean.q dpars['u'] = ml_mean.u dpars['v'] = ml_mean.v # theta_vs_first_inconsistent = \ # ((air_balloon.theta.iloc[valid_indices[0]] - air_balloon.theta.iloc[valid_indices[1]]) > 0.2) # theta_vs_first_inconsistent = \ # ((air_balloon.theta.iloc[valid_indices[0]] - dpars['theta']) > 0.1) # if theta_vs_first_inconsistent: # valid_indices = valid_indices[1:] # print("warning! too large difference between near surface value and abl value of theta. I'm taking the next one as near surface vlue") else: dpars['theta'] = np.nan dpars['q'] = np.nan dpars['u'] = np.nan dpars['v'] = np.nan # theta_bl_inconsistent = False air_ap_head = air_balloon[0: 0] #pd.DataFrame(columns = air_balloon.columns) # All other data points above the mixed-layer fit air_ap_tail = air_balloon[air_balloon.z > dpars['h']] air_ap_head.z = pd.Series(np.array([2., dpars['h'], dpars['h']])) jump = air_ap_head.iloc[0] * np.nan if air_ap_tail.shape[0] > 1: # we originally used THTA, but that has another definition than the # variable theta that we need which should be the temperature that # one would have if brought to surface (NOT reference) pressure. for column in ['theta', 'q', 'u', 'v']: # initialize the profile head with the mixed-layer values air_ap_head[column] = ml_mean[column] # calculate jump values at mixed-layer height, which will be # added to the third datapoint of the profile head jump[column] = (air_ap_tail[column].iloc[1]\ -\ air_ap_tail[column].iloc[0])\ /\ (air_ap_tail.z.iloc[1]\ - air_ap_tail.z.iloc[0])\ *\ (dpars['h']- air_ap_tail.z.iloc[0])\ +\ air_ap_tail[column].iloc[0]\ -\ ml_mean[column] if column == 'theta': # for potential temperature, we need to set a lower limit to # avoid the model to crash jump.theta = np.max((0.1, jump.theta)) air_ap_head[column][2] += jump[column] air_ap_head.WSPD = np.sqrt(air_ap_head.u**2 + air_ap_head.v**2) # only select samples monotonically increasing with height air_ap_tail_orig = pd.DataFrame(air_ap_tail) air_ap_tail = pd.DataFrame() print(air_ap_tail_orig) air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0], ignore_index=True) for ibottom in range(1, len(air_ap_tail_orig)): if air_ap_tail_orig.iloc[ibottom].z > air_ap_tail.iloc[-1].z + 10.: air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom], ignore_index=True) # make theta increase strong enough to avoid numerical # instability air_ap_tail_orig = pd.DataFrame(air_ap_tail) # air_ap_tail = pd.DataFrame() # #air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[0],ignore_index=True) # theta_low = air_ap_head['theta'].iloc[2] # z_low = air_ap_head['z'].iloc[2] # ibottom = 0 # for itop in range(0,len(air_ap_tail_orig)): # theta_mean = air_ap_tail_orig.theta.iloc[ibottom:(itop+1)].mean() # z_mean = air_ap_tail_orig.z.iloc[ibottom:(itop+1)].mean() # if ( # #(z_mean > z_low) and \ # (z_mean > (z_low+10.)) and \ # #(theta_mean > (theta_low+0.2) ) and \ # #(theta_mean > (theta_low+0.2) ) and \ # (((theta_mean - theta_low)/(z_mean - z_low)) > 0.00001)): # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[ibottom:(itop+1)].mean(),ignore_index=True) # ibottom = itop+1 # theta_low = air_ap_tail.theta.iloc[-1] # z_low = air_ap_tail.z.iloc[-1] # # elif (itop > len(air_ap_tail_orig)-10): # # air_ap_tail = air_ap_tail.append(air_ap_tail_orig.iloc[itop],ignore_index=True) # air_ap = \ pd.concat((air_ap_head,air_ap_tail)).reset_index().drop(['index'],axis=1) # we copy the pressure at ground level from balloon sounding. The # pressure at mixed-layer height will be determined internally by class rho = 1.2 # density of air [kg m-3] g = 9.81 # gravity acceleration [m s-2] air_ap['p'].iloc[0] = dpars['Ps'] air_ap['p'].iloc[1] = (dpars['Ps'] - rho * g * dpars['h']) air_ap['p'].iloc[2] = (dpars['Ps'] - rho * g * dpars['h'] - 0.1) dpars['lat'] = dpars['latitude'] # this is set to zero because we use local (sun) time as input (as if we were in Greenwhich) dpars['lon'] = 0. # this is the real longitude that will be used to extract ground data # dpars['ldatetime'] = ldate+dt.timedelta(hours=lhour) # dpars['datetime'] = dpars['ldatetime'] + dt.timedelta(hours=+4) dpars['datetime'] = ldate + dt.timedelta(hours=lhour) dpars['ldatetime'] = dpars['datetime'] + dt.timedelta(hours=-4) dpars['doy'] = dpars['datetime'].timetuple().tm_yday dpars['SolarAltitude'] = \ Pysolar.GetAltitude(\ dpars['latitude'],\ dpars['longitude'],\ dpars['datetime']\ ) dpars['SolarAzimuth'] = Pysolar.GetAzimuth(\ dpars['latitude'],\ dpars['longitude'],\ dpars['datetime']\ ) dpars['lSunrise'], dpars['lSunset'] \ = Pysolar.util.GetSunriseSunset(dpars['latitude'], 0., dpars['ldatetime'],0.) # Warning!!! Unfortunatly!!!! WORKAROUND!!!! Even though we actually write local solar time, we need to assign the timezone to UTC (which is WRONG!!!). Otherwise ruby cannot understand it (it always converts tolocal computer time :( ). dpars['lSunrise'] = pytz.utc.localize(dpars['lSunrise']) dpars['lSunset'] = pytz.utc.localize(dpars['lSunset']) # This is the nearest datetime when the sun is up (for class) dpars['ldatetime_daylight'] = \ np.min(\ (np.max(\ (dpars['ldatetime'],\ dpars['lSunrise'])\ ),\ dpars['lSunset']\ )\ ) # apply the same time shift for UTC datetime dpars['datetime_daylight'] = dpars['datetime'] \ +\ (dpars['ldatetime_daylight']\ -\ dpars['ldatetime']) # We set the starting time to the local sun time, since the model # thinks we are always at the meridian (lon=0). This way the solar # radiation is calculated correctly. dpars['tstart'] = dpars['ldatetime_daylight'].hour \ + \ dpars['ldatetime_daylight'].minute/60.\ + \ dpars['ldatetime_daylight'].second/3600. dpars['sw_lit'] = False # convert numpy types to native python data types. This provides # cleaner data IO with yaml: for key, value in dpars.items(): if type(value).__module__ == 'numpy': dpars[key] = dpars[key].item() decimals = { 'p': 0, 't': 2, 'theta': 4, 'z': 2, 'q': 5, 'u': 4, 'v': 4 } # for column, decimal in decimals.items(): air_balloon[column] = air_balloon[column].round(decimal) air_ap[column] = air_ap[column].round(decimal) dpars['gammatheta_lower_limit'] = 0.0001 updateglobal = False if c4gli is None: c4gli = class4gl_input() updateglobal = True print('updating...') print(column) c4gli.update(source='goamazon',\ # pars=pars, pars=dpars,\ air_balloon=air_balloon,\ air_ap=air_ap) if updateglobal: c4gli.get_global_input(globaldata) # if profile_ini: # c4gli.runtime = 10 * 3600 if not ((dpars['ldatetime'].hour <=12) or\ ((dpars['lSunset'].hour - dpars['ldatetime'].hour) >= (2.))): c4gli = None # if profile_ini: # c4gl = class4gl(c4gli) # c4gl.run() # c4gl.dump(file_model,\ # include_input=True,\ # timeseries_only=timeseries_only) # # # This will cash the observations and model tables per station for # # the interface # # if profile_ini: # profile_ini=False # else: # profile_ini=True return c4gli
def get_record_yaml(yaml_file, index_start, index_end, mode='model_output'): filename = yaml_file.name #filename = path_yaml+'/'+format(current_station.name,'05d')+suffix #yaml_file = open(filename) shortfn = filename.split('/')[-1] #print('going to next observation',filename) yaml_file.seek(index_start) print('index_start', index_start) print('index_end', index_end) buf = yaml_file.read(index_end - index_start).replace( 'inf', '9e19').replace('nan', '9e19').replace('---', '') # # Ruby way -> fast # os.system('mkdir -p '+TEMPDIR) # filebuffer = open(TEMPDIR+'/'+shortfn+'.buffer.yaml.'+str(index_start),'w') # filebuffer.write(buf) # filebuffer.close() # # print("HHHEEELOOOO",filename+'.buffer.yaml'+str(index_start)) # # if which('ruby') is None: # raise RuntimeError ('ruby is not found. Aborting...') # command = 'ruby -rjson -ryaml -e "'+"puts YAML.load_file('"+TEMPDIR+'/'+shortfn+".buffer.yaml."+str(index_start)+"').to_json"+'" > '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start)+' ' # #command = '/apps/gent/CO7/sandybridge/software/Ruby/2.4.2-foss-2017b/bin/ruby -rjson -ryaml -e "'+"puts YAML.load(ARGF.read()).to_json"+'"' # print(command) # os.system(command) # jsonstream = open(TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start)) # record_dict = json.load(jsonstream) # jsonstream.close() # os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.yaml.'+str(index_start)) record_dict = yaml.load(buf, Loader=CLoader) if mode == 'model_output': modelout = class4gl() modelout.load_yaml_dict(record_dict) # # needed in case of Ruby # os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start)) print('hello') return modelout elif mode == 'model_input': # datetimes are incorrectly converted to strings. We need to convert them # again to datetimes for key, value in record_dict['pars'].items(): # # needed in case of ruby # we don't want the key with columns that have none values # if value is not None: # if key in ['lSunrise','lSunset','datetime','ldatetime','ldatetime_daylight','datetime_daylight',]:#(type(value) == str): # # elif (type(value) == str): # record_dict['pars'][key] = dt.datetime.strptime(value,"%Y-%m-%d %H:%M:%S %z") if (value == 0.9e19) or (value == '.9e19'): record_dict['pars'][key] = np.nan for key in record_dict.keys(): #print(key) if key in [ 'air_ap', 'air_balloon', ]: #NNprint('check') for datakey, datavalue in record_dict[key].items(): record_dict[key][datakey] = [ np.nan if (x == '.9e19') else x for x in record_dict[key][datakey] ] c4gli = class4gl_input() #print(c4gli.logger,'hello') c4gli.load_yaml_dict(record_dict) # # needed in case of ruby # os.system('rm '+TEMPDIR+'/'+shortfn+'.buffer.json.'+str(index_start)) return c4gli else: print('Warning. Mode ' + mode + ' not recorgnized. Returning None') return None