#lidar_path = 'C:/Users/there/Documents/Arduino/LEX_2018_grenzschicht/'
#lidar_path = '//192.168.206.173/lex2018/messdaten/lidar/LID/'
#lidar_filename = '0829.LID'
lidar_path = 'C:/Users/there/Documents/Arduino/LEX_2018_lokal/Daten/Daten/lidar/'
lidar_filename = 'lidar_data.npy'
ref = 0
p_intv_no = 20 # number of pressure levels to interpolate
plot_path = "//192.168.206.173//lex2018/profil/Plots/BL_height/"
start_time = datetime.datetime(2018, 8, 29, 7, 50)
end_time = datetime.datetime(2018, 8, 29, 14, 50)
#==============================================================================
data = read_data(data_path, file_name)
# apply calibration
data_calib = apply_correction(data)
lidar_data = read_lidar_pkl(lidar_path + lidar_filename)
# interpolate on pressure levels and a unit time
#unit_time,p_levels,Temp_pint,RH_pint,Theta= data_interpolation_p_t(data_calib,ref,p_intv_no,instrument_spef)
unit_time, z_levels, Temp_zint, RH_zint, Theta_zint, p_zint = data_interpolation_z_t(
data_calib, ref, p_intv_no, instrument_spef)
virt_pot_temp = calc_virt_pot(RH_zint, Temp_zint, p_zint)
winddir, windspeed_hor, windspeed_vert = interpolate_lidar_data(
lidar_data, unit_time, z_levels)
Ri = calc_bulk_richardson(virt_pot_temp, np.transpose(windspeed_hor),
np.tile(z_levels, (len(unit_time), 1)).transpose())
datanames[4] = '20180904124938_Grenzschichtentwicklung5'
datanames[5] = '20180905094034_Grenzschichtentwicklung6'
lengths = 6 * ['']
lengths[0] = np.array((2, 52, 152, 252, 352, 452, 552, 652, 752, 852, 952))
lengths[1] = np.array((3, 53, 153, 253, 353, 453, 553, 653, 753, 853, 950))
lengths[2] = np.array((2, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000))
lengths[3] = np.array((4, 100, 200, 300, 400, 500, 600, 700, 820, 900, 1000))
lengths[4] = np.array((4, 100, 200, 300, 400, 500, 600, 700, 820, 900, 1000))
lengths[5] = np.array((4, 100, 200, 300, 400, 500, 600, 700, 820, 900, 1000))
#dataname = '20180831105131_Grenzschichtentwicklung2'
for i in range(len(datanames)):
data = np.load('//192.168.206.173/lex2018/profil/Daten/' + datanames[i] +
'.npy')
data = apply_correction(data)
einzelschitt = np.load(
'C:/Users/Uni/LEX_2018_grenzschicht/Messdaten/einzelanschnitt.npy')
X_displacement = np.zeros(11)
Z_displacement = np.zeros(11)
Time = data[1][1, 0]
for zeit in data[1][1:, 0]:
try:
Xdummy, Zdummy = tether(zeit, data, i)
Time = np.append(Time, zeit)
X_displacement = np.vstack((X_displacement, Xdummy))
Z_displacement = np.vstack((Z_displacement, Zdummy))
#f.write(str(zeit) + ';' + str(Xdummy) + ';' + str(Zdummy)+'\n') # maybe on other systems "" needs to be ''
def compare_sonde(sondepath,
launchname,
groundtemp,
groundhum,
alpaca_filename,
plotpath,
month=8,
heliheight=600):
"""
Reads in sonde data and alpaca data, autmatically finds the
radiosonde launch time and the time when the sonde reached
the balloon height. Compares the sonde profile with ALPACA
profile averaged around the sonde launch time window. Computes
additional statistics.
"""
##### READ RADIOSONDE DATA
##### Radiosonde data file and info file
filename = sondepath + launchname + '_UTC_EDT.tsv'
infofile = sondepath + launchname + '_UTC_EDT_AuditTrail.tsv'
##### Read in sonde data
sondedata = np.genfromtxt(filename, skip_header=39)
##### Set missing values to NaN
sondedata[np.where(sondedata == -32768.)] = np.nan
## Time: 0
## Temperature: 2
## Humidity: 3
## v: 4
## u: 5
## Height 6
## Pressure 7
## Dewpoint 8
##### Convert pressure to height using temperature
# FIXME TODO
##### Convert temperature to celsius
sondedata[:, 2] -= 273.15
##### Find time when Radiosonde reached highest point, use only values
##### before that
apogee = np.argmax(sondedata[:, 6])
sondedata = sondedata[0:apogee, :]
##### Find the time when the Radiosonde was above 10 m. This should be
##### the time when the Sonde was launched. Use values after that
launchtime = np.where(sondedata[:, 6] > 10)[0][0]
heliheighttime = np.where(sondedata[:, 6] > heliheight)[0][0]
timetolaunch = sondedata[launchtime, 0]
timetoheliheight = sondedata[heliheighttime, 0]
sondedata = sondedata[launchtime - 1:heliheighttime, :]
##### Set the first value of the sonde temp and humidity to the ground values
#sondedata[0,2] = groundtemp
#sondedata[0,3] = groundhum
##### Get Radiosonde start time from the data
##### Get start of record time from infofile
with open(infofile, 'rb') as f:
clean_lines = (line.replace(b':', b' ') for line in f)
sondeinfo = np.genfromtxt(clean_lines,
dtype=int,
skip_header=4,
max_rows=1)
secondofstart = (sondeinfo[5] + 2) * 3600 + sondeinfo[6] * 60
##### Determine how many hours, minutes, seconds until the sonde was launched
hourstolaunch = int((timetolaunch + secondofstart) / 3600)
minutestolaunch = int(np.mod((timetolaunch + secondofstart), 3600) / 60)
secondstolaunch = int(
np.mod(np.mod((timetolaunch + secondofstart), 3600), 60))
##### Determine time of launch
sondelaunch = datetime.datetime(sondeinfo[4], month, sondeinfo[2],
hourstolaunch, minutestolaunch,
secondstolaunch)
##### Determine how many hours, minutes, seconds until the sonde reached helikite
hourstoheliheight = int((timetoheliheight + secondofstart) / 3600)
minutestoheliheight = int(
np.mod((timetoheliheight + secondofstart), 3600) / 60)
secondstoheliheight = int(
np.mod(np.mod((timetoheliheight + secondofstart), 3600), 60))
##### Determine time when sonde reached helikite
sondeheli = datetime.datetime(sondeinfo[4], month, sondeinfo[2],
hourstoheliheight, minutestoheliheight,
secondstoheliheight)
##### Create a nice string of the launch time for title, filename of plot
titletime = str(sondelaunch)
titletime = titletime.replace('-', '')
titletime = titletime.replace(' ', '')
titletime = titletime.replace(':', '')
#########################################################################################
##### READ ALPACA FILE
data = np.load(alpaca_filename)
data = apply_correction(data)
temp = {}
hum = {}
pres = {}
try:
for alpaca in data:
temp[alpaca] = data[alpaca][
np.logical_and(data[alpaca][:, 0] >= date2num(sondelaunch),
data[alpaca][:, 0] <= date2num(sondeheli)), 1]
hum[alpaca] = data[alpaca][
np.logical_and(data[alpaca][:, 0] >= date2num(sondelaunch),
data[alpaca][:, 0] <= date2num(sondeheli)), 2]
pres[alpaca] = data[alpaca][
np.logical_and(data[alpaca][:, 0] >= date2num(sondelaunch),
data[alpaca][:, 0] <= date2num(sondeheli)), 3]
if len(temp[alpaca]) == 0:
temp[alpaca] = np.array([
data[alpaca][data[alpaca][:, 0] >= date2num(sondelaunch),
1][0]
])
hum[alpaca] = np.array([
data[alpaca][data[alpaca][:, 0] >= date2num(sondelaunch),
2][0]
])
pres[alpaca] = np.array([
data[alpaca][data[alpaca][:, 0] >= date2num(sondelaunch),
3][0]
])
print('Arduino {}: Number of averaged timesteps: {}'.format(
alpaca, len(temp[alpaca])))
temp[alpaca] = np.mean(temp[alpaca])
hum[alpaca] = np.mean(hum[alpaca])
pres[alpaca] = np.mean(pres[alpaca])
except:
sys.exit({
"Something went wrong with the ALPACA data. The time of the \n \
ALPACAS and the Sonde launch are not matching. Try choosing \n \
a different ALPACA file"
})
alpacapres = np.asarray(list(pres.values()))
alpacatemp = np.asarray(list(temp.values()))
alpacahum = np.asarray(list(hum.values()))
#########################################################################################
##### Create figure with the comparison
#img = imread("alpaka2.png")
#img = np.flip(img,0)
fig, (axtemp, axhum) = plt.subplots(1, 2, figsize=(10, 8))
axtemp.plot(sondedata[:, 2],
sondedata[:, 7],
linestyle='--',
marker='x',
color='red',
markersize=10,
zorder=1)
axtemp.plot(alpacatemp,
alpacapres,
linestyle='--',
marker='x',
color='blue',
markersize=10,
zorder=1)
#axtemp.imshow(img,zorder=0,extent=[13, 20, 940, 1020],aspect=0.15,alpha=0.1)
axtemp.invert_yaxis()
axtemp.set_xlabel('Temperature [°C]')
axtemp.set_ylabel('Pressure [hPa]')
axtemp.legend(['Radiosonde', 'ALPACAS'])
#axtemp.grid(linestyle='--',alpha=0.1)
axtemp.set_title('Temperature')
fig.suptitle('Radiosonde-ALPACA comparison, Launchtime: ' + titletime)
axhum.plot(sondedata[:, 3],
sondedata[:, 7],
linestyle='--',
marker='x',
color='red',
markersize=10,
zorder=1)
axhum.plot(alpacahum,
alpacapres,
linestyle='--',
marker='x',
color='blue',
markersize=10,
zorder=1)
#axhum.imshow(img,zorder=0,extent=[56, 72, 940, 1020],aspect=0.34,alpha=0.1)
axhum.invert_yaxis()
axhum.set_xlabel('Rel. Humidity [%]')
axhum.legend(['Radiosonde', 'ALPACAS'])
axhum.grid(linestyle='--', alpha=0.1)
axhum.set_title('Relative Humidity')
fig.savefig(plotpath + 'Radiosonde_ALPACA_' + titletime + '.png')
print('Figure saved in ' + plotpath + 'Radiosonde_APLACA_' + titletime +
'.png')
###################################################################################
##### Compute additional statistics
##### Interpolate sonde data to pressure of ALPACAS
sondetemp = interp1d(sondedata[:, 7],
sondedata[:, 2],
fill_value='extrapolate',
kind='linear')(alpacapres)
sondehum = interp1d(sondedata[:, 7],
sondedata[:, 3],
fill_value='extrapolate',
kind='linear')(alpacapres)
##### Calculate BIAS, RMS
biastemp = np.mean(alpacatemp - sondetemp)
rmsetemp = np.sqrt(np.mean((alpacatemp - sondetemp)**2))
rtemp = np.corrcoef(alpacatemp, sondetemp)[0, 1]
print('Temperature:')
print('BIAS: ', biastemp)
print('RMSE: ', rmsetemp)
print('R: ', rtemp)
biashum = np.mean(alpacahum - sondehum)
rmsehum = np.sqrt(np.mean((alpacahum - sondehum)**2))
rhum = np.corrcoef(alpacahum, sondehum)[0, 1]
print('Humidity:')
print('BIAS: ', biashum)
print('RMSE: ', rmsehum)
print('R: ', rhum)
##### Scatter plot of ALPACA vs. sonde
fig, (axtemp, axhum) = plt.subplots(1, 2, figsize=(10, 8))
axtemp.plot([min(sondetemp) - 2, max(sondetemp) + 2],
[min(sondetemp) - 2, max(sondetemp) + 2],
color='black')
axtemp.scatter(sondetemp, alpacatemp, linestyle='--', color='red')
axtemp.set_xlabel('Temperature Sonde [°C]')
axtemp.set_ylabel('Pressure ALPACAS [°C]')
axtemp.set_xlim([min(sondetemp) - 2, max(sondetemp) + 2])
axtemp.set_ylim([min(sondetemp) - 2, max(sondetemp) + 2])
axtemp.grid(linestyle='--', alpha=0.1)
axtemp.set_title('BIAS: ' + str(round(biastemp, 2)) + ' RMSE: ' +
str(round(rmsetemp, 2)) + ' R: ' + str(round(rtemp, 2)),
fontsize=15)
axhum.plot([min(sondehum) - 2, max(sondehum) + 2],
[min(sondehum) - 2, max(sondehum) + 2],
color='black')
axhum.scatter(sondehum, alpacahum, linestyle='--', color='red')
axhum.set_xlabel('Humidity Sonde [%]')
axhum.set_ylabel('Humidity ALPACAS [%]')
axhum.set_xlim([min(sondehum) - 2, max(sondehum) + 2])
axhum.set_ylim([min(sondehum) - 2, max(sondehum) + 2])
axhum.grid(linestyle='--', alpha=0.1)
axhum.set_title('BIAS: ' + str(round(biashum, 2)) + ' RMSE: ' +
str(round(rmsehum, 2)) + ' R: ' + str(round(rhum, 2)),
fontsize=15)
fig.suptitle('Radiosonde-ALPACA comparison, Launchtime: ' + titletime)
fig.savefig(plotpath + 'Radiosonde_APLACA_scatter_' + titletime + '.png')
print('Figure saved in ' + plotpath + 'Radiosonde_APLACA_scatter_' +
titletime + '.png')
return sondedata[:, 7], sondedata[:,
2], sondedata[:,
3], sondelaunch, sondeheli
def compare_alpaca_bl_to_ceilometer(day, month):
instrument_spef = 0 #leave it 0!)
data_path = "C:/Users/Henning/Desktop/LEX_Netzwerk/profil/Daten/"
date_alpaca = [
"20180829062417", "20180831105131", "20180901095651", "20180903132225",
"20180904124938", "20180905094034"
]
date = ["29", "31", "01", "03", "04", "05"]
l = [date.index(i) for i in date if day in i]
idx = int(l[0]) + 1
if day == "03":
file_name_alpaca = date_alpaca[
l[0]] + "_Grenzschichtentwicklung" + str(idx) + "_2.npy"
else:
file_name_alpaca = date_alpaca[
l[0]] + "_Grenzschichtentwicklung" + str(idx) + ".npy"
file = data_path + file_name_alpaca
ref = 0
p_intv_no = 20
z_intv_no = p_intv_no # number of pressure levels to interpolate
plot_path = "C:/Users/Henning/Desktop/LEX_Netzwerk/profil/Plots/Liveplots/"
#==========================================================================
#==========================================================================
#Arduino based analysis
#use of functions from processing_functions.py
#=========================================================================
alpaca = read_data(data_path, file_name_alpaca)
alpaca_calib = apply_correction(alpaca)
unit_time, p_levels, Temp_pint, RH_pint, Theta = data_interpolation_p_t(
alpaca_calib, ref, p_intv_no, instrument_spef)
p_mid_levels, Temp_diff, Theta_diff, RH_diff = get_gradients(
unit_time, p_levels, Temp_pint, Theta, RH_pint)
z_BL_q, p_BL_q = boundary_layer_height(RH_pint, Temp_pint, p_levels,
'specific_humidity')
z_BL_rh, p_BL_rh = boundary_layer_height(RH_pint, Temp_pint, p_levels,
'relative_humidity')
z_BL_theta, p_BL_theta = boundary_layer_height(RH_pint, Temp_pint,
p_levels,
'potential_temperature')
z_BL_theta_e, p_BL_theta_e = boundary_layer_height(
RH_pint, Temp_pint, p_levels, 'pseudopotential_temperature')
ceilo_path = "C:/Users/Henning/Desktop/LEX_Netzwerk/Export/"
a = "CL_BLHGTA_2018" + month + day + "0000-2018" + month + day + "2359.csv"
b = "CL_BLHGTB_2018" + month + day + "0000-2018" + month + day + "2359.csv"
c = "CL_BLHGTC_2018" + month + day + "0000-2018" + month + day + "2359.csv"
if day == "05":
a = "CL_BLHGTA_2018" + month + day + "0000-2018" + month + day + "1800.csv"
b = "CL_BLHGTB_2018" + month + day + "0000-2018" + month + day + "1800.csv"
c = "CL_BLHGTC_2018" + month + day + "0000-2018" + month + day + "1800.csv"
file_name = [ceilo_path + a, ceilo_path + b, ceilo_path + c]
if day == "05":
ceilo_bl = np.zeros([1081, 4])
j = 0
first_date = datetime.datetime(2018, int(month), int(day), 0, 0)
time = first_date + np.arange(1081) * datetime.timedelta(minutes=1)
else:
ceilo_bl = np.zeros([1440, 4])
j = 0
first_date = datetime.datetime(2018, int(month), int(day), 0, 0)
time = first_date + np.arange(1440) * datetime.timedelta(minutes=1)
ceilo_bl[:, 0] = date2num(time)
for i in file_name:
data = np.genfromtxt(i)
ceilo_bl[:, j + 1] = data
j += 1
ceilo_new = ceilo_bl[np.where(
np.logical_and(ceilo_bl[:, 0] > unit_time[1],
ceilo_bl[:, 0] <= unit_time[-1:]))]
#Interpolate Ceilometer on Alpaca Resolution
# ceilo_tint=np.zeros([len(unit_time),4])
# ceilo_tint[:,0]=unit_time
# for j in range(1,4): # 0 Time, 1 Temp, 2 RH, 3
# ceilo_tint[:,j]= interp1d(ceilo_new[:,0],ceilo_new[:,j],axis=0,fill_value='extrapolate')(unit_time)
# print(file_name[j-1][45:51], " Ceilometer height; time interpolated")
z_bl_rh = np.zeros(len(ceilo_new[:, 0]))
z_bl_q = np.zeros(len(ceilo_new[:, 0]))
z_bl_theta = np.zeros(len(ceilo_new[:, 0]))
z_bl_theta_e = np.zeros(len(ceilo_new[:, 0]))
for i in range(len(ceilo_new) - 1):
alpaca_bl_rh = z_BL_rh[np.where(
np.logical_and(unit_time >= ceilo_new[i, 0],
unit_time <= ceilo_new[i + 1, 0]))]
alpaca_bl_q = z_BL_q[np.where(
np.logical_and(unit_time >= ceilo_new[i, 0],
unit_time <= ceilo_new[i + 1, 0]))]
alpaca_bl_theta = z_BL_theta[np.where(
np.logical_and(unit_time >= ceilo_new[i, 0],
unit_time <= ceilo_new[i + 1, 0]))]
alpaca_bl_theta_e = z_BL_theta_e[np.where(
np.logical_and(unit_time >= ceilo_new[i, 0],
unit_time <= ceilo_new[i + 1, 0]))]
z_bl_rh[i] = np.mean(alpaca_bl_rh)
z_bl_q[i] = np.mean(alpaca_bl_q)
z_bl_theta[i] = np.mean(alpaca_bl_theta)
z_bl_theta_e[i] = np.mean(alpaca_bl_theta_e)
cosmo_ceilo_time, cosmo_all_time, cosmo_data, c_potT_bnd, c_pseudoPotT_bnd, c_relH_bnd, c_QV_bnd, ceil_values_for_cosmo, = process_cosmo_data_for_ceilo_meter(
day, month, ceilo_new)
fig_name = "BL_height_1m_avg" + file_name_alpaca[:-4] + ".png"
fig_title = "Date " + file_name_alpaca[6:8] + "." + file_name_alpaca[
4:6] + "." + file_name_alpaca[
0:4] + " ,Start Time: " + file_name_alpaca[
8:10] + ":" + file_name_alpaca[10:12] + ":" + file_name_alpaca[
12:14]
print('Plotting Boundary layer height')
fig = plt.figure(figsize=(20, 10))
ax = fig.add_subplot(212)
plt.rcParams.update({'font.size': 12})
ax.plot(num2date(ceilo_new[:, 0]), z_bl_rh, label='Relative Feuchte')
ax.set_xlabel('Uhrzeit [MESZ]')
ax.set_ylabel('Grenzschicht Höhe [m]')
ax.set_ylim([0, 1500])
ax.set_xticks(ax.get_xticks()[::])
ax.xaxis.set_major_formatter(dates.DateFormatter('%H:%M:%S'))
ax.plot(num2date(ceilo_new[:, 0]), z_bl_q, label='Spezifische Feuchte')
ax.plot(num2date(ceilo_new[:, 0]), z_bl_theta, label='Pot. Temperatur')
ax.plot(num2date(ceilo_new[:, 0]),
z_bl_theta_e,
label='Pseudopot. Temperatur')
ax.plot(num2date(ceilo_new[:, 0]),
ceilo_new[:, 1],
color="black",
label="Ceilometer")
ax.plot(cosmo_ceilo_time, c_relH_bnd, 'b+', label='RH (Cosmo)', alpha=0.8)
ax.plot(cosmo_ceilo_time,
c_QV_bnd,
'o',
color='orange',
label='Spez. Feuchte (Cosmo)',
alpha=0.8)
ax.plot(cosmo_ceilo_time,
c_potT_bnd,
'g*',
label='Pot. T. (Cosmo)',
alpha=0.8)
ax.plot(cosmo_ceilo_time,
c_pseudoPotT_bnd,
'rv',
label='Pseudopot. T. (Cosmo)',
alpha=1.0)
ax.legend(loc='upper center',
bbox_to_anchor=(0.5, -0.15),
fancybox=True,
ncol=5)
ax.grid()
fig.savefig(plot_path + fig_name, dpi=500, bbox_inches="tight")
print('BL_ALPACA_CEILO plotted and stored on server')
plt.close()
return ceilo_new, z_bl_q, z_bl_rh, z_bl_theta, z_bl_theta_e, file_name_alpaca