def plot_height_bias_histograms(caObj):
height_c = 1000*caObj.calipso.all_arrays['layer_top_altitude'][:,0]
height_pps = caObj.avhrr.all_arrays['ctth_height']
height_pps[height_pps>=0] = height_pps[height_pps>=0] + caObj.calipso.all_arrays['elevation'][height_pps>=0]
bias = 0.001*(height_pps - height_c)
#bias = caObj.avhrr.all_arrays['ctth_temperature'] -273.15 - caObj.calipso.all_arrays['layer_top_temperature'][:,0]
use = np.logical_and(caObj.avhrr.all_arrays['ctth_height']>-8,
#caObj.calipso.all_arrays['number_layers_found'][:]>0)
caObj.calipso.all_arrays['layer_top_altitude'][:,0]>-8)
bins = 4*np.array(xrange(-10,30, 1))
bins = 0.5*np.array(xrange(-14,8, 1))
#bins[0] = -30
#bins[-1] = 30
fig = plt.figure(figsize = (16,10))
ax = fig.add_subplot(331)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=0))], bins, alpha=0.5, label='0 transparent overcast', facecolor ='r')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(332)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=1))], bins, alpha=0.5, label='1 opaque overcast')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(333)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=2))], bins, alpha=0.5, label='2 transition cumulus')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(334)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=3))], bins, alpha=0.5, label='3 low broken cumuls')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(335)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=4))], bins, alpha=0.5, label='4 altocumuls')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(336)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=5))], bins, alpha=0.5, label='5 altostratus')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(337)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=6))], bins, alpha=0.5, label='6 cirrus', facecolor ='r')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(338)
plt.hist(bias[np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=7))],bins, alpha=0.5, label='7 deep convective')
plt.legend(loc='upper left')
#ax.set_ylim(0,600000)
ax = fig.add_subplot(339)
bins = np.array(xrange(0,20000, 1000))
plt.hist(height_pps[use], bins, alpha=0.5, label='pps heights')
plt.legend(loc='upper right')
plt.suptitle('Ctth height bias')
#plt.legend(loc='upper left')
#ax.set_ylabel('number of observations')
#ax.set_xlabel("bias ")
#plt.title("ctth height bias histogram (day)")
#ax.set_xlim(-20,30)
plt.savefig("ctth_bias_temp_hist.png")
plt.show()
def plot_ct_table2(caObj):
from get_flag_info import get_calipso_clouds_of_type_i
fig = plt.figure(figsize=(30, 64))
print "N, low, frac, medium, high, cirrus"
for type_i in xrange(0, 8):
if type_i == 1:
continue
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
pps_ctype = caObj.avhrr.all_arrays['cloudtype']
use = np.logical_and(pps_ctype > 4, pps_ctype < 23)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
print "N", ("%d" % (N)).rjust(9, ' '),
for ind_ct in xrange(4, 9):
pps_ok = np.logical_and(pps_ctype >= ct_min_v[ind_ct],
pps_ctype <= ct_max_v[ind_ct])
these = np.logical_and(is_type_i, pps_ok)
print("%3.1f" % (np.sum(these) * 1.0 / N * 100)).rjust(5, ' '),
print cc_type_name[type_i], np.percentile(
caObj.calipso.all_arrays['layer_top_pressure'][:, 0][
np.logical_and(
these,
caObj.calipso.all_arrays['layer_top_pressure'][:, 0] > 0)],
0.1)
def plot_ct_table(caObj):
from get_flag_info import get_calipso_clouds_of_type_i
print "N, 0 clear clear clear clear very-low, low, medium, high very-high, fractional, cirrus, cirrus, cirrus, cirrus-above-low"
for type_i in xrange(0, 8):
if type_i == 1:
continue
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
pps_ctype = caObj.avhrr.all_arrays['cloudtype']
use = np.logical_and(pps_ctype > 0, pps_ctype < 23)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
print "N", ("%d" % (N)).rjust(9, ' '),
for ind_ct in xrange(15):
these = np.logical_and(is_type_i, pps_ctype == ind_ct)
print("%3.1f" % (np.sum(these) * 1.0 / N * 100)).rjust(5, ' '),
if type_i == 2 and ind_ct == 2:
plt.plot(caObj.avhrr.all_arrays['longitude'][these],
caObj.avhrr.all_arrays['latitude'][these],
'b.',
alpha=0.4,
label="cal-%d,pps-%d" % (type_i, ind_ct))
plt.legend()
plt.show()
print cc_type_name[type_i]
def plot_one(feature, use, caObj, cc_type, color='b'):
plt.hist(feature[
np.logical_and(use,
get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=cc_type))],
bins, alpha=0.5, label =cc_type_name[cc_type], facecolor = color)
plt.legend(loc='upper left')
def plot_sub_scatter_plot_cttht_t11(caObj,cc_type, use, t11, ctth, ax,
xmin=-15, xmax=350, title=''):
import time
plt.title(cc_type_name[cc_type])
my_use = np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=cc_type))
plot_sub_scatter_plot_bias_diff_inner(caObj, my_use, t11, ctth, ax,
xmin=xmin, xmax=xmax)
def get_ctth_bias_type(self, caObj, calipso_cloudtype=0):
from get_flag_info import get_calipso_clouds_of_type_i
wanted_clouds = get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=calipso_cloudtype)
detected_typei = np.logical_and(self.detected_height, wanted_clouds[self.use])
delta_h = self.height_bias.copy()
delta_h[~detected_typei]=0
self.height_bias_type[calipso_cloudtype] = delta_h
self.detected_height_type[calipso_cloudtype] = detected_typei
def plot_ct_table6(caObj, use_in=None):
from get_flag_info import get_calipso_clouds_of_type_i
fig = plt.figure(figsize=(30, 64))
pps_ctype = caObj.avhrr.all_arrays['cloudtype']
use = np.logical_and(pps_ctype > 0, pps_ctype < 23)
use = np.logical_and(use, use_in)
use = np.logical_and(use, caObj.calipso.all_arrays['cloud_fraction'] > 0.9)
#for type_i in [0,2,3,4,5,6,7]:
# print cc_type_name[type_i].replace(' ','_'),
print "N ",
for type_i in [0, 2, 3, 4, 5, 6, 7]:
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
print("%d" % (N)).rjust(5, ' '),
print ""
print "clear: ",
for type_i in [0, 2, 3, 4, 5, 6, 7]:
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
if N < 50:
print("---").rjust(5, ' '),
else:
these = np.logical_and(is_type_i, pps_ctype < 5)
print("%3.1f" % (np.sum(these) * 1.0 / N * 100)).rjust(5, ' '),
print ""
print "cloudy:",
for type_i in [0, 2, 3, 4, 5, 6, 7]:
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
if N < 50:
print("---").rjust(5, ' '),
else:
these = np.logical_and(is_type_i, pps_ctype > 4)
print("%3.1f" % (np.sum(these) * 1.0 / N * 100)).rjust(5, ' '),
print ""
def plot_ct_table5(caObj, use_in=None):
from get_flag_info import get_calipso_clouds_of_type_i
fig = plt.figure(figsize=(30, 64))
pps_ctype = caObj.avhrr.all_arrays['cloudtype']
use = np.logical_and(pps_ctype > 0, pps_ctype < 23)
use = np.logical_and(use, use_in)
for type_i in [0, 2, 3, 4, 5, 6, 7]:
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
print "N", ("%d" % (N)).rjust(9, ' '),
if N < 50:
print "------------------", cc_type_name[type_i]
continue
these = np.logical_and(is_type_i, pps_ctype < 5)
print("%3.1f" % (np.sum(these) * 1.0 / N * 100)).rjust(5, ' '),
these = np.logical_and(is_type_i, pps_ctype > 4)
print("%3.1f" % (np.sum(these) * 1.0 / N * 100)).rjust(5, ' '),
print cc_type_name[type_i]
def plot_ct_table4(caObj, use_in=None):
from get_flag_info import get_calipso_clouds_of_type_i
fig = plt.figure(figsize=(30, 64))
print "N, land, sea, snow, ice, low, frac, medium, high, cirrus"
for type_i in xrange(0, 8):
if type_i == 1:
continue
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
pps_ctype = caObj.avhrr.all_arrays['cloudtype']
use = np.logical_and(pps_ctype > 0, pps_ctype < 23)
if use_in is not None:
use = np.logical_and(use, use_in)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
print "N", ("%d" % (N)).rjust(9, ' '),
for ind_ct in xrange(0, 9):
pps_ok = np.logical_and(pps_ctype >= ct_min_v[ind_ct],
pps_ctype <= ct_max_v[ind_ct])
these = np.logical_and(is_type_i, pps_ok)
print("%3.1f" % (np.sum(these) * 1.0 / N * 100)).rjust(5, ' '),
print cc_type_name[type_i]
def find_suspicious_files(caObj, filename):
height_c = (1000*caObj.calipso.all_arrays['layer_top_altitude'][:,0] -
caObj.calipso.all_arrays['elevation'])
height_pps = caObj.avhrr.all_arrays['ctth_height']
bias = height_pps - height_c
new_pps_h = calculate_lapse_rate_h(caObj)
bias_l_rate = new_pps_h - height_c
d45 = caObj.avhrr.all_arrays['bt11micron'] - caObj.avhrr.all_arrays['bt12micron']
use = np.logical_and(caObj.avhrr.all_arrays['ctth_height']>-8,
caObj.calipso.all_arrays['layer_top_altitude'][:,0]>-999)
#caObj.calipso.all_arrays['number_layers_found'][:]>0)
for cc_type in xrange(8):
use_this_type = np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=cc_type))
if len(bias[use_this_type])>1:
print cc_type, len(bias[use_this_type]), np.mean(bias[use_this_type]),
print np.mean(bias_l_rate[use_this_type]), np.mean(d45[use_this_type]),
print "%d %d %d %d"%(np.percentile(new_pps_h[use_this_type],10),
np.percentile(new_pps_h[use_this_type],90),
np.percentile(height_pps[use_this_type],10),
np.percentile(height_pps[use_this_type],90))
if len(bias[use_this_type])>2 and np.mean(bias[use_this_type])>20000 :
print filename, cc_type, np.max(bias[use_this_type]), np.percentile(bias[use_this_type],90)
def plot_ct_table3(caObj):
from get_flag_info import get_calipso_clouds_of_type_i
fig = plt.figure(figsize=(30, 64))
print "N, 0 clear clear clear clear very-low, low, medium, high very-high, fractional, cirrus, cirrus, cirrus, cirrus-above-low"
for type_i in xrange(0, 8):
if type_i == 1:
continue
is_type_i = get_calipso_clouds_of_type_i(caObj,
calipso_cloudtype=type_i)
pps_ctype = caObj.avhrr.all_arrays['cloudtype']
use = np.logical_and(pps_ctype > 0, pps_ctype < 23)
is_type_i = np.logical_and(is_type_i, use)
N = np.sum(is_type_i)
print "N", ("%d" % (N)).rjust(9, ' '),
for ind_ct in xrange(0, 9):
pps_ok = np.logical_and(pps_ctype >= ct_min_v[ind_ct],
pps_ctype <= ct_max_v[ind_ct])
these = np.logical_and(is_type_i, pps_ok)
#if type_i==2 and ind_ct==2:
sub_ind = type_i
if sub_ind > 0:
sub_ind -= 1
ax = fig.add_subplot(7, 9, sub_ind * 9 + ind_ct + 1)
frame1 = plt.gca()
frame1.axes.get_xaxis().set_ticks([])
frame1.axes.get_yaxis().set_ticks([])
plt.plot(caObj.avhrr.all_arrays['longitude'][these],
caObj.avhrr.all_arrays['latitude'][these],
'b.',
alpha=0.4,
label="cal-%d,pps-%d-%d" %
(type_i, ct_min_v[ind_ct], ct_max_v[ind_ct]))
#plt.legend()
plt.show()
def calculate_lapse_rate(caObj, filename):
c_height = (1000*caObj.calipso.all_arrays['layer_top_altitude'][:,0] -
caObj.calipso.all_arrays['elevation'])
new_pps_h = calculate_lapse_rate_h(caObj)
pps_height = caObj.avhrr.all_arrays['ctth_height']
#bias = 1000*(pps_height -c_height)
tsur = caObj.avhrr.all_arrays['surftemp']
tsur = caObj.avhrr.all_arrays['segment_nwp_temp'][:,0]
temp_diff = 273.15 + caObj.calipso.all_arrays['layer_top_temperature'][:,0] - tsur
temp_diff_pps = caObj.avhrr.all_arrays['ctth_temperature'][:] - tsur
rate = 1000*temp_diff/c_height
t_test = caObj.avhrr.all_arrays['ttro']-caObj.avhrr.all_arrays['surftemp']
bias_l_rate = new_pps_h - 1000*c_height
use = np.logical_and(caObj.calipso.all_arrays['layer_top_temperature'][:,0]>-999,
caObj.calipso.all_arrays['layer_top_altitude'][:,0]>-999)
use = np.logical_and(use,
caObj.avhrr.all_arrays['ctth_temperature'][:]>0)
#use = np.logical_and(use,
# get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=6))
fig = plt.figure(figsize = (16,10))
ax = fig.add_subplot(321)
plt.plot(temp_diff[use],c_height[use],'.b',alpha=0.01)
plt.plot([0,-120],[0,1000*120/6.5],'k')
ax.set_ylabel('true height')
ax = fig.add_subplot(323)
plt.plot(temp_diff[use],pps_height[use],'.r',alpha=0.01)
plt.plot([0,-120],[0,1000*120/6.5],'k')
ax.set_ylabel('pps height')
ax = fig.add_subplot(325)
plt.plot(temp_diff[use],new_pps_h[use],'.b',alpha=0.01)
plt.plot([0,-120],[0,1000*120/6.5],'k')
ax.set_ylabel('new lapse-rate height')
ax.set_xlabel('true tempdiff')
ax = fig.add_subplot(322)
plt.plot(temp_diff_pps[use],c_height[use],'.c',alpha=0.01)
plt.plot([0,-120],[0,1000*120/6.5],'k')
ax = fig.add_subplot(324)
plt.plot(temp_diff_pps[use],pps_height[use],'.g',alpha=0.1)
ax = fig.add_subplot(326)
plt.plot(temp_diff_pps[use],new_pps_h[use],'.m',alpha=0.01)
plt.plot([0,-120],[0,1000*120/6.5],'k')
ax.set_xlabel('ctth tempdiff')
#plt.plot(caObj.avhrr.all_arrays['ctth_temperature'][use]-tsur[use],1000*c_height[use],'.b',alpha=0.1)
fig = plt.figure(figsize = (16,10))
ax = fig.add_subplot(211)
plt.plot(t_test[use],c_height[use],'.b',alpha=0.1)
ax = fig.add_subplot(212)
plt.plot(t_test[use],pps_height[use],'.r',alpha=0.1)
plt.show()
my_use = np.logical_and(use, temp_diff>0)
my_use = np.logical_and(use, temp_diff<0)
d45 = caObj.avhrr.all_arrays['bt11micron'] - caObj.avhrr.all_arrays['bt12micron']
for cc_type in xrange(8):
my_use = np.logical_and(caObj.avhrr.all_arrays['sunz']>95,my_use)
these = np.logical_and(my_use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=cc_type))
if len(rate[these])>1:
perc= np.percentile(rate[these],[10,50,90])
print cc_type, "%2.1f"%(np.mean(d45[these])), "night", len(rate[these]),
print np.percentile(pps_height[these],95), "%3.1f"%(np.max(c_height[these])),
print "%2.1f %2.1f %2.1f"%(perc[0], perc[1], perc[2])
my_use = np.logical_and(caObj.avhrr.all_arrays['sunz']<60,my_use)
these = np.logical_and(my_use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=cc_type))
if len(rate[these])>1:
perc= np.percentile(rate[these],[10,50,90])
print cc_type, "%2.1f"%(np.mean(d45[these])), "day", len(rate[these]),
print np.percentile(pps_height[these],95), "%3.1f"%(np.max(c_height[these])),
print "%2.1f %2.1f %2.1f"%(perc[0], perc[1], perc[2])
def plot_sub_scatter_plot_bias_diff(caObj,cc_type, use, bias, bias_new, ax,
xmin=-5000, xmax=15000, title=''):
plt.title(cc_type_name[cc_type])
my_use = np.logical_and(use,get_calipso_clouds_of_type_i(caObj, calipso_cloudtype=cc_type))
plot_sub_scatter_plot_bias_diff_inner(caObj, my_use, bias, bias_new, ax,
xmin=xmin, xmax=xmax)
ROOT_DIR = "/home/a001865/DATA_MISC/reshaped_files/clara_a2_rerun/Reshaped_Files_CLARA_A2_final/"
files = glob(ROOT_DIR + "merged/noaa18*h5")
files = files + glob(ROOT_DIR + "merged/noaa19*h5")
#files = glob(ROOT_DIR + "noaa18/5km/20??/??/*/*noaa*h5")
#files = files +glob(ROOT_DIR + "noaa19/5km/20??/??/*/*noaa*h5")
#files = glob(ROOT_DIR + "noaa18/5km/2014/*/*/*noaa*h5")
#files = glob(ROOT_DIR + "noaa18/5km/2006/10/*/*noaa*20061004_1335*h5")
#files = glob(ROOT_DIR + "noaa18/5km/2009/12/*/*noaa*20091220_0727*h5")
#files = files + glob(ROOT_DIR + "noaa18/5km/2010/12/*/*noaa*20101221_0000*h5")
#files = files + glob(ROOT_DIR + "noaa18/5km/2011/10/*/*noaa*20111011_0146*h5")
#files = files + glob(ROOT_DIR + "noaa18/5km/2012/04/*/*noaa*20120415_0319*h5")
caObj = CalipsoAvhrrTrackObject()
from get_flag_info import get_calipso_clouds_of_type_i
for filename in files:
print os.path.basename(filename)
caObj_new=readCaliopAvhrrMatchObj(filename)#, var_to_skip='segment')
print np.sum(get_calipso_clouds_of_type_i(caObj_new, calipso_cloudtype=7))
#plot_height_bias_histograms(caObj_new)
filename = os.path.basename(filename)
#plot_feature_histograms(caObj_new, os.path.basename(filename))
#prototype_ctth_alg_lapse_rate(caObj_new, filename)
#calculate_lapse_rate(caObj_new, filename)
#find_suspicious_files(caObj_new, os.path.basename(filename))
#plot_height_bias_histograms(caObj_new)
caObj = caObj + caObj_new
prototype_ctth_alg_lapse_rate(caObj, filename)
#find_suspicious_files(caObj, os.path.basename(filename))
plot_height_bias_histograms(caObj)
#prototype_ctth_alg_lapse_rate(caObj, filename)
