def investigate_nn_ctth_modis_may():
#november
#may
"""
ROOT_DIR_MODIS_nn = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_MAY/Reshaped_Files/merged/")
ROOT_DIR_MODIS_nn_viirs = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_MAY_nnviirs_20161205/Reshaped_Files/merged/")
ROOT_DIR_MODIS_nn_mersi2 = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_MAY_nnmersi2_20161206/Reshaped_Files/merged/")
"""
ROOT_DIR_MODIS_old = (
"/home/a001865/DATA_MISC/reshaped_files/"
"global_modis_14th_created20161108/Reshaped_Files/merged/")
caobj_dict = {}
for ROOT_DIR, name in zip(
[ROOT_DIR_MODIS_nn, ROOT_DIR_MODIS_nn_viirs,
ROOT_DIR_MODIS_nn_mersi2, ROOT_DIR_MODIS_old],
["modis_november_CTTHnn_AVHRR",
"modis_november_CTTHnn_VIIRS",
"modis_november_CTTHnn_MERSI2",
"modis_november_CTTHold"]):
print name
files = glob(ROOT_DIR + "/*11*.h5")
caObj = CalipsoAvhrrTrackObject()
for filename in files:
#print filename
caObj += readCaliopAvhrrMatchObj(filename)
caobj_dict[name] = caObj
make_boxplot(caObj, name)
def investigate_nn_ctth_modis_lvl2():
#november
ROOT_DIR_MODIS_nn_avhrr = (
"/home/a001865/DATA_MISC/reshaped_files/"
"global_modis_14th_created20170324/Reshaped_Files_merged/eos2/1km/2010/%s/*h5"
)
ROOT_DIR_MODIS_old = (
"/home/a001865/DATA_MISC/reshaped_files/"
"global_modis_14th_created20161108/Reshaped_Files/merged/*%s*h5")
for month in ["06", "09", "01"]:
for ROOT_DIR, name in zip([
ROOT_DIR_MODIS_nn_avhrr, ROOT_DIR_MODIS_nn_avhrr,
ROOT_DIR_MODIS_old
], ["modis_nnAVHRR", "modis_lvl2_C6", "modis_CTTHold"]):
name = "%s_%s" % (name, month)
print ROOT_DIR
files = glob(ROOT_DIR % (month))
caObj = CalipsoAvhrrTrackObject()
for filename in files:
#print filename
caObj += readCaliopAvhrrMatchObj(filename)
modis_lvl2 = False
if "modis_lvl2" in name:
modis_lvl2 = True
make_violinplot(caObj, name, modis_lvl2=modis_lvl2)
make_violinplot_pressure(caObj, name, modis_lvl2=modis_lvl2)
make_violinplot_temperature(caObj, name, modis_lvl2=modis_lvl2)
def investigate_nn_ctth():
ROOT_DIR_GAC_nn = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nn21/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_old = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_old/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn_new = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nn20161125/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn_18 = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nn20161130/Reshaped_Files/noaa18/")
re_name = re.compile("_RESULTS_GAC_(\w+)\/")
caobj_dict = {}
for ROOT_DIR, name in zip([
ROOT_DIR_GAC_nn, ROOT_DIR_GAC_old, ROOT_DIR_GAC_nn_new,
ROOT_DIR_GAC_nn_18
], [
"gac_CTTHnn_21", "gac_CTTHold", "gac_CTTHnn_20161125",
"gac_CTTHnn_AVHRR"
]):
files = glob(ROOT_DIR + "5km/2009/*/*/*h5")
caObj = CalipsoAvhrrTrackObject()
for filename in files:
caObj += readCaliopAvhrrMatchObj(filename)
caobj_dict[name] = caObj
make_boxplot(caObj, name)
def investigate_nn_ctth_modis_november():
#november
ROOT_DIR_MODIS_nn_avhrr = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_NOVEMBER/Reshaped_Files/merged/")
ROOT_DIR_MODIS_nn_viirs = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_NOVEMBER_nnviirs_20161205/Reshaped_Files/merged/")
ROOT_DIR_MODIS_nn_mersi2 = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_NOVEMBER_AEROSOL/Reshaped_Files/merged/")
ROOT_DIR_MODIS_nn_viirs_tuned = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_NOVEMBER_nnVIIRS_20170315/Reshaped_Files/merged/")
ROOT_DIR_MODIS_nn_mersi2_tuned = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_NOVEMBER_nnMERSI2/Reshaped_Files/merged/")
ROOT_DIR_MODIS_nn_avhrr_tuned = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_MODIS_NOVEMBER_nnAVHRR_20170315/Reshaped_Files/merged/")
ROOT_DIR_MODIS_old = (
"/home/a001865/DATA_MISC/reshaped_files/"
"global_modis_14th_created20161108/Reshaped_Files/merged/")
caobj_dict = {}
for ROOT_DIR, name in zip(
[ROOT_DIR_MODIS_nn_avhrr,
ROOT_DIR_MODIS_nn_viirs,
ROOT_DIR_MODIS_nn_mersi2,
ROOT_DIR_MODIS_old,
ROOT_DIR_MODIS_nn_viirs_tuned,
ROOT_DIR_MODIS_nn_mersi2_tuned,
ROOT_DIR_MODIS_nn_avhrr_tuned],
["modis_nov_nnAVHRR",
"modis_nov_nnVIIRS",
"modis_nov_nnMERSI2",
"modis_nov_CTTHold",
"modis_nov_nnVIIRS_tuned",
"modis_nov_nnMERSI2_tuned",
"modis_nov_nnAVHRR_tuned",
]):
files = glob(ROOT_DIR + "/*11*.h5")
caObj = CalipsoAvhrrTrackObject()
for filename in files:
#print filename
caObj += readCaliopAvhrrMatchObj(filename)
caobj_dict[name] = caObj
make_boxplot(caObj, name)
def investigate_nn_ctth_viirs():
ROOT_DIR_14bug_maia = (
"/home/a001865/DATA_MISC/reshaped_files/"
"NPP_FULL_ORBIT_2014/Reshaped_Files/")
ROOT_DIR_14bug = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_v2014_before_ctthbug_correction/"
"Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_v2014 = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_v2014_bug_corrected_20170313/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_avhrr = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_nnAVHRR_20170313/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_avhrr_tuned = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_nnAVHRR_20170313_tuned/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_avhrr1 = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_nnAVHRR1_20170313/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_viirs = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_nnVIIRS_20170310/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_viirs_new = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_nnVIIRS_20170313_new/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_viirs_CLAY4 = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_nnVIIRS_20170310_CLAY4/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_viirs_lm = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_viirs_lm/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_avhrr_lm = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_avhrr_lm/Reshaped_Files/npp/1km/2015/07/*/")
ROOT_DIR_nn_avhrr_wg = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_NPP_AVHRR_with_gac/Reshaped_Files/npp/1km/2015/07/*/")
caobj_dict = {}
for ROOT_DIR, name in zip(
[ROOT_DIR_nn_avhrr_wg, ROOT_DIR_nn_avhrr1,ROOT_DIR_nn_avhrr,ROOT_DIR_nn_avhrr_tuned, ROOT_DIR_v2014, ROOT_DIR_14bug, ROOT_DIR_14bug_maia, ROOT_DIR_nn_viirs, ROOT_DIR_nn_viirs_CLAY4, ROOT_DIR_nn_viirs_new,ROOT_DIR_nn_viirs_lm,ROOT_DIR_nn_avhrr_lm],
["npp_CTTHnn_AVHRR_with_gac", "npp_CTTHnn_AVHRR1","npp_CTTHnn_AVHRR","npp_CTTHnn_AVHRR_tuned", "npp_CTTHv2014","npp_CTTHv2014_buggy","npp_CTTHv2014_buggy_maia","npp_CTTHnn_VIIRS","npp_CTTHnn_VIIRS_C4","npp_CTTHnn_VIIRS_tuned", "npp_nnVIIRS_LessIsMore", "npp_nnAVHRR_LessIsMore"]):
#print ROOT_DIR
files = glob(ROOT_DIR + "*.h5")
caObj = CalipsoAvhrrTrackObject()
for filename in files:
#print filename
caObj += readCaliopAvhrrMatchObj(filename)
caobj_dict[name] = caObj
make_boxplot(caObj, name)
def investigate_nn_ctth():
ROOT_DIR_GAC_nnNina = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nnNina/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nn21/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_old = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_old/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_oldCTTH_12x12 = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_oldCTTH_12x12/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_v2014 = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_v2014/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_v2014_12x12 = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_v2014_12x12/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn_new = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nn20161125/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn_avhrr = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nn20161130/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn_avhrr_tuned = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_tuned_nnAVHRR/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn_avhrr1_tuned = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nnAVHRR1/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_nn_avhrr_with_gac = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_AVHRR_with_gac/Reshaped_Files/noaa18/")
re_name = re.compile("_RESULTS_GAC_(\w+)\/")
caobj_dict = {}
for ROOT_DIR, name in zip([ROOT_DIR_GAC_nnNina, ROOT_DIR_GAC_nn, ROOT_DIR_GAC_old, ROOT_DIR_GAC_v2014,
ROOT_DIR_GAC_nn_new, ROOT_DIR_GAC_nn_avhrr,
ROOT_DIR_GAC_nn_avhrr_tuned, ROOT_DIR_GAC_nn_avhrr1_tuned,
ROOT_DIR_GAC_v2014_12x12,
ROOT_DIR_GAC_oldCTTH_12x12,
ROOT_DIR_GAC_nn_avhrr_with_gac],
["gac_nnLessIsMore","gac_nn21", "gac_CTTHold","gac_2014",
"gac_nn20161125", "gac_nnAVHRR",
"gac_nnAVHRR_tuned", "gac_nnAVHRR1_tuned",
"gac_v2014_12x12","gac_CTTHold_12x12", "gac_17var_modis_noaa19"]):
files = glob(ROOT_DIR + "5km/2009/*/*/*h5")
caObj = CalipsoAvhrrTrackObject()
for filename in files:
caObj += readCaliopAvhrrMatchObj(filename)
caobj_dict[name] = caObj
make_boxplot(caObj, name)
"ATRAIN_RESULTS_GAC/Reshaped_Files/noaa18/")
files = glob(ROOT_DIR + "Reshaped_Files/merged/modis*h5")
files = glob(ROOT_DIR_GAC + "5km/2009/*/*/*h5")
test_list_file = "/home/a001865/git/acpg_develop/acpg/pges/cloudmask/pps_pge01_cmasktests.h"
TEST_NAMEFILE = open(test_list_file, 'r')
name_dict = {
'cma_testlist0': {},
'cma_testlist1': {},
'cma_testlist2': {},
'cma_testlist3': {},
'cma_testlist4': {},
'cma_testlist5': {}
}
for line in TEST_NAMEFILE:
if 'define SM_ACMG_' in line:
list_of_line = line.split(' ')
(name, list_nr, bit) = (list_of_line[1].replace('SM_ACMG_', ''),
list_of_line[2].replace(',',
''), list_of_line[3])
var = 'cma_testlist' + list_nr
name_dict[var][int(bit)] = name
caObjPPS = CalipsoAvhrrTrackObject()
for filename in files:
caObjPPS += readCaliopAvhrrMatchObj(filename)
print "Scene %s" % (os.path.basename(filename))
use = caObjPPS.avhrr.all_arrays['bt11micron'] > -9
print_common_stats(caObjPPS, use, name_dict)
print "PPS", y_month, satellite, int(min(sunz[use_pps])), int(
max(sunz[use_pps])),
get_clear_cloudy_vectors(caObjPPS, use_pps) #, both_have_ct)
MAIA_ROOT_DIR = ("/home/a001865/VALIDATIONS/MAIA/Reshaped_Files/" "*/1km/")
PPS_ROOT_DIR = ("/home/a001865/VALIDATIONS/PPS/Reshaped_Files/" "*/1km/")
print "Algorithm time satellite min-sunz, max-sunz, N PODcloudy FARcloudy PODclear FARclear Hitrate Kuipers Max-time-diff"
for y_month in [
'2012/06', '2012/07', '2012/08', '2012/10', '2015/1201', '2015/1207',
'2015/07'
]:
maia_files = glob(MAIA_ROOT_DIR + "%s/*/*h5" % (y_month))
pps_files = glob(PPS_ROOT_DIR + "%s/*/*h5" % (y_month))
caObjMAIA = CalipsoAvhrrTrackObject()
caObjPPS = CalipsoAvhrrTrackObject()
for filename in maia_files:
caObjMAIA += readCaliopAvhrrMatchObj(filename)
for filename in pps_files:
caObjPPS += readCaliopAvhrrMatchObj(filename)
satellite = 'Suomi-NPP'
if y_month in ['2015/1201', '2015/1207']:
satellite = 'Metop-B'
for dnt in ['day and twilight', 'night']:
#print "Year, month, satellite %s %s %s"%(y_month, satellite, dnt)
print_common_stats(caObjMAIA,
caObjPPS,
y_month,
satellite,
FAR_high = 100 * (np.sum(pps_high_i) - n_high_ok) / np.sum(pps_high_i)
print "%3.1f %3.1f %3.1f %3.1f %3.1f %3.1f" % (
POD_low, POD_medium, POD_high, FAR_low, FAR_medium, FAR_high)
# ----------------------------------------
if __name__ == "__main__":
isModis1km = True
isNPP_v2014 = False
isGAC_v2014_morning_sat = False
isGAC_v2014 = True
ROOT_DIR_GAC_nn = (
"/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_nnavhrr_20161202/Reshaped_Files/noaa18/")
ROOT_DIR_GAC_old = ("/home/a001865/DATA_MISC/reshaped_files/"
"ATRAIN_RESULTS_GAC_v2014/Reshaped_Files/noaa18/")
files = glob(ROOT_DIR_GAC_nn + "5km/2009/*/*/*h5")
caObj = CalipsoAvhrrTrackObject()
for filename in files:
#print os.path.basename(filename)
caObj += readCaliopAvhrrMatchObj(filename) #, var_to_skip='segment')
#table_5_per_illumination(caObj)
plot_ct_table4(caObj)
plot_ct_table2(caObj)
table_21_do_for_atbd(caObj)
#plot_ct_table3(caObj)
def match_calipso_avhrr(values,
calipsoObj, calipsoObjAerosol,
imagerGeoObj, imagerObj,
ctype, cma, ctth, cppCph, nwp_obj,
avhrrAngObj, nwp_segments, options, res=resolution):
import time
import string
retv = CalipsoAvhrrTrackObject()
dsec = time.mktime((1993,1,1,0,0,0,0,0,0)) - time.timezone # Convert from TAI time to UTC in seconds since 1970
if res == 1:
lonCalipso = calipsoObj.longitude.ravel()
latCalipso = calipsoObj.latitude.ravel()
timeCalipso_tai = calipsoObj.profile_time_tai[::,0].ravel()
timeCalipso = calipsoObj.profile_time_tai[::,0].ravel() + dsec
timeCalipso_utc = calipsoObj.profile_utc_time[::,0].ravel()
elevationCalipso = calipsoObj.dem_surface_elevation.ravel()
if res == 5:
# Use [:,1] Since 5km data has start, center, and end for each pixel
lonCalipso = calipsoObj.longitude[:,1].ravel()
latCalipso = calipsoObj.latitude[:,1].ravel()
timeCalipso_tai = calipsoObj.profile_time_tai[:,1].ravel()
timeCalipso = calipsoObj.profile_time_tai[:,1].ravel() + dsec
timeCalipso_utc = calipsoObj.profile_utc_time[:,1].ravel()
elevationCalipso = calipsoObj.dem_surface_elevation[::,2].ravel()
ndim = lonCalipso.shape[0]
# --------------------------------------------------------------------
#cal,cap = get_calipso_avhrr_linpix(imagerGeoObj,values,lonCalipso,latCalipso,timeCalipso, options)
# This function (match_calipso_avhrr) could use the MatchMapper object
# created in map_avhrr() to make things a lot simpler... See usage in
# amsr_avhrr_match.py
#Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
from common import map_avhrr
cal, cap = map_avhrr(imagerGeoObj, lonCalipso.ravel(), latCalipso.ravel(),
radius_of_influence=RESOLUTION*0.7*1000.0) # somewhat larger than radius...
print cal, cap
calnan = np.where(cal == NODATA, np.nan, cal)
if (~np.isnan(calnan)).sum() == 0:
raise MatchupError("No matches within region.")
if len(imagerGeoObj.time.shape)>1:
imager_time_vector = [imagerGeoObj.time[line,pixel] for line, pixel in zip(cal,cap)]
avhrr_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector, np.nan)
else:
avhrr_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal], np.nan)
# avhrr_lines_sec_1970 = calnan * DSEC_PER_AVHRR_SCALINE + imagerGeoObj.sec1970_start
# Find all matching Calipso pixels within +/- sec_timeThr from the AVHRR data
# pdb.set_trace()
"""
import matplotlib.pyplot as plt
fig = plt.figure(figsize = (9,8))
ax = fig.add_subplot(111)
plt.plot(timeCalipso, 'b.')
plt.plot(avhrr_lines_sec_1970, 'g.')
plt.show()
fig.savefig("nina_test_fil.png")
"""
idx_match = elements_within_range(timeCalipso, avhrr_lines_sec_1970, sec_timeThr)
if idx_match.sum() == 0:
raise MatchupError("No matches in region within time threshold %d s." % sec_timeThr)
retv.calipso = calipso_track_from_matched(retv.calipso, calipsoObj, idx_match)
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
retv.calipso.avhrr_linnum = cal_on_avhrr.astype('i')
retv.calipso.avhrr_pixnum = cap_on_avhrr.astype('i')
logger.info("cap_on_avhrr.shape: %s",cap_on_avhrr.shape)
lon_calipso = np.repeat(lonCalipso, idx_match)
lat_calipso = np.repeat(latCalipso, idx_match)
# Calipso line,pixel inside AVHRR swath:
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
logger.info("Start and end times: %s %s",
time.gmtime(timeCalipso[0]),
time.gmtime(timeCalipso[ndim-1]))
retv.calipso.sec_1970 = np.repeat(timeCalipso,idx_match)
retv.calipso.latitude = np.repeat(latCalipso,idx_match)
retv.calipso.longitude = np.repeat(lonCalipso,idx_match)
retv.calipso.profile_time_tai = np.repeat(timeCalipso_tai,idx_match)
# Elevation is given in km's. Convert to meters:
retv.calipso.elevation = np.repeat(elevationCalipso.ravel()*1000.0,
idx_match.ravel()).astype('d')
# Time
if len(imagerGeoObj.time.shape)>1:
retv.avhrr.sec_1970= [imagerGeoObj.time[line,pixel] for line, pixel in zip(cal_on_avhrr,cap_on_avhrr)]
else:
retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
retv.diff_sec_1970 = retv.calipso.sec_1970 - retv.avhrr.sec_1970
min_diff = np.minimum.reduce(retv.diff_sec_1970)
max_diff = np.maximum.reduce(retv.diff_sec_1970)
logger.info("Maximum and minimum time differences in sec (avhrr-calipso): %d %d",
np.maximum.reduce(retv.diff_sec_1970),np.minimum.reduce(retv.diff_sec_1970))
logger.info("AVHRR observation time of first calipso-avhrr match: %s",
time.gmtime(retv.avhrr.sec_1970[0]))
logger.info("AVHRR observation time of last calipso-avhrr match: %s",
time.gmtime(retv.avhrr.sec_1970[-1]))
# Make the latitude and pps cloudtype on the calipso track:
# line and pixel arrays have equal dimensions
logger.info("Generate the latitude,cloudtype tracks!")
# -------------------------------------------------------------------------
# Pick out the data from the track from AVHRR
from extract_imager_along_track import avhrr_track_from_matched
retv = avhrr_track_from_matched(retv, imagerGeoObj, imagerObj, avhrrAngObj,
nwp_obj, ctth, ctype, cma, cal_on_avhrr,
cap_on_avhrr, avhrrCph=cppCph,
nwp_segments=nwp_segments)
if calipsoObjAerosol is not None:
retv.calipso_aerosol = calipso_track_from_matched(retv.calipso_aerosol, calipsoObjAerosol, idx_match)
# -------------------------------------------------------------------------
logger.info("AVHRR-PPS Cloud Type,latitude: shapes = %s %s",
retv.avhrr.cloudtype.shape,retv.avhrr.latitude.shape)
ll = []
for i in range(ndim):
#ll.append(("%7.3f %7.3f %d\n"%(lonCalipso[i],latCalipso[i],0)))
ll.append(("%7.3f %7.3f %d\n"%(lonCalipso[i],latCalipso[i],idx_match[i])))
max_cloud_top_calipso = np.maximum.reduce(retv.calipso.layer_top_altitude.ravel())
logger.info("max_cloud_top_calipso: %2.1f",max_cloud_top_calipso)
return retv,min_diff,max_diff
def match_calipso_avhrr(values,
caObj,
caObjAerosol,
imagerGeoObj,
imagerObj,
ctype,
cma,
ctth,
cpp,
nwp_obj,
avhrrAngObj,
nwp_segments,
options,
res=resolution):
import string
from common import map_avhrr
retv = CalipsoAvhrrTrackObject()
lonCalipso = caObj.longitude.ravel()
latCalipso = caObj.latitude.ravel()
#Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
cal, cap = map_avhrr(imagerGeoObj,
lonCalipso.ravel(),
latCalipso.ravel(),
radius_of_influence=RESOLUTION * 0.7 *
1000.0) # somewhat larger than radius...
#warn if no matches
calnan = np.where(cal == NODATA, np.nan, cal)
if (~np.isnan(calnan)).sum() == 0:
raise MatchupError("No matches within region.")
#check if it is within time limits:
if len(imagerGeoObj.time.shape) > 1:
imager_time_vector = [
imagerGeoObj.time[line, pixel] for line, pixel in zip(cal, cap)
]
imager_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector,
np.nan)
else:
imager_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal],
np.nan)
idx_match = elements_within_range(caObj.sec_1970, imager_lines_sec_1970,
sec_timeThr)
if idx_match.sum() == 0:
raise MatchupError("No matches in region within time threshold %d s." %
sec_timeThr)
retv.calipso = calipso_track_from_matched(retv.calipso, caObj, idx_match)
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
retv.calipso.avhrr_linnum = cal_on_avhrr.astype('i')
retv.calipso.avhrr_pixnum = cap_on_avhrr.astype('i')
#logger.info("calipso matched with avhrr shape: %s",cap_on_avhrr.shape)
# Calipso line,pixel inside AVHRR swath:
cal_on_avhrr = np.repeat(cal, idx_match)
cap_on_avhrr = np.repeat(cap, idx_match)
# Imager time
if len(imagerGeoObj.time.shape) > 1:
retv.avhrr.sec_1970 = [
imagerGeoObj.time[line, pixel]
for line, pixel in zip(cal_on_avhrr, cap_on_avhrr)
]
else:
retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
retv.diff_sec_1970 = retv.calipso.sec_1970 - retv.avhrr.sec_1970
do_some_logging(retv, caObj)
logger.info("Generate the latitude,cloudtype tracks!")
from extract_imager_along_track import avhrr_track_from_matched
retv = avhrr_track_from_matched(retv,
imagerGeoObj,
imagerObj,
avhrrAngObj,
nwp_obj,
ctth,
ctype,
cma,
cal_on_avhrr,
cap_on_avhrr,
cpp=cpp,
nwp_segments=nwp_segments)
if caObjAerosol is not None:
retv.calipso_aerosol = calipso_track_from_matched(
retv.calipso_aerosol, caObjAerosol, idx_match)
max_cloud_top_calipso = np.maximum.reduce(
retv.calipso.layer_top_altitude.ravel())
logger.info("max_cloud_top_calipso: %2.1f", max_cloud_top_calipso)
return retv
#BASE_DIR = "/home/a001865/DATA_MISC/reshaped_files/ATRAIN_RESULTS_MODIS_NOVEMBER_nnAVHRR_with_gac"
ROOT_DIR = BASE_DIR + "/Reshaped_Files/eos2/1km/2010/%s/*/*caliop*.h5"
OUT_DIR_TEMPLATE = BASE_DIR + "/Reshaped_Files_merged/eos2/1km/2010/%s/"
outfile_template = "1km_%s_eos2_2010%s14_0000_00000_caliop_modis_match.h5"
BASE_DIR = "/home/a001865/DATA_MISC/reshaped_files/global_modis_01st_created20170519"
ROOT_DIR = BASE_DIR + "/Reshaped_Files/eos2/1km/2010/%s/*/*caliop*.h5"
OUT_DIR_TEMPLATE = BASE_DIR + "/Reshaped_Files_merged_calipso_cbase/eos2/1km/2010/%s/"
outfile_template = "1km_%s_eos2_2010%s01_0000_00000_caliop_modis_match.h5"
BASE_DIR = "/home/a001865/DATA_MISC/reshaped_files/global_modis_14th_created20171121"
ROOT_DIR = BASE_DIR + "/Reshaped_Files/eos2/1km/2010/%s/*/*caliop*.h5"
OUT_DIR_TEMPLATE = BASE_DIR + "/Reshaped_Files_merged_calipso_cbase//eos2/1km/2010/%s/"
outfile_template = "1km_%s_eos2_2010%s14_0000_00000_caliop_modis_match.h5"
caObj_night = CalipsoAvhrrTrackObject()
caObj_day = CalipsoAvhrrTrackObject()
for year in [
"2010"
]: #2012/02","2012/05", "2012/08", "2013/07", "2014/02", "2014/04", "2014/09"]:
#for month in ["01","02","03","04","05","06","07","08","09","10","11","12"]:
#for month in ["06"]:
for month in [
"02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12",
"01"
]:
OUT_DIR = OUT_DIR_TEMPLATE % (month)
if not os.path.exists(OUT_DIR):
os.makedirs(OUT_DIR)