Esempio n. 1
0
def investigate_nn_ctth_modis_may():
   #november
    #may
    """
    ROOT_DIR_MODIS_nn = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_MAY/Reshaped_Files/merged/")
    ROOT_DIR_MODIS_nn_viirs = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_MAY_nnviirs_20161205/Reshaped_Files/merged/")
    ROOT_DIR_MODIS_nn_mersi2 = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_MAY_nnmersi2_20161206/Reshaped_Files/merged/")
    """
    ROOT_DIR_MODIS_old = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "global_modis_14th_created20161108/Reshaped_Files/merged/")

    caobj_dict = {}
    for ROOT_DIR, name in zip(
            [ROOT_DIR_MODIS_nn, ROOT_DIR_MODIS_nn_viirs, 
             ROOT_DIR_MODIS_nn_mersi2, ROOT_DIR_MODIS_old], 
            ["modis_november_CTTHnn_AVHRR",
             "modis_november_CTTHnn_VIIRS",
             "modis_november_CTTHnn_MERSI2",
             "modis_november_CTTHold"]):
        print name
        files = glob(ROOT_DIR + "/*11*.h5")
        caObj = CalipsoAvhrrTrackObject()
        for filename in files:
            #print filename
            caObj +=  readCaliopAvhrrMatchObj(filename) 
        caobj_dict[name] = caObj    
        make_boxplot(caObj, name) 
def investigate_nn_ctth_modis_lvl2():
    #november

    ROOT_DIR_MODIS_nn_avhrr = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "global_modis_14th_created20170324/Reshaped_Files_merged/eos2/1km/2010/%s/*h5"
    )

    ROOT_DIR_MODIS_old = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "global_modis_14th_created20161108/Reshaped_Files/merged/*%s*h5")

    for month in ["06", "09", "01"]:
        for ROOT_DIR, name in zip([
                ROOT_DIR_MODIS_nn_avhrr, ROOT_DIR_MODIS_nn_avhrr,
                ROOT_DIR_MODIS_old
        ], ["modis_nnAVHRR", "modis_lvl2_C6", "modis_CTTHold"]):
            name = "%s_%s" % (name, month)
            print ROOT_DIR
            files = glob(ROOT_DIR % (month))
            caObj = CalipsoAvhrrTrackObject()
            for filename in files:
                #print filename
                caObj += readCaliopAvhrrMatchObj(filename)
            modis_lvl2 = False
            if "modis_lvl2" in name:
                modis_lvl2 = True
            make_violinplot(caObj, name, modis_lvl2=modis_lvl2)
            make_violinplot_pressure(caObj, name, modis_lvl2=modis_lvl2)
            make_violinplot_temperature(caObj, name, modis_lvl2=modis_lvl2)
Esempio n. 3
0
def investigate_nn_ctth():
    ROOT_DIR_GAC_nn = ("/home/a001865/DATA_MISC/reshaped_files/"
                       "ATRAIN_RESULTS_GAC_nn21/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_old = ("/home/a001865/DATA_MISC/reshaped_files/"
                        "ATRAIN_RESULTS_GAC_old/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn_new = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_GAC_nn20161125/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn_18 = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_GAC_nn20161130/Reshaped_Files/noaa18/")

    re_name = re.compile("_RESULTS_GAC_(\w+)\/")
    caobj_dict = {}
    for ROOT_DIR, name in zip([
            ROOT_DIR_GAC_nn, ROOT_DIR_GAC_old, ROOT_DIR_GAC_nn_new,
            ROOT_DIR_GAC_nn_18
    ], [
            "gac_CTTHnn_21", "gac_CTTHold", "gac_CTTHnn_20161125",
            "gac_CTTHnn_AVHRR"
    ]):
        files = glob(ROOT_DIR + "5km/2009/*/*/*h5")
        caObj = CalipsoAvhrrTrackObject()
        for filename in files:
            caObj += readCaliopAvhrrMatchObj(filename)
        caobj_dict[name] = caObj
        make_boxplot(caObj, name)
Esempio n. 4
0
def investigate_nn_ctth_modis_november():
    #november
    ROOT_DIR_MODIS_nn_avhrr = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_NOVEMBER/Reshaped_Files/merged/")
    ROOT_DIR_MODIS_nn_viirs = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_NOVEMBER_nnviirs_20161205/Reshaped_Files/merged/")
    ROOT_DIR_MODIS_nn_mersi2 = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_NOVEMBER_AEROSOL/Reshaped_Files/merged/")
    ROOT_DIR_MODIS_nn_viirs_tuned = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_NOVEMBER_nnVIIRS_20170315/Reshaped_Files/merged/")
    ROOT_DIR_MODIS_nn_mersi2_tuned = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_NOVEMBER_nnMERSI2/Reshaped_Files/merged/")
    ROOT_DIR_MODIS_nn_avhrr_tuned = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_MODIS_NOVEMBER_nnAVHRR_20170315/Reshaped_Files/merged/")

    ROOT_DIR_MODIS_old = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "global_modis_14th_created20161108/Reshaped_Files/merged/")

    caobj_dict = {}
    for ROOT_DIR, name in zip(
            [ROOT_DIR_MODIS_nn_avhrr, 
             ROOT_DIR_MODIS_nn_viirs, 
             ROOT_DIR_MODIS_nn_mersi2, 
             ROOT_DIR_MODIS_old,
             ROOT_DIR_MODIS_nn_viirs_tuned,  
             ROOT_DIR_MODIS_nn_mersi2_tuned,
             ROOT_DIR_MODIS_nn_avhrr_tuned], 
            ["modis_nov_nnAVHRR",
             "modis_nov_nnVIIRS",
             "modis_nov_nnMERSI2",
             "modis_nov_CTTHold",

             "modis_nov_nnVIIRS_tuned",
             "modis_nov_nnMERSI2_tuned",
             "modis_nov_nnAVHRR_tuned",
]):
        files = glob(ROOT_DIR + "/*11*.h5")
        caObj = CalipsoAvhrrTrackObject()
        for filename in files:
            #print filename
            caObj +=  readCaliopAvhrrMatchObj(filename) 
        caobj_dict[name] = caObj    
        make_boxplot(caObj, name) 
Esempio n. 5
0
def investigate_nn_ctth_viirs():
    ROOT_DIR_14bug_maia = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "NPP_FULL_ORBIT_2014/Reshaped_Files/")
    ROOT_DIR_14bug = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_v2014_before_ctthbug_correction/"
        "Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_v2014 = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_v2014_bug_corrected_20170313/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_avhrr = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_nnAVHRR_20170313/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_avhrr_tuned = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_nnAVHRR_20170313_tuned/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_avhrr1 = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_nnAVHRR1_20170313/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_viirs = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_nnVIIRS_20170310/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_viirs_new = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_nnVIIRS_20170313_new/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_viirs_CLAY4 = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_nnVIIRS_20170310_CLAY4/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_viirs_lm = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_viirs_lm/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_avhrr_lm = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_avhrr_lm/Reshaped_Files/npp/1km/2015/07/*/")
    ROOT_DIR_nn_avhrr_wg = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_NPP_AVHRR_with_gac/Reshaped_Files/npp/1km/2015/07/*/")
    caobj_dict = {}
    for ROOT_DIR, name in zip(
            [ROOT_DIR_nn_avhrr_wg, ROOT_DIR_nn_avhrr1,ROOT_DIR_nn_avhrr,ROOT_DIR_nn_avhrr_tuned, ROOT_DIR_v2014, ROOT_DIR_14bug, ROOT_DIR_14bug_maia, ROOT_DIR_nn_viirs, ROOT_DIR_nn_viirs_CLAY4, ROOT_DIR_nn_viirs_new,ROOT_DIR_nn_viirs_lm,ROOT_DIR_nn_avhrr_lm], 
            ["npp_CTTHnn_AVHRR_with_gac", "npp_CTTHnn_AVHRR1","npp_CTTHnn_AVHRR","npp_CTTHnn_AVHRR_tuned", "npp_CTTHv2014","npp_CTTHv2014_buggy","npp_CTTHv2014_buggy_maia","npp_CTTHnn_VIIRS","npp_CTTHnn_VIIRS_C4","npp_CTTHnn_VIIRS_tuned", "npp_nnVIIRS_LessIsMore", "npp_nnAVHRR_LessIsMore"]):
        #print ROOT_DIR
        files = glob(ROOT_DIR + "*.h5")
        caObj = CalipsoAvhrrTrackObject()
        for filename in files:
            #print filename
            caObj +=  readCaliopAvhrrMatchObj(filename) 
        caobj_dict[name] = caObj    
        make_boxplot(caObj, name) 
Esempio n. 6
0
def investigate_nn_ctth():
    ROOT_DIR_GAC_nnNina = ("/home/a001865/DATA_MISC/reshaped_files/"
                       "ATRAIN_RESULTS_GAC_nnNina/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn = ("/home/a001865/DATA_MISC/reshaped_files/"
                       "ATRAIN_RESULTS_GAC_nn21/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_old = ("/home/a001865/DATA_MISC/reshaped_files/"
                        "ATRAIN_RESULTS_GAC_old/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_oldCTTH_12x12 = ("/home/a001865/DATA_MISC/reshaped_files/"
                        "ATRAIN_RESULTS_GAC_oldCTTH_12x12/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_v2014 = ("/home/a001865/DATA_MISC/reshaped_files/"
                        "ATRAIN_RESULTS_GAC_v2014/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_v2014_12x12 = ("/home/a001865/DATA_MISC/reshaped_files/"
                                "ATRAIN_RESULTS_GAC_v2014_12x12/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn_new = ("/home/a001865/DATA_MISC/reshaped_files/"
                           "ATRAIN_RESULTS_GAC_nn20161125/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn_avhrr = ("/home/a001865/DATA_MISC/reshaped_files/"
                           "ATRAIN_RESULTS_GAC_nn20161130/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn_avhrr_tuned = ("/home/a001865/DATA_MISC/reshaped_files/"
                           "ATRAIN_RESULTS_GAC_tuned_nnAVHRR/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn_avhrr1_tuned = ("/home/a001865/DATA_MISC/reshaped_files/"
                           "ATRAIN_RESULTS_GAC_nnAVHRR1/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_nn_avhrr_with_gac = ("/home/a001865/DATA_MISC/reshaped_files/"
                           "ATRAIN_RESULTS_GAC_AVHRR_with_gac/Reshaped_Files/noaa18/")
    re_name = re.compile("_RESULTS_GAC_(\w+)\/")
    caobj_dict = {}
    for ROOT_DIR, name in zip([ROOT_DIR_GAC_nnNina, ROOT_DIR_GAC_nn, ROOT_DIR_GAC_old, ROOT_DIR_GAC_v2014, 
                               ROOT_DIR_GAC_nn_new, ROOT_DIR_GAC_nn_avhrr, 
                               ROOT_DIR_GAC_nn_avhrr_tuned, ROOT_DIR_GAC_nn_avhrr1_tuned, 
                               ROOT_DIR_GAC_v2014_12x12,
                               ROOT_DIR_GAC_oldCTTH_12x12,
                               ROOT_DIR_GAC_nn_avhrr_with_gac],
                              ["gac_nnLessIsMore","gac_nn21", "gac_CTTHold","gac_2014", 
                               "gac_nn20161125", "gac_nnAVHRR", 
                               "gac_nnAVHRR_tuned", "gac_nnAVHRR1_tuned", 
                               "gac_v2014_12x12","gac_CTTHold_12x12", "gac_17var_modis_noaa19"]):
        files = glob(ROOT_DIR + "5km/2009/*/*/*h5")
        caObj = CalipsoAvhrrTrackObject()
        for filename in files:
            caObj +=  readCaliopAvhrrMatchObj(filename) 
        caobj_dict[name] = caObj    
        make_boxplot(caObj, name) 
Esempio n. 7
0
                "ATRAIN_RESULTS_GAC/Reshaped_Files/noaa18/")

files = glob(ROOT_DIR + "Reshaped_Files/merged/modis*h5")
files = glob(ROOT_DIR_GAC + "5km/2009/*/*/*h5")

test_list_file = "/home/a001865/git/acpg_develop/acpg/pges/cloudmask/pps_pge01_cmasktests.h"
TEST_NAMEFILE = open(test_list_file, 'r')
name_dict = {
    'cma_testlist0': {},
    'cma_testlist1': {},
    'cma_testlist2': {},
    'cma_testlist3': {},
    'cma_testlist4': {},
    'cma_testlist5': {}
}
for line in TEST_NAMEFILE:
    if 'define SM_ACMG_' in line:
        list_of_line = line.split(' ')
        (name, list_nr, bit) = (list_of_line[1].replace('SM_ACMG_', ''),
                                list_of_line[2].replace(',',
                                                        ''), list_of_line[3])
        var = 'cma_testlist' + list_nr
        name_dict[var][int(bit)] = name

caObjPPS = CalipsoAvhrrTrackObject()
for filename in files:
    caObjPPS += readCaliopAvhrrMatchObj(filename)
    print "Scene %s" % (os.path.basename(filename))
    use = caObjPPS.avhrr.all_arrays['bt11micron'] > -9
print_common_stats(caObjPPS, use, name_dict)
Esempio n. 8
0
        print "PPS", y_month, satellite, int(min(sunz[use_pps])), int(
            max(sunz[use_pps])),
        get_clear_cloudy_vectors(caObjPPS, use_pps)  #, both_have_ct)


MAIA_ROOT_DIR = ("/home/a001865/VALIDATIONS/MAIA/Reshaped_Files/" "*/1km/")
PPS_ROOT_DIR = ("/home/a001865/VALIDATIONS/PPS/Reshaped_Files/" "*/1km/")

print "Algorithm time satellite min-sunz, max-sunz, N PODcloudy FARcloudy PODclear FARclear Hitrate Kuipers Max-time-diff"
for y_month in [
        '2012/06', '2012/07', '2012/08', '2012/10', '2015/1201', '2015/1207',
        '2015/07'
]:
    maia_files = glob(MAIA_ROOT_DIR + "%s/*/*h5" % (y_month))
    pps_files = glob(PPS_ROOT_DIR + "%s/*/*h5" % (y_month))
    caObjMAIA = CalipsoAvhrrTrackObject()
    caObjPPS = CalipsoAvhrrTrackObject()
    for filename in maia_files:
        caObjMAIA += readCaliopAvhrrMatchObj(filename)
    for filename in pps_files:
        caObjPPS += readCaliopAvhrrMatchObj(filename)
    satellite = 'Suomi-NPP'

    if y_month in ['2015/1201', '2015/1207']:
        satellite = 'Metop-B'
        for dnt in ['day and twilight', 'night']:
            #print "Year, month, satellite %s %s %s"%(y_month, satellite, dnt)
            print_common_stats(caObjMAIA,
                               caObjPPS,
                               y_month,
                               satellite,
Esempio n. 9
0
        FAR_high = 100 * (np.sum(pps_high_i) - n_high_ok) / np.sum(pps_high_i)
        print "%3.1f %3.1f %3.1f %3.1f %3.1f %3.1f" % (
            POD_low, POD_medium, POD_high, FAR_low, FAR_medium, FAR_high)


# ----------------------------------------
if __name__ == "__main__":
    isModis1km = True
    isNPP_v2014 = False
    isGAC_v2014_morning_sat = False
    isGAC_v2014 = True

    ROOT_DIR_GAC_nn = (
        "/home/a001865/DATA_MISC/reshaped_files/"
        "ATRAIN_RESULTS_GAC_nnavhrr_20161202/Reshaped_Files/noaa18/")
    ROOT_DIR_GAC_old = ("/home/a001865/DATA_MISC/reshaped_files/"
                        "ATRAIN_RESULTS_GAC_v2014/Reshaped_Files/noaa18/")

    files = glob(ROOT_DIR_GAC_nn + "5km/2009/*/*/*h5")

    caObj = CalipsoAvhrrTrackObject()
    for filename in files:
        #print  os.path.basename(filename)
        caObj += readCaliopAvhrrMatchObj(filename)  #, var_to_skip='segment')

    #table_5_per_illumination(caObj)
    plot_ct_table4(caObj)
    plot_ct_table2(caObj)
    table_21_do_for_atbd(caObj)
    #plot_ct_table3(caObj)
Esempio n. 10
0
def match_calipso_avhrr(values, 
                        calipsoObj, calipsoObjAerosol, 
                        imagerGeoObj, imagerObj, 
                        ctype, cma, ctth, cppCph, nwp_obj,
                        avhrrAngObj, nwp_segments, options, res=resolution):

    import time
    import string
    
    retv = CalipsoAvhrrTrackObject()
    dsec = time.mktime((1993,1,1,0,0,0,0,0,0)) - time.timezone # Convert from TAI time to UTC in seconds since 1970
    if res == 1:
        lonCalipso = calipsoObj.longitude.ravel()
        latCalipso = calipsoObj.latitude.ravel()
        timeCalipso_tai = calipsoObj.profile_time_tai[::,0].ravel()
        timeCalipso = calipsoObj.profile_time_tai[::,0].ravel() + dsec
        timeCalipso_utc = calipsoObj.profile_utc_time[::,0].ravel()
        elevationCalipso = calipsoObj.dem_surface_elevation.ravel()
    if res == 5:
        # Use [:,1] Since 5km data has start, center, and end for each pixel
        lonCalipso = calipsoObj.longitude[:,1].ravel()
        latCalipso = calipsoObj.latitude[:,1].ravel()
        timeCalipso_tai = calipsoObj.profile_time_tai[:,1].ravel()
        timeCalipso = calipsoObj.profile_time_tai[:,1].ravel() + dsec
        timeCalipso_utc = calipsoObj.profile_utc_time[:,1].ravel()
        elevationCalipso = calipsoObj.dem_surface_elevation[::,2].ravel()
    
    ndim = lonCalipso.shape[0]
    
    # --------------------------------------------------------------------
    #cal,cap = get_calipso_avhrr_linpix(imagerGeoObj,values,lonCalipso,latCalipso,timeCalipso, options)
    # This function (match_calipso_avhrr) could use the MatchMapper object
    # created in map_avhrr() to make things a lot simpler... See usage in
    # amsr_avhrr_match.py
    #Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
    from common import map_avhrr
    cal, cap = map_avhrr(imagerGeoObj, lonCalipso.ravel(), latCalipso.ravel(),
                         radius_of_influence=RESOLUTION*0.7*1000.0) # somewhat larger than radius...
    print cal, cap
    calnan = np.where(cal == NODATA, np.nan, cal)
    if (~np.isnan(calnan)).sum() == 0:
        raise MatchupError("No matches within region.")

    
    if len(imagerGeoObj.time.shape)>1:
        imager_time_vector = [imagerGeoObj.time[line,pixel] for line, pixel in zip(cal,cap)]
        avhrr_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector, np.nan)
    else:
        avhrr_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal], np.nan)

    #    avhrr_lines_sec_1970 = calnan * DSEC_PER_AVHRR_SCALINE + imagerGeoObj.sec1970_start
    # Find all matching Calipso pixels within +/- sec_timeThr from the AVHRR data
    #    pdb.set_trace()

    """
    import matplotlib.pyplot as plt
    fig = plt.figure(figsize = (9,8))
    ax = fig.add_subplot(111)
    plt.plot(timeCalipso, 'b.')
    plt.plot(avhrr_lines_sec_1970, 'g.')
    plt.show()
    fig.savefig("nina_test_fil.png")
    """

    idx_match = elements_within_range(timeCalipso, avhrr_lines_sec_1970, sec_timeThr) 
    if idx_match.sum() == 0:
        raise MatchupError("No matches in region within time threshold %d s." % sec_timeThr)
    retv.calipso = calipso_track_from_matched(retv.calipso, calipsoObj, idx_match)
    cal_on_avhrr = np.repeat(cal, idx_match)
    cap_on_avhrr = np.repeat(cap, idx_match)
    retv.calipso.avhrr_linnum = cal_on_avhrr.astype('i')
    retv.calipso.avhrr_pixnum = cap_on_avhrr.astype('i')
    logger.info("cap_on_avhrr.shape: %s",cap_on_avhrr.shape)

    lon_calipso = np.repeat(lonCalipso, idx_match)
    lat_calipso = np.repeat(latCalipso, idx_match)
    # Calipso line,pixel inside AVHRR swath:
    cal_on_avhrr = np.repeat(cal, idx_match)
    cap_on_avhrr = np.repeat(cap, idx_match)
    logger.info("Start and end times: %s %s",
              time.gmtime(timeCalipso[0]),
              time.gmtime(timeCalipso[ndim-1]))
    
    retv.calipso.sec_1970 = np.repeat(timeCalipso,idx_match)
    retv.calipso.latitude = np.repeat(latCalipso,idx_match)
    retv.calipso.longitude = np.repeat(lonCalipso,idx_match)
    retv.calipso.profile_time_tai = np.repeat(timeCalipso_tai,idx_match)

    # Elevation is given in km's. Convert to meters:
    retv.calipso.elevation = np.repeat(elevationCalipso.ravel()*1000.0,
                                            idx_match.ravel()).astype('d')
    # Time
    if len(imagerGeoObj.time.shape)>1:
        retv.avhrr.sec_1970= [imagerGeoObj.time[line,pixel] for line, pixel in zip(cal_on_avhrr,cap_on_avhrr)]
    else:
        retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
    retv.diff_sec_1970 = retv.calipso.sec_1970 - retv.avhrr.sec_1970

    min_diff = np.minimum.reduce(retv.diff_sec_1970)
    max_diff = np.maximum.reduce(retv.diff_sec_1970)
    logger.info("Maximum and minimum time differences in sec (avhrr-calipso): %d %d",
          np.maximum.reduce(retv.diff_sec_1970),np.minimum.reduce(retv.diff_sec_1970))
    logger.info("AVHRR observation time of first calipso-avhrr match: %s",
          time.gmtime(retv.avhrr.sec_1970[0]))
    logger.info("AVHRR observation time of last calipso-avhrr match: %s",
          time.gmtime(retv.avhrr.sec_1970[-1]))

    # Make the latitude and pps cloudtype on the calipso track:
    # line and pixel arrays have equal dimensions
    logger.info("Generate the latitude,cloudtype tracks!")
    # -------------------------------------------------------------------------
    # Pick out the data from the track from AVHRR
    from extract_imager_along_track import avhrr_track_from_matched
    retv = avhrr_track_from_matched(retv, imagerGeoObj, imagerObj, avhrrAngObj, 
                                    nwp_obj, ctth, ctype, cma,  cal_on_avhrr, 
                                    cap_on_avhrr, avhrrCph=cppCph, 
                                    nwp_segments=nwp_segments)

    if calipsoObjAerosol is not None:
        retv.calipso_aerosol = calipso_track_from_matched(retv.calipso_aerosol, calipsoObjAerosol, idx_match)

    # -------------------------------------------------------------------------    
    logger.info("AVHRR-PPS Cloud Type,latitude: shapes = %s %s",
          retv.avhrr.cloudtype.shape,retv.avhrr.latitude.shape)
    ll = []
    for i in range(ndim):        
        #ll.append(("%7.3f  %7.3f  %d\n"%(lonCalipso[i],latCalipso[i],0)))
        ll.append(("%7.3f  %7.3f  %d\n"%(lonCalipso[i],latCalipso[i],idx_match[i])))
    max_cloud_top_calipso = np.maximum.reduce(retv.calipso.layer_top_altitude.ravel())
    logger.info("max_cloud_top_calipso: %2.1f",max_cloud_top_calipso)
    return retv,min_diff,max_diff
Esempio n. 11
0
def match_calipso_avhrr(values,
                        caObj,
                        caObjAerosol,
                        imagerGeoObj,
                        imagerObj,
                        ctype,
                        cma,
                        ctth,
                        cpp,
                        nwp_obj,
                        avhrrAngObj,
                        nwp_segments,
                        options,
                        res=resolution):

    import string
    from common import map_avhrr

    retv = CalipsoAvhrrTrackObject()
    lonCalipso = caObj.longitude.ravel()
    latCalipso = caObj.latitude.ravel()

    #Nina 20150313 Swithcing to mapping without area as in cpp. Following suggestion from Jakob
    cal, cap = map_avhrr(imagerGeoObj,
                         lonCalipso.ravel(),
                         latCalipso.ravel(),
                         radius_of_influence=RESOLUTION * 0.7 *
                         1000.0)  # somewhat larger than radius...
    #warn if no matches
    calnan = np.where(cal == NODATA, np.nan, cal)
    if (~np.isnan(calnan)).sum() == 0:
        raise MatchupError("No matches within region.")

    #check if it is within time limits:
    if len(imagerGeoObj.time.shape) > 1:
        imager_time_vector = [
            imagerGeoObj.time[line, pixel] for line, pixel in zip(cal, cap)
        ]
        imager_lines_sec_1970 = np.where(cal != NODATA, imager_time_vector,
                                         np.nan)
    else:
        imager_lines_sec_1970 = np.where(cal != NODATA, imagerGeoObj.time[cal],
                                         np.nan)
    idx_match = elements_within_range(caObj.sec_1970, imager_lines_sec_1970,
                                      sec_timeThr)
    if idx_match.sum() == 0:
        raise MatchupError("No matches in region within time threshold %d s." %
                           sec_timeThr)
    retv.calipso = calipso_track_from_matched(retv.calipso, caObj, idx_match)

    cal_on_avhrr = np.repeat(cal, idx_match)
    cap_on_avhrr = np.repeat(cap, idx_match)
    retv.calipso.avhrr_linnum = cal_on_avhrr.astype('i')
    retv.calipso.avhrr_pixnum = cap_on_avhrr.astype('i')
    #logger.info("calipso matched with avhrr shape: %s",cap_on_avhrr.shape)

    # Calipso line,pixel inside AVHRR swath:
    cal_on_avhrr = np.repeat(cal, idx_match)
    cap_on_avhrr = np.repeat(cap, idx_match)
    # Imager time
    if len(imagerGeoObj.time.shape) > 1:
        retv.avhrr.sec_1970 = [
            imagerGeoObj.time[line, pixel]
            for line, pixel in zip(cal_on_avhrr, cap_on_avhrr)
        ]
    else:
        retv.avhrr.sec_1970 = imagerGeoObj.time[cal_on_avhrr]
    retv.diff_sec_1970 = retv.calipso.sec_1970 - retv.avhrr.sec_1970
    do_some_logging(retv, caObj)
    logger.info("Generate the latitude,cloudtype tracks!")
    from extract_imager_along_track import avhrr_track_from_matched
    retv = avhrr_track_from_matched(retv,
                                    imagerGeoObj,
                                    imagerObj,
                                    avhrrAngObj,
                                    nwp_obj,
                                    ctth,
                                    ctype,
                                    cma,
                                    cal_on_avhrr,
                                    cap_on_avhrr,
                                    cpp=cpp,
                                    nwp_segments=nwp_segments)
    if caObjAerosol is not None:
        retv.calipso_aerosol = calipso_track_from_matched(
            retv.calipso_aerosol, caObjAerosol, idx_match)
    max_cloud_top_calipso = np.maximum.reduce(
        retv.calipso.layer_top_altitude.ravel())
    logger.info("max_cloud_top_calipso: %2.1f", max_cloud_top_calipso)
    return retv
#BASE_DIR = "/home/a001865/DATA_MISC/reshaped_files/ATRAIN_RESULTS_MODIS_NOVEMBER_nnAVHRR_with_gac"
ROOT_DIR = BASE_DIR + "/Reshaped_Files/eos2/1km/2010/%s/*/*caliop*.h5"
OUT_DIR_TEMPLATE = BASE_DIR + "/Reshaped_Files_merged/eos2/1km/2010/%s/"
outfile_template = "1km_%s_eos2_2010%s14_0000_00000_caliop_modis_match.h5"

BASE_DIR = "/home/a001865/DATA_MISC/reshaped_files/global_modis_01st_created20170519"
ROOT_DIR = BASE_DIR + "/Reshaped_Files/eos2/1km/2010/%s/*/*caliop*.h5"
OUT_DIR_TEMPLATE = BASE_DIR + "/Reshaped_Files_merged_calipso_cbase/eos2/1km/2010/%s/"
outfile_template = "1km_%s_eos2_2010%s01_0000_00000_caliop_modis_match.h5"

BASE_DIR = "/home/a001865/DATA_MISC/reshaped_files/global_modis_14th_created20171121"
ROOT_DIR = BASE_DIR + "/Reshaped_Files/eos2/1km/2010/%s/*/*caliop*.h5"
OUT_DIR_TEMPLATE = BASE_DIR + "/Reshaped_Files_merged_calipso_cbase//eos2/1km/2010/%s/"
outfile_template = "1km_%s_eos2_2010%s14_0000_00000_caliop_modis_match.h5"

caObj_night = CalipsoAvhrrTrackObject()
caObj_day = CalipsoAvhrrTrackObject()

for year in [
        "2010"
]:  #2012/02","2012/05", "2012/08", "2013/07", "2014/02", "2014/04", "2014/09"]:
    #for month in ["01","02","03","04","05","06","07","08","09","10","11","12"]:
    #for month in ["06"]:
    for month in [
            "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12",
            "01"
    ]:
        OUT_DIR = OUT_DIR_TEMPLATE % (month)
        if not os.path.exists(OUT_DIR):
            os.makedirs(OUT_DIR)