def detect_cloud_mask(agri_l1_file_path, agri_geo_file_path, agri_clm_tif_path): fy4_l1 = FY4AAGRIL1FDIDISK4KM(agri_l1_file_path) fy4_geo = FY4AAGRIL1GEODISK4KM(agri_geo_file_path) month = fy4_l1.start_time_stamp.month fy4_nav_file_path = os.path.join( '/FY4COMM/NBCLM/data', 'Assist', 'fygatNAV.FengYun-4A.xxxxxxx.4km_M%.2d.h5' % month) fy4_nav = FY4NavFile(fy4_nav_file_path) snow_mask = fy4_nav.get_snow_mask() snow_mask = snow_mask == 3 pix_lat = fy4_nav.get_latitude() pix_lon = fy4_nav.get_longitude() dem = fy4_nav.get_dem() sft = fy4_nav.prepare_surface_type_to_cspp() coastal = fy4_nav.get_coastal() coastal_mask = coastal > 0 space_mask = fy4_nav.get_space_mask(b=True) sun_zen = fy4_geo.get_sun_zenith() bt_372 = fy4_l1.get_band_by_channel('bt_372_low') ems_372 = fy4_l1.get_band_by_channel('ems_372') bt_850 = fy4_l1.get_band_by_channel('bt_850') bt_1080 = fy4_l1.get_band_by_channel('bt_1080') r = [] # 1 TStd tstd = TStd( lut_file_path=r"/FY4COMM/NBCLM/data/LUT/T_Std_M%.2d_handfix.nc" % month) x = tstd.prepare_feature(bt_1080) valid_mask = tstd.prepare_valid_mask(bt_1080, dem, sft, coastal_mask, space_mask) ratio, prob = tstd.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 2 Bt1185 bt1185 = Bt1185( lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Btd_11_85_M%.2d_handfix.nc" % month) x = bt1185.prepare_feature(bt_1080, bt_850) valid_mask = bt1185.prepare_valid_mask(bt_1080, bt_850, sft, space_mask) ratio, prob = bt1185.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 3 T11 t11 = T11(lut_file_path=r"/FY4COMM/NBCLM/data/LUT/T_11_M%.2d_handfix.nc" % month) x = t11.prepare_feature(bt_1080) valid_mask = t11.prepare_valid_mask(bt_1080, sft, space_mask) ratio, prob = t11.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 4 Btd37511Night btd37511 = Btd37511Night( lut_file_path= r"/FY4COMM/NBCLM/data/LUT/Btd_375_11_Night_M%.2d_handfix.nc" % month) x = btd37511.prepare_feature(bt_372, bt_1080) valid_mask = btd37511.prepare_valid_mask(bt_372, bt_1080, sft, sun_zen, space_mask) ratio, prob = btd37511.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 5 Tmax-T tmax_t = TmaxT( lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Tmax_T_M%.2d_handfix.nc" % month) x = tmax_t.prepare_feature(bt_1080) valid_mask = tmax_t.prepare_valid_mask(bt_1080, dem, sft, coastal_mask, space_mask) ratio, prob = tmax_t.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 6 GeoColor geo_color_tif_path = fy4_l1.fname.replace('FDI', 'CLR').replace( 'V0001.HDF', 'GeoColor.tif') geo_color = GeoColorRGB( lut_file_path=r"/FY4COMM/NBCLM/data/LUT/GeoColorRGB_M%.2d_handifx.nc" % month) x = geo_color.prepare_feature(geo_color_tif_path) valid_mask = geo_color.prepare_valid_mask(x, sft, space_mask) ratio, prob = geo_color.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 7 Emiss4Day emiss4day = Emiss375Day( lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Emiss_375_Day_M%.2d_handifx.nc" % month) x = emiss4day.prepare_feature(ems_372) valid_mask = emiss4day.prepare_valid_mask(ems_372, sft, sun_zen, space_mask) ratio, prob = emiss4day.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 8 Emiss4Night emiss4night = Emiss375Night( lut_file_path= r"/FY4COMM/NBCLM/data/LUT/Emiss_375_Night_M%.2d_handifx.nc" % month) x = emiss4night.prepare_feature(ems_372) valid_mask = emiss4night.prepare_valid_mask(ems_372, sft, sun_zen, space_mask) ratio, prob = emiss4night.infer(x, sft, valid_mask, space_mask, prob=True) r.append(ratio) # 融合 r_s = np.stack(r) r_s_p = np.prod(r_s, 0) prior_yes = t11.lut_ds['prior_yes'].data[sft[sft > 0] - 1] p = np.ma.masked_array(np.zeros(x.shape), space_mask) p[sft > 0] = 1.0 / (1.0 + r_s_p[sft > 0] / prior_yes - r_s_p[sft > 0]) space_mask_i = space_mask.astype(np.int) dig_p = np.ones(space_mask.shape, np.uint8) * 4 dig_p[p >= 0.9] = 0 dig_p[np.logical_and(p >= 0.5, p < 0.9)] = 1 dig_p[np.logical_and(p > 0.1, p < 0.5)] = 2 dig_p[p <= 0.1] = 3 dig_p[space_mask_i == 1] = 126 tiff.imwrite(agri_clm_tif_path, dig_p) return 0
tstd = TStd() tstd_lut_ds_x = copy.deepcopy(tstd.lut_ds) for idx, csft in enumerate(tstd_lut_ds_x.cspp_sft.data.tolist()): sft_cursor = idx + 1 sft_mask = sft == sft_cursor start_idx = 40 for i in tqdm.trange(start_idx, 100, desc='%s' % csft): # ignore 01 01 agri_l1_file_path = df.loc[df.index[i], ('l1')] agri_geo_file_path = df.loc[df.index[i], ('geo')] agri_clm_file_path = df.loc[df.index[i], ('clm')] l1 = FY4AAGRIL1FDIDISK4KM(agri_l1_file_path) clm = FY4AAGRICLM4KM(agri_clm_file_path) geo = FY4AAGRIL1GEODISK4KM(agri_geo_file_path) bt_1080 = l1.get_band_by_channel('bt_1080') tstd = TStd() x = tstd.prepare_feature(bt_1080) valid_mask = tstd.prepare_valid_mask(bt_1080, dem, sft, coastal_mask, space_mask) sft_valide_mask = np.logical_and(valid_mask, sft_mask) clm_array = clm.get_clm() label = clm_array[sft_valide_mask] fea = x[sft_valide_mask] cloudy_f = fea[label == 0]
def main(): parser = argparse.ArgumentParser(description='Naive Bayes Cloud Mask Algorithm for FY4A AGRI.') parser.add_argument('valid_csv', type=str, help='valid csv with parameters') parser.add_argument('--sft_str', '-s', type=str, help='surface type string', choices=['Space', 'DeepOcean', 'ShallowOcean', 'UnfrozenLand', 'SnowLand', 'Arctic', 'Antarctic', 'Desert']) parser.add_argument('--dn_str', '-d', type=str, help='day night flag', choices=['Day', ' Night', 'ALL']) parser.add_argument('--out', '-o', type=str, help='output directory') args = parser.parse_args() logger = logging.getLogger('fy4 nb clm log') logger.setLevel(logging.INFO) ch = logging.StreamHandler() ch.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) df = pd.read_csv(args.valid_csv) df.dropna(inplace=True) df.reset_index(drop=True, inplace=True) ddf = df.sample(frac=0.2, replace=True, random_state=29) ddf.reset_index(drop=True, inplace=True) ddf['p_flag'] = None for i in tqdm.trange(len(ddf)): t1_str = ddf.loc[i, ('dt')] l1_fpath = ddf.loc[i, ('l1')] geo_fpath = ddf.loc[i, ('geo')] clm_fpath = ddf.loc[i, ('clm')] clm_tif = clm_fpath.replace('.NC', '.tif') nb_clm_tif = clm_fpath.replace('_CLM', '_NBCLM').replace('.NC', '.tif') save_fy4_clm_to_tiff(clm_tif, clm_fpath) logger.info('convert clm format!') c4 = os.system("/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/geo_color_fy4/__main__.py {}".format(l1_fpath)) if c4: c5 = os.system( "/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/metesatpy/scripts/entry.py {} {} {}".format(l1_fpath, geo_fpath, nb_clm_tif)) if c5 == 0: logger.info('nb clm infer complete!') logger.info('evaluating...') fy4_l1 = FY4AAGRIL1FDIDISK4KM(l1_fpath) fy4_geo = FY4AAGRIL1GEODISK4KM(geo_fpath) month = fy4_l1.start_time_stamp.month fy4_nav_file_path = os.path.join('/FY4COMM/NBCLM/data', 'Assist', 'fygatNAV.FengYun-4A.xxxxxxx.4km_M%.2d.h5' % month) fy4_nav = FY4NavFile(fy4_nav_file_path) sft = fy4_nav.prepare_surface_type_to_cspp() ref_clm = tiff.imread(clm_tif) obj_clm = tiff.imread(nb_clm_tif) sft_cursor = fy4_nav.ET_SFT_class.get(args.sft_str, None) if sft_cursor == 0: sft_mask = sft > 0 else: sft_mask = sft == sft_cursor dn_flag = args.dn_str if dn_flag == 'Day': sun_zen = fy4_geo.get_sun_zenith() time_mask = np.logical_and(sft > 0, sun_zen <= 85.0) elif dn_flag == 'Night': sun_zen = fy4_geo.get_sun_zenith() time_mask = np.logical_and(sft > 0, sun_zen > 85.0) else: time_mask = sft_mask final_mask = np.logical_and(sft_mask, time_mask) a, b, c, d = confusion(ref_clm[final_mask], obj_clm[final_mask]) data = {'dt': [], 'a': [], 'b': [], 'c': [], 'd': []} data['dt'].append(t1_str) data['a'].append(int(a)) data['b'].append(int(b)) data['c'].append(int(c)) data['d'].append(int(d)) df = pd.DataFrame(data) df = data_frame_cm_metric(df) month_csv = os.path.join(args.out, fy4_l1.start_time_stamp.strftime( '%Y%m_{}_{}.csv'.format(args.sft_str, args.dn_str))) appendDFToCSV_void(df, month_csv) logger.info('evaluate complete!')