def detect_cloud_mask(agri_l1_file_path, agri_geo_file_path,
agri_clm_tif_path):
fy4_l1 = FY4AAGRIL1FDIDISK4KM(agri_l1_file_path)
fy4_geo = FY4AAGRIL1GEODISK4KM(agri_geo_file_path)
month = fy4_l1.start_time_stamp.month
fy4_nav_file_path = os.path.join(
'/FY4COMM/NBCLM/data', 'Assist',
'fygatNAV.FengYun-4A.xxxxxxx.4km_M%.2d.h5' % month)
fy4_nav = FY4NavFile(fy4_nav_file_path)
snow_mask = fy4_nav.get_snow_mask()
snow_mask = snow_mask == 3
pix_lat = fy4_nav.get_latitude()
pix_lon = fy4_nav.get_longitude()
dem = fy4_nav.get_dem()
sft = fy4_nav.prepare_surface_type_to_cspp()
coastal = fy4_nav.get_coastal()
coastal_mask = coastal > 0
space_mask = fy4_nav.get_space_mask(b=True)
sun_zen = fy4_geo.get_sun_zenith()
bt_372 = fy4_l1.get_band_by_channel('bt_372_low')
ems_372 = fy4_l1.get_band_by_channel('ems_372')
bt_850 = fy4_l1.get_band_by_channel('bt_850')
bt_1080 = fy4_l1.get_band_by_channel('bt_1080')
r = []
# 1 TStd
tstd = TStd(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/T_Std_M%.2d_handfix.nc" %
month)
x = tstd.prepare_feature(bt_1080)
valid_mask = tstd.prepare_valid_mask(bt_1080, dem, sft, coastal_mask,
space_mask)
ratio, prob = tstd.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 2 Bt1185
bt1185 = Bt1185(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Btd_11_85_M%.2d_handfix.nc" %
month)
x = bt1185.prepare_feature(bt_1080, bt_850)
valid_mask = bt1185.prepare_valid_mask(bt_1080, bt_850, sft, space_mask)
ratio, prob = bt1185.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 3 T11
t11 = T11(lut_file_path=r"/FY4COMM/NBCLM/data/LUT/T_11_M%.2d_handfix.nc" %
month)
x = t11.prepare_feature(bt_1080)
valid_mask = t11.prepare_valid_mask(bt_1080, sft, space_mask)
ratio, prob = t11.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 4 Btd37511Night
btd37511 = Btd37511Night(
lut_file_path=
r"/FY4COMM/NBCLM/data/LUT/Btd_375_11_Night_M%.2d_handfix.nc" % month)
x = btd37511.prepare_feature(bt_372, bt_1080)
valid_mask = btd37511.prepare_valid_mask(bt_372, bt_1080, sft, sun_zen,
space_mask)
ratio, prob = btd37511.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 5 Tmax-T
tmax_t = TmaxT(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Tmax_T_M%.2d_handfix.nc" %
month)
x = tmax_t.prepare_feature(bt_1080)
valid_mask = tmax_t.prepare_valid_mask(bt_1080, dem, sft, coastal_mask,
space_mask)
ratio, prob = tmax_t.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 6 GeoColor
geo_color_tif_path = fy4_l1.fname.replace('FDI', 'CLR').replace(
'V0001.HDF', 'GeoColor.tif')
geo_color = GeoColorRGB(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/GeoColorRGB_M%.2d_handifx.nc" %
month)
x = geo_color.prepare_feature(geo_color_tif_path)
valid_mask = geo_color.prepare_valid_mask(x, sft, space_mask)
ratio, prob = geo_color.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 7 Emiss4Day
emiss4day = Emiss375Day(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Emiss_375_Day_M%.2d_handifx.nc"
% month)
x = emiss4day.prepare_feature(ems_372)
valid_mask = emiss4day.prepare_valid_mask(ems_372, sft, sun_zen,
space_mask)
ratio, prob = emiss4day.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 8 Emiss4Night
emiss4night = Emiss375Night(
lut_file_path=
r"/FY4COMM/NBCLM/data/LUT/Emiss_375_Night_M%.2d_handifx.nc" % month)
x = emiss4night.prepare_feature(ems_372)
valid_mask = emiss4night.prepare_valid_mask(ems_372, sft, sun_zen,
space_mask)
ratio, prob = emiss4night.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 融合
r_s = np.stack(r)
r_s_p = np.prod(r_s, 0)
prior_yes = t11.lut_ds['prior_yes'].data[sft[sft > 0] - 1]
p = np.ma.masked_array(np.zeros(x.shape), space_mask)
p[sft > 0] = 1.0 / (1.0 + r_s_p[sft > 0] / prior_yes - r_s_p[sft > 0])
space_mask_i = space_mask.astype(np.int)
dig_p = np.ones(space_mask.shape, np.uint8) * 4
dig_p[p >= 0.9] = 0
dig_p[np.logical_and(p >= 0.5, p < 0.9)] = 1
dig_p[np.logical_and(p > 0.1, p < 0.5)] = 2
dig_p[p <= 0.1] = 3
dig_p[space_mask_i == 1] = 126
tiff.imwrite(agri_clm_tif_path, dig_p)
return 0
tstd = TStd()
tstd_lut_ds_x = copy.deepcopy(tstd.lut_ds)
for idx, csft in enumerate(tstd_lut_ds_x.cspp_sft.data.tolist()):
sft_cursor = idx + 1
sft_mask = sft == sft_cursor
start_idx = 40
for i in tqdm.trange(start_idx, 100, desc='%s' % csft): # ignore 01 01
agri_l1_file_path = df.loc[df.index[i], ('l1')]
agri_geo_file_path = df.loc[df.index[i], ('geo')]
agri_clm_file_path = df.loc[df.index[i], ('clm')]
l1 = FY4AAGRIL1FDIDISK4KM(agri_l1_file_path)
clm = FY4AAGRICLM4KM(agri_clm_file_path)
geo = FY4AAGRIL1GEODISK4KM(agri_geo_file_path)
bt_1080 = l1.get_band_by_channel('bt_1080')
tstd = TStd()
x = tstd.prepare_feature(bt_1080)
valid_mask = tstd.prepare_valid_mask(bt_1080, dem, sft, coastal_mask,
space_mask)
sft_valide_mask = np.logical_and(valid_mask, sft_mask)
clm_array = clm.get_clm()
label = clm_array[sft_valide_mask]
fea = x[sft_valide_mask]
cloudy_f = fea[label == 0]
def main():
parser = argparse.ArgumentParser(description='Naive Bayes Cloud Mask Algorithm for FY4A AGRI.')
parser.add_argument('valid_csv', type=str,
help='valid csv with parameters')
parser.add_argument('--sft_str', '-s', type=str,
help='surface type string',
choices=['Space', 'DeepOcean', 'ShallowOcean', 'UnfrozenLand', 'SnowLand',
'Arctic', 'Antarctic', 'Desert'])
parser.add_argument('--dn_str', '-d', type=str,
help='day night flag',
choices=['Day', ' Night', 'ALL'])
parser.add_argument('--out', '-o', type=str,
help='output directory')
args = parser.parse_args()
logger = logging.getLogger('fy4 nb clm log')
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
df = pd.read_csv(args.valid_csv)
df.dropna(inplace=True)
df.reset_index(drop=True, inplace=True)
ddf = df.sample(frac=0.2, replace=True, random_state=29)
ddf.reset_index(drop=True, inplace=True)
ddf['p_flag'] = None
for i in tqdm.trange(len(ddf)):
t1_str = ddf.loc[i, ('dt')]
l1_fpath = ddf.loc[i, ('l1')]
geo_fpath = ddf.loc[i, ('geo')]
clm_fpath = ddf.loc[i, ('clm')]
clm_tif = clm_fpath.replace('.NC', '.tif')
nb_clm_tif = clm_fpath.replace('_CLM', '_NBCLM').replace('.NC', '.tif')
save_fy4_clm_to_tiff(clm_tif, clm_fpath)
logger.info('convert clm format!')
c4 = os.system("/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/geo_color_fy4/__main__.py {}".format(l1_fpath))
if c4:
c5 = os.system(
"/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/metesatpy/scripts/entry.py {} {} {}".format(l1_fpath,
geo_fpath,
nb_clm_tif))
if c5 == 0:
logger.info('nb clm infer complete!')
logger.info('evaluating...')
fy4_l1 = FY4AAGRIL1FDIDISK4KM(l1_fpath)
fy4_geo = FY4AAGRIL1GEODISK4KM(geo_fpath)
month = fy4_l1.start_time_stamp.month
fy4_nav_file_path = os.path.join('/FY4COMM/NBCLM/data', 'Assist',
'fygatNAV.FengYun-4A.xxxxxxx.4km_M%.2d.h5' % month)
fy4_nav = FY4NavFile(fy4_nav_file_path)
sft = fy4_nav.prepare_surface_type_to_cspp()
ref_clm = tiff.imread(clm_tif)
obj_clm = tiff.imread(nb_clm_tif)
sft_cursor = fy4_nav.ET_SFT_class.get(args.sft_str, None)
if sft_cursor == 0:
sft_mask = sft > 0
else:
sft_mask = sft == sft_cursor
dn_flag = args.dn_str
if dn_flag == 'Day':
sun_zen = fy4_geo.get_sun_zenith()
time_mask = np.logical_and(sft > 0, sun_zen <= 85.0)
elif dn_flag == 'Night':
sun_zen = fy4_geo.get_sun_zenith()
time_mask = np.logical_and(sft > 0, sun_zen > 85.0)
else:
time_mask = sft_mask
final_mask = np.logical_and(sft_mask, time_mask)
a, b, c, d = confusion(ref_clm[final_mask], obj_clm[final_mask])
data = {'dt': [], 'a': [], 'b': [], 'c': [], 'd': []}
data['dt'].append(t1_str)
data['a'].append(int(a))
data['b'].append(int(b))
data['c'].append(int(c))
data['d'].append(int(d))
df = pd.DataFrame(data)
df = data_frame_cm_metric(df)
month_csv = os.path.join(args.out,
fy4_l1.start_time_stamp.strftime(
'%Y%m_{}_{}.csv'.format(args.sft_str, args.dn_str)))
appendDFToCSV_void(df, month_csv)
logger.info('evaluate complete!')