import os
import copy
import re
import tqdm
import pandas as pd
import dask as da
from metesatpy.production.FY4A import FY4NavFile, FY4AAGRIL1FDIDISK4KM, FY4AAGRIL1GEODISK4KM, FY4AAGRICLM4KM
from metesatpy.algorithms.CloudMask import TStd
data_root_dir = os.getenv('METEPY_DATA_PATH', 'data')
fy4_nav_file_name = 'fygatNAV.FengYun-4A.xxxxxxx.4km_M1.h5'
fy4_nav_file_path = os.path.join(data_root_dir, fy4_nav_file_name)
fy4_nav = FY4NavFile(fy4_nav_file_path)
pix_lat = fy4_nav.get_latitude()
pix_lon = fy4_nav.get_longitude()
dem = fy4_nav.get_dem()
sft = fy4_nav.prepare_surface_type_to_cspp()
coastal = fy4_nav.get_coastal()
coastal_mask = coastal > 0
space_mask = fy4_nav.get_space_mask(b=True)
sat_lat = 0
sat_lon = 104.7
workspace = r"D:\WorkSpace\20200429\project\data"
df = pd.read_csv('train.csv')
def main():
parser = argparse.ArgumentParser(description='Naive Bayes Cloud Mask Algorithm for FY4A AGRI.')
parser.add_argument('valid_csv', type=str,
help='valid csv with parameters')
parser.add_argument('--sft_str', '-s', type=str,
help='surface type string',
choices=['Space', 'DeepOcean', 'ShallowOcean', 'UnfrozenLand', 'SnowLand',
'Arctic', 'Antarctic', 'Desert'])
parser.add_argument('--dn_str', '-d', type=str,
help='day night flag',
choices=['Day', ' Night', 'ALL'])
parser.add_argument('--out', '-o', type=str,
help='output directory')
args = parser.parse_args()
logger = logging.getLogger('fy4 nb clm log')
logger.setLevel(logging.INFO)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
df = pd.read_csv(args.valid_csv)
df.dropna(inplace=True)
df.reset_index(drop=True, inplace=True)
ddf = df.sample(frac=0.2, replace=True, random_state=29)
ddf.reset_index(drop=True, inplace=True)
ddf['p_flag'] = None
for i in tqdm.trange(len(ddf)):
t1_str = ddf.loc[i, ('dt')]
l1_fpath = ddf.loc[i, ('l1')]
geo_fpath = ddf.loc[i, ('geo')]
clm_fpath = ddf.loc[i, ('clm')]
clm_tif = clm_fpath.replace('.NC', '.tif')
nb_clm_tif = clm_fpath.replace('_CLM', '_NBCLM').replace('.NC', '.tif')
save_fy4_clm_to_tiff(clm_tif, clm_fpath)
logger.info('convert clm format!')
c4 = os.system("/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/geo_color_fy4/__main__.py {}".format(l1_fpath))
if c4:
c5 = os.system(
"/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/metesatpy/scripts/entry.py {} {} {}".format(l1_fpath,
geo_fpath,
nb_clm_tif))
if c5 == 0:
logger.info('nb clm infer complete!')
logger.info('evaluating...')
fy4_l1 = FY4AAGRIL1FDIDISK4KM(l1_fpath)
fy4_geo = FY4AAGRIL1GEODISK4KM(geo_fpath)
month = fy4_l1.start_time_stamp.month
fy4_nav_file_path = os.path.join('/FY4COMM/NBCLM/data', 'Assist',
'fygatNAV.FengYun-4A.xxxxxxx.4km_M%.2d.h5' % month)
fy4_nav = FY4NavFile(fy4_nav_file_path)
sft = fy4_nav.prepare_surface_type_to_cspp()
ref_clm = tiff.imread(clm_tif)
obj_clm = tiff.imread(nb_clm_tif)
sft_cursor = fy4_nav.ET_SFT_class.get(args.sft_str, None)
if sft_cursor == 0:
sft_mask = sft > 0
else:
sft_mask = sft == sft_cursor
dn_flag = args.dn_str
if dn_flag == 'Day':
sun_zen = fy4_geo.get_sun_zenith()
time_mask = np.logical_and(sft > 0, sun_zen <= 85.0)
elif dn_flag == 'Night':
sun_zen = fy4_geo.get_sun_zenith()
time_mask = np.logical_and(sft > 0, sun_zen > 85.0)
else:
time_mask = sft_mask
final_mask = np.logical_and(sft_mask, time_mask)
a, b, c, d = confusion(ref_clm[final_mask], obj_clm[final_mask])
data = {'dt': [], 'a': [], 'b': [], 'c': [], 'd': []}
data['dt'].append(t1_str)
data['a'].append(int(a))
data['b'].append(int(b))
data['c'].append(int(c))
data['d'].append(int(d))
df = pd.DataFrame(data)
df = data_frame_cm_metric(df)
month_csv = os.path.join(args.out,
fy4_l1.start_time_stamp.strftime(
'%Y%m_{}_{}.csv'.format(args.sft_str, args.dn_str)))
appendDFToCSV_void(df, month_csv)
logger.info('evaluate complete!')
def detect_cloud_mask(agri_l1_file_path, agri_geo_file_path,
agri_clm_tif_path):
fy4_l1 = FY4AAGRIL1FDIDISK4KM(agri_l1_file_path)
fy4_geo = FY4AAGRIL1GEODISK4KM(agri_geo_file_path)
month = fy4_l1.start_time_stamp.month
fy4_nav_file_path = os.path.join(
'/FY4COMM/NBCLM/data', 'Assist',
'fygatNAV.FengYun-4A.xxxxxxx.4km_M%.2d.h5' % month)
fy4_nav = FY4NavFile(fy4_nav_file_path)
snow_mask = fy4_nav.get_snow_mask()
snow_mask = snow_mask == 3
pix_lat = fy4_nav.get_latitude()
pix_lon = fy4_nav.get_longitude()
dem = fy4_nav.get_dem()
sft = fy4_nav.prepare_surface_type_to_cspp()
coastal = fy4_nav.get_coastal()
coastal_mask = coastal > 0
space_mask = fy4_nav.get_space_mask(b=True)
sun_zen = fy4_geo.get_sun_zenith()
bt_372 = fy4_l1.get_band_by_channel('bt_372_low')
ems_372 = fy4_l1.get_band_by_channel('ems_372')
bt_850 = fy4_l1.get_band_by_channel('bt_850')
bt_1080 = fy4_l1.get_band_by_channel('bt_1080')
r = []
# 1 TStd
tstd = TStd(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/T_Std_M%.2d_handfix.nc" %
month)
x = tstd.prepare_feature(bt_1080)
valid_mask = tstd.prepare_valid_mask(bt_1080, dem, sft, coastal_mask,
space_mask)
ratio, prob = tstd.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 2 Bt1185
bt1185 = Bt1185(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Btd_11_85_M%.2d_handfix.nc" %
month)
x = bt1185.prepare_feature(bt_1080, bt_850)
valid_mask = bt1185.prepare_valid_mask(bt_1080, bt_850, sft, space_mask)
ratio, prob = bt1185.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 3 T11
t11 = T11(lut_file_path=r"/FY4COMM/NBCLM/data/LUT/T_11_M%.2d_handfix.nc" %
month)
x = t11.prepare_feature(bt_1080)
valid_mask = t11.prepare_valid_mask(bt_1080, sft, space_mask)
ratio, prob = t11.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 4 Btd37511Night
btd37511 = Btd37511Night(
lut_file_path=
r"/FY4COMM/NBCLM/data/LUT/Btd_375_11_Night_M%.2d_handfix.nc" % month)
x = btd37511.prepare_feature(bt_372, bt_1080)
valid_mask = btd37511.prepare_valid_mask(bt_372, bt_1080, sft, sun_zen,
space_mask)
ratio, prob = btd37511.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 5 Tmax-T
tmax_t = TmaxT(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Tmax_T_M%.2d_handfix.nc" %
month)
x = tmax_t.prepare_feature(bt_1080)
valid_mask = tmax_t.prepare_valid_mask(bt_1080, dem, sft, coastal_mask,
space_mask)
ratio, prob = tmax_t.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 6 GeoColor
geo_color_tif_path = fy4_l1.fname.replace('FDI', 'CLR').replace(
'V0001.HDF', 'GeoColor.tif')
geo_color = GeoColorRGB(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/GeoColorRGB_M%.2d_handifx.nc" %
month)
x = geo_color.prepare_feature(geo_color_tif_path)
valid_mask = geo_color.prepare_valid_mask(x, sft, space_mask)
ratio, prob = geo_color.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 7 Emiss4Day
emiss4day = Emiss375Day(
lut_file_path=r"/FY4COMM/NBCLM/data/LUT/Emiss_375_Day_M%.2d_handifx.nc"
% month)
x = emiss4day.prepare_feature(ems_372)
valid_mask = emiss4day.prepare_valid_mask(ems_372, sft, sun_zen,
space_mask)
ratio, prob = emiss4day.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 8 Emiss4Night
emiss4night = Emiss375Night(
lut_file_path=
r"/FY4COMM/NBCLM/data/LUT/Emiss_375_Night_M%.2d_handifx.nc" % month)
x = emiss4night.prepare_feature(ems_372)
valid_mask = emiss4night.prepare_valid_mask(ems_372, sft, sun_zen,
space_mask)
ratio, prob = emiss4night.infer(x, sft, valid_mask, space_mask, prob=True)
r.append(ratio)
# 融合
r_s = np.stack(r)
r_s_p = np.prod(r_s, 0)
prior_yes = t11.lut_ds['prior_yes'].data[sft[sft > 0] - 1]
p = np.ma.masked_array(np.zeros(x.shape), space_mask)
p[sft > 0] = 1.0 / (1.0 + r_s_p[sft > 0] / prior_yes - r_s_p[sft > 0])
space_mask_i = space_mask.astype(np.int)
dig_p = np.ones(space_mask.shape, np.uint8) * 4
dig_p[p >= 0.9] = 0
dig_p[np.logical_and(p >= 0.5, p < 0.9)] = 1
dig_p[np.logical_and(p > 0.1, p < 0.5)] = 2
dig_p[p <= 0.1] = 3
dig_p[space_mask_i == 1] = 126
tiff.imwrite(agri_clm_tif_path, dig_p)
return 0
def main(dt, save_space):
logger = logging.getLogger('fy4 nb clm log')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
download_command = "wget -c --ftp-user=guoxuexingNRT --ftp-password=za4158 -P {save_space} {url}"
t1_str = dt.strftime('%Y%m%d%H%M%S')
t2 = dt + datetime.timedelta(minutes=14, seconds=59)
t2_str = t2.strftime('%Y%m%d%H%M%S')
l1_file_name = 'FY4A-_AGRI--_N_DISK_1047E_L1-_FDI-_MULT_NOM_{t1_str}_{t2_str}_4000M_V0001.HDF'.format(
t1_str=t1_str, t2_str=t2_str)
geo_file_name = 'FY4A-_AGRI--_N_DISK_1047E_L1-_GEO-_MULT_NOM_{t1_str}_{t2_str}_4000M_V0001.HDF'.format(
t1_str=t1_str, t2_str=t2_str)
clm_file_name = 'FY4A-_AGRI--_N_DISK_1047E_L2-_CLM-_MULT_NOM_{t1_str}_{t2_str}_4000M_V0001.NC'.format(
t1_str=t1_str, t2_str=t2_str)
l1_url_prefix = dt.strftime(
'ftp://ftp.nsmc.org.cn/FY4A/AGRI/L1/FDI/DISK/4000M/%Y/%Y%m%d/')
geo_url_prefix = dt.strftime(
'ftp://ftp.nsmc.org.cn/FY4A/AGRI/L1/FDI/DISK/GEO/%Y/%Y%m%d/')
clm_url_prefix = dt.strftime(
'ftp://ftp.nsmc.org.cn/FY4A/AGRI/L2/CLM/DISK/NOM/%Y/%Y%m%d/')
logger.info('time cursor: %s -- %s' % (t1_str, t2_str))
logger.info('l1 downloading!')
c1 = os.system(
download_command.format(save_space=save_space,
url=l1_url_prefix + l1_file_name))
if c1 == 0:
logger.info('l1 download complete!')
else:
logger.error('l1 download failed!')
return -1
logger.info('geo downloading!')
c2 = os.system(
download_command.format(save_space=save_space,
url=geo_url_prefix + geo_file_name))
if c2 == 0:
logger.info('geo download complete!')
else:
logger.error('geo download failed!')
return -1
logger.info('clm downloading!')
c3 = os.system(
download_command.format(save_space=save_space,
url=clm_url_prefix + clm_file_name))
if c3 == 0:
logger.info('clm download complete!')
else:
logger.error('clm download failed!')
return -1
l1_fpath = os.path.join(save_space, l1_file_name)
logger.info('ploting geo color!')
c4 = os.system(
"/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/geo_color_fy4/__main__.py {}"
.format(l1_fpath))
if c4 == 0:
logger.info('geo color complete!')
geo_fpath = os.path.join(save_space, geo_file_name)
clm_fpath = os.path.join(save_space, clm_file_name)
clm_tif = clm_fpath.replace('.NC', '.tif')
save_fy4_clm_to_tiff(clm_tif, clm_fpath)
nb_clm_tif = clm_fpath.replace('_CLM', '_NBCLM').replace('.NC', '.tif')
c5 = os.system(
"/home/fy4/miniconda3/bin/python /FY4COMM/NBCLM/metesatpy/scripts/entry.py {} {} {}"
.format(l1_fpath, geo_fpath, nb_clm_tif))
if c5 == 0:
logger.info('nb clm infer complete!')
logger.info('evaluating...')
fy4_l1 = FY4AAGRIL1FDIDISK4KM(l1_fpath)
month = fy4_l1.start_time_stamp.month
fy4_nav_file_path = os.path.join(
'/FY4COMM/NBCLM/data', 'Assist',
'fygatNAV.FengYun-4A.xxxxxxx.4km_M%.2d.h5' % month)
fy4_nav = FY4NavFile(fy4_nav_file_path)
sft = fy4_nav.prepare_surface_type_to_cspp()
sft_mask = sft > 0
ref_clm = tiff.imread(clm_tif)
obj_clm = tiff.imread(nb_clm_tif)
a, b, c, d = confusion(ref_clm[sft_mask], obj_clm[sft_mask])
data = {'dt': [], 'a': [], 'b': [], 'c': [], 'd': []}
data['dt'].append(t1_str)
data['a'].append(int(a))
data['b'].append(int(b))
data['c'].append(int(c))
data['d'].append(int(d))
df = pd.DataFrame(data)
df = data_frame_cm_metric(df)
logger.debug(str(df))
month_csv = os.path.join(save_space, dt.strftime('%Y%m.csv'))
appendDFToCSV_void(df, month_csv)
logger.info('evaluate complete!')
os.remove(l1_fpath)
os.remove(geo_fpath)
os.remove(clm_fpath)
os.remove(
l1_fpath.replace('_FDI', '_CLR').replace('V0001.HDF',
'GeoColor.tif'))
os.remove(clm_tif)
os.remove(nb_clm_tif)