def main(yamlfile):
"""
对L3产品做长时间序列图
"""
YamlFile = yamlfile
if not os.path.isfile(YamlFile):
print 'Not Found %s' % YamlFile
sys.exit(-1)
with open(YamlFile, 'r') as stream:
cfg = yaml.load(stream)
ymd = cfg["INFO"]['ymd']
pair = cfg["INFO"]['pair']
file_list = cfg["PATH"]['ipath']
out_path = cfg["PATH"]['opath']
print "-" * 100
# date_start = ymd2date(s_time)
# date_end = ymd2date(e_time)
for in_file in file_list:
h5 = LoadH5()
data_dict = {}
datetime_list = []
file_name = os.path.basename(in_file)
if 'L3' in pair:
reg = '.*_(\d{8}).*.HDF'
m = re.match(reg, file_name)
ymd = m.group(1)
datetime_list.append(ymd2date(ymd))
elif 'L2' in pair:
reg = '.*_(\d{14}).*.HDF'
m = re.match(reg, file_name)
ymd = m.group(1)
ymdhms = datetime.strptime('%s' % (ymd), "%Y%m%d%H%M%S")
datetime_list.append(ymdhms)
h5.load(in_file, ymd)
dict_data = {}
for key in h5.data.keys():
if key in ['Longitude', 'ymd', 'Latitude']:
continue
data = h5.data[key]
idx = np.where(data > 0)
mean = np.mean(data[idx])
# print key, type(mean)
dict_data[key] = mean.reshape(1, 1)
# print dict_data
file_name = '%s_%s_COMBINE.HDF' % (pair, ymd)
ofile = os.path.join(out_path, file_name)
write_h5(ofile, dict_data)
def plot_abc(date_D, a_D, b_D, c_D, date_M, a_M, b_M, c_M, picPath, title,
date_s, date_e, var):
fig = plt.figure(figsize=(6, 6))
ax1 = plt.subplot(311)
ax2 = plt.subplot(312, sharex=ax1)
ax3 = plt.subplot(313, sharex=ax1)
# format the Xticks
xlim_min = pb_time.ymd2date('%04d%02d01' % (date_s.year, date_s.month))
xlim_max = date_e
ax1.set_xlim(xlim_min, xlim_max)
# format the Yticks\
if var == "tbb-tbb" or var == "ref-ref":
ax1.set_ylim(0, 2)
ax1.yaxis.set_major_locator(MultipleLocator(0.5))
ax1.yaxis.set_minor_locator(MultipleLocator(0.1))
elif var == "dn-ref":
ax1.set_ylim(0.0002, 0.0007)
ax1.yaxis.set_major_locator(MultipleLocator(0.0001))
ax1.yaxis.set_minor_locator(MultipleLocator(0.00005))
if var == "tbb-tbb":
ax2.set_ylim(-30, 30)
ax2.yaxis.set_major_locator(MultipleLocator(10))
ax2.yaxis.set_minor_locator(MultipleLocator(2))
elif var == "ref-ref":
ax2.set_ylim(-0.1, 0.1)
ax2.yaxis.set_major_locator(MultipleLocator(0.02))
ax2.yaxis.set_minor_locator(MultipleLocator(0.01))
elif var == "dn-ref":
ax2.set_ylim(-0.08, 0.08)
ax2.yaxis.set_major_locator(MultipleLocator(0.02))
ax2.yaxis.set_minor_locator(MultipleLocator(0.01))
ax3.set_ylim(0, 7)
ax3.yaxis.set_major_locator(MultipleLocator(1))
ax3.yaxis.set_minor_locator(MultipleLocator(0.2))
# plot ax1 -------------------------------------------------
plt.sca(ax1)
plt.plot(date_D,
a_D,
'x',
ms=5,
markerfacecolor=None,
markeredgecolor=BLUE,
alpha=0.8,
mew=0.3,
label='Daily')
plt.plot(date_M, a_M, 'o-', ms=4, lw=0.6, c=RED, mew=0, label='Monthly')
plt.ylabel('Slope', fontsize=11, fontproperties=FONT0)
plt.grid(True)
plt.title(title, fontsize=12, fontproperties=FONT0)
set_tick_font(ax1)
plt.setp(ax1.get_xticklabels(), visible=False)
# plot ax2 -------------------------------------------------
plt.sca(ax2)
plt.plot(date_D,
b_D,
'x',
ms=5,
markerfacecolor=None,
markeredgecolor=BLUE,
alpha=0.8,
mew=0.3,
label='Daily')
plt.plot(date_M, b_M, 'o-', ms=4, lw=0.6, c=RED, mew=0, label='Monthly')
plt.ylabel('Intercept', fontsize=11, fontproperties=FONT0)
plt.grid(True)
set_tick_font(ax2)
plt.setp(ax2.get_xticklabels(), visible=False)
# point number -------------------------------------------------
plt.sca(ax3)
plt.fill_between(date_D, 0, c_D, edgecolor=BLUE, facecolor=BLUE, alpha=0.6)
# plt.fill_between(date_M, 0, c_M,
# edgecolor=RED, facecolor=RED, alpha=0.5)
# plt.bar(date_M, c_M, width=1, align='edge', # "center",
# color=RED, linewidth=0)
plt.plot(date_M, c_M, 'o-', ms=4, lw=0.6, c=RED, mew=0, label='Monthly')
plt.ylabel('Number of sample points\nlog (base = 10)',
fontsize=11,
fontproperties=FONT0)
plt.grid(True)
set_tick_font(ax3)
setXLocator(ax3, xlim_min, xlim_max)
# circle1 = mpatches.Circle((430, 563), 5, color=BLUE, ec=EDGE_GRAY, lw=0)
# circle2 = mpatches.Circle((508, 563), 5, color=RED, ec=EDGE_GRAY, lw=0)
# fig.patches.extend([circle1, circle2])
#
# fig.text(0.74, 0.93, 'Daily', color=BLUE, fontproperties=FONT0)
# fig.text(0.86, 0.93, 'Monthly', color=RED, fontproperties=FONT0)
#---------------
plt.tight_layout()
fig.subplots_adjust(bottom=0.14)
circle1 = mpatches.Circle((74, 18), 6, color=BLUE, ec=EDGE_GRAY, lw=0)
circle2 = mpatches.Circle((164, 18), 6, color=RED, ec=EDGE_GRAY, lw=0)
fig.patches.extend([circle1, circle2])
fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0)
fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0)
ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d')
if ymd_s != ymd_e:
fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0)
else:
fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0)
fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0)
#---------------
pb_io.make_sure_path_exists(os.path.dirname(picPath))
fig.savefig(picPath)
plt.close()
fig.clear
def plot_tbbias(date_D, bias_D, date_M, bias_M, picPath, title, date_s, date_e,
satName):
'''
画偏差时序折线图
'''
fig = plt.figure(figsize=(6, 4))
# plt.subplots_adjust(left=0.13, right=0.95, bottom=0.11, top=0.97)
if (np.isnan(bias_D)).all():
Log.error('Everything is NaN: %s' % picPath)
return
plt.plot(date_D,
bias_D,
'x',
ms=6,
markerfacecolor=None,
markeredgecolor=BLUE,
alpha=0.8,
mew=0.3,
label='Daily')
plt.plot(date_M, bias_M, 'o-', ms=5, lw=0.6, c=RED, mew=0, label='Monthly')
plt.grid(True)
plt.ylabel('DTB($K$)', fontsize=11, fontproperties=FONT0)
xlim_min = pb_time.ymd2date('%04d%02d01' % (date_s.year, date_s.month))
xlim_max = date_e
plt.xlim(xlim_min, xlim_max)
if "FY2" in satName:
plt.ylim(-8, 8)
elif "FY3" in satName:
plt.ylim(-8, 8)
elif "FY4" in satName:
plt.ylim(-2, 2)
ax = plt.gca()
# format the ticks
setXLocator(ax, xlim_min, xlim_max)
set_tick_font(ax)
# title
plt.title(title, fontsize=12, fontproperties=FONT0)
plt.tight_layout()
#--------------------
fig.subplots_adjust(bottom=0.2)
circle1 = mpatches.Circle((74, 15), 6, color=BLUE, ec=EDGE_GRAY, lw=0)
circle2 = mpatches.Circle((164, 15), 6, color=RED, ec=EDGE_GRAY, lw=0)
fig.patches.extend([circle1, circle2])
fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0)
fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0)
ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d')
if ymd_s != ymd_e:
fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0)
else:
fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0)
fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0)
#---------------
pb_io.make_sure_path_exists(os.path.dirname(picPath))
plt.savefig(picPath)
fig.clear()
plt.close()
def run(pair, date_s, date_e):
'''
pair: sat1+sensor1_sat2+sensor2
date_s: datetime of start date
None 处理 从发星 到 有数据的最后一天
date_e: datetime of end date
None 处理 从发星 到 有数据的最后一天
'''
isLaunch = False
if date_s is None or date_e is None:
isLaunch = True
part1, part2 = pair.split('_')
sat1, sensor1 = part1.split('+')
sat2, sensor2 = part2.split('+')
# load yaml config file
plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2))
plt_cfg = pb_io.loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
return
# must be in 'all', 'day', 'night'
Day_Night = ['all', 'day', 'night']
if "day_night" in plt_cfg.keys():
Day_Night = plt_cfg['day_night']
for each in Day_Night:
if each not in ['all', 'day', 'night']:
Day_Night.remove(each)
Bias = ["tbb"]
if "bias" in plt_cfg.keys():
Bias = plt_cfg["bias"]
co = coeff_abr(pair)
if co.error: return
for each in plt_cfg['plot']:
xname, yname = each.split('-')
if 'tbb' in xname and 'tbb' in yname:
o_name = 'TBBCalCoeff'
elif 'ref' in xname and 'ref' in yname:
o_name = 'CorrcCoeff'
elif 'dn' in xname and 'ref' in yname:
o_name = 'CalCoeff'
else:
continue
for DayOrNight in Day_Night:
if DayOrNight == 'all':
DayOrNight = DayOrNight.upper() # all -> ALL
else:
DayOrNight = DayOrNight[0].upper() + DayOrNight[1:]
co.loadCoeff(o_name, DayOrNight)
if co.error: continue
date_D = co.Coeff_D['date']
date_M = co.Coeff_M['date']
if isLaunch:
if sat1 in inCfg['LUANCH_DATE']:
date_s = pb_time.ymd2date(str(inCfg['LUANCH_DATE'][sat1]))
else:
Log.error(
'%s not in LUANCH_DATE of Cfg, use the first day in txt instead.'
)
date_s = date_D[0]
date_e = date_D[-1]
ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d')
Log.info(u"----- Start Drawing ABC TBBias Pic, PAIR: {}, YMD: {}-{}: " \
u"{}-----".format(pair, ymd_s, ymd_e, DayOrNight))
idx_D = np.where(np.logical_and(date_D >= date_s,
date_D <= date_e))
idx_M = np.where(
np.logical_and(date_M >= pb_time.ymd2date(ymd_s[:6] + '01'),
date_M <= date_e))
for i, chan in enumerate(plt_cfg['chan']):
a_D = co.Coeff_D['a_%s' % chan]
b_D = co.Coeff_D['b_%s' % chan]
c_D = np.log10(co.Coeff_D['c_%s' % chan])
a_M = co.Coeff_M['a_%s' % chan]
b_M = co.Coeff_M['b_%s' % chan]
c_M = np.log10(co.Coeff_M['c_%s' % chan])
# plot slope Intercept count ------------------------
title = 'Time Series of Slope Intercept & Counts %s %s\n(%s = Slope * %s + Intercept)' % \
(chan, DayOrNight,
part1.replace('+', '_'),
part2.replace('+', '_'))
if isLaunch:
picPath = os.path.join(
ABC_DIR, pair, '%s_%s_ABC_%s_%s_Launch.png' %
(pair, o_name, chan, DayOrNight))
else:
picPath = os.path.join(
ABC_DIR, pair, ymd_e, '%s_%s_ABC_%s_%s_Year_%s.png' %
(pair, o_name, chan, DayOrNight, ymd_e))
plot_abc(date_D[idx_D], a_D[idx_D], b_D[idx_D], c_D[idx_D],
date_M[idx_M], a_M[idx_M], b_M[idx_M], c_M[idx_M],
picPath, title, date_s, date_e, each)
# plot TBBias ------------------------
if "tbb" not in Bias:
continue
reference_list = plt_cfg['tbb']['reference'][i]
for ref_temp in reference_list:
ref_temp_f = float(ref_temp)
# plot since launch
# bias_D = ref_temp_f - np.divide((ref_temp_f - b_D), a_D)
# bias_M = ref_temp_f - np.divide((ref_temp_f - b_M), a_M)
# 20161122 change to FY2 - IASI
bias_D = ref_temp_f - (ref_temp_f * a_D + b_D)
bias_M = ref_temp_f - (ref_temp_f * a_M + b_M)
title = 'Time Series of Brightness Temperature Bias Between\n%s %s %s %sK' % \
(pair, chan, DayOrNight, ref_temp)
if isLaunch:
picPath = os.path.join(
OMB_DIR, pair, '%s_TBBias_%s_%s_Launch_%dK.png' %
(pair, chan, DayOrNight, ref_temp))
else:
# plot latest year
picPath = os.path.join(
OMB_DIR, pair, ymd_e,
'%s_TBBias_%s_%s_Year_%s_%dK.png' %
(pair, chan, DayOrNight, ymd_e, ref_temp))
plot_tbbias(date_D[idx_D], bias_D[idx_D], date_M[idx_M],
bias_M[idx_M], picPath, title, date_s, date_e,
sat1)
# plot interpolated TBBias img (obs minus backgroud) -------------
title = 'Brightness Temperature Correction\n%s %s %s' % \
(pair, chan, DayOrNight)
if isLaunch:
picPath = os.path.join(
OMB_DIR, pair, '%s_TBBOMB_%s_%s_Launch.png' %
(pair, chan, DayOrNight))
else:
picPath = os.path.join(
OMB_DIR, pair, ymd_e, '%s_TBBOMB_%s_%s_Year_%s.png' %
(pair, chan, DayOrNight, ymd_e))
plot_omb(date_D[idx_D], a_D[idx_D], b_D[idx_D], picPath, title,
date_s, date_e)
def run(pair1, pair2, date_s, date_e):
"""
pair: sat1+sensor1_sat2+sensor2
date_s: datetime of start date
None 处理 从发星 到 有数据的最后一天
date_e: datetime of end date
None 处理 从发星 到 有数据的最后一天
"""
Log.info(u'开始运行双差统计图绘制程序%s %s-----------------' % (pair1, pair2))
isLaunch = False
if date_s is None or date_e is None:
isLaunch = True
satsen11, satsen12 = pair1.split("_")
satsen21, satsen22 = pair2.split("_")
if satsen11 != satsen21:
Log.error("%s and %s not the same, can't do double bias" % (satsen11, satsen21))
return
# 读取传感器对的配置文件
sat11, sen11 = satsen11.split('+')
sat12, sen12 = satsen12.split('+')
plt_cfg_file = os.path.join(MAIN_PATH, "cfg", '%s_%s_3d.yaml' % (sen11, sen12))
plt_cfg = loadYamlCfg(plt_cfg_file)
chans = plt_cfg["rad-rad"]["chan"]
# change sensor Name
if "VISSR" in satsen11: # VISSR -> SVISSR
satsen11 = satsen11.replace("VISSR", "SVISSR")
satsen21 = satsen21.replace("VISSR", "SVISSR")
if "METOP-" in satsen12: # METOP -> MetOp
satsen12 = satsen12.replace("METOP-", "MetOp")
if "METOP-" in satsen22: # METOP -> MetOp
satsen22 = satsen22.replace("METOP-", "MetOp")
flst = [e for e in os.listdir(StdNC_DIR) if os.path.isfile(os.path.join(StdNC_DIR, e))]
nc1_path = nc2_path = None
for each in flst:
if satsen11 in each and satsen12 in each:
nc1_path = os.path.join(StdNC_DIR, each)
if satsen21 in each and satsen22 in each:
nc2_path = os.path.join(StdNC_DIR, each)
nc1 = stdNC()
if not nc1.LoadData(nc1_path):
return
nc2 = stdNC()
if not nc2.LoadData(nc2_path):
return
time1 = nc1.time[:, 0]
time2 = nc2.time[:, 0]
tbbias1 = nc1.tbbias
tbbias2 = nc2.tbbias
reftmp = nc1.reftmp
if date_s is None: # TODO:
# timestamp_s = max(time1[0], time2[0])
# date_s = datetime.fromtimestamp(timestamp_s, tz=pytz.utc)
sat1, sen1 = satsen11.split("+")
date_s = pb_time.ymd2date(GLOBAL_CONFIG["LUANCH_DATE"][sat1])
date_s = pytz.utc.localize(date_s)
timestamp_s = calendar.timegm(date_s.timetuple())
if date_e is None:
timestamp_e = min(time1[-1], time2[-1])
date_e = datetime.fromtimestamp(timestamp_e, tz=pytz.utc)
else:
date_e = pytz.utc.localize(date_e)
timestamp_e = calendar.timegm(date_e.timetuple())
days1 = tbbias1.shape[0]
days2 = tbbias2.shape[0]
index1 = []
index2 = []
date_D = []
for i in xrange(days1):
if time1[i] < timestamp_s or time1[i] > timestamp_e:
continue
idxs2 = np.where(time2 == time1[i])[0]
if len(idxs2) != 1:
continue
date_D.append(datetime.fromtimestamp(time1[i]))
index1.append(i)
index2.append(idxs2[0])
if len(date_D) == 0:
return
for k, ch in enumerate(chans):
ch = chans[k]
ref_temp = reftmp[k]
tb1 = tbbias1[index1, k]
tb2 = tbbias2[index2, k]
bias_D = tb1 - tb2
idx = np.logical_or(tb1 < -998, tb2 < -998)
bias_D[idx] = None
date_M, bias_M = month_mean(date_D, bias_D)
title = 'Time Series of Double Bias \n%s_%s Miuns %s_%s %s %dK' % \
(satsen11, satsen12, satsen11, satsen22, ch, ref_temp)
if isLaunch:
picPath = os.path.join(DBB_DIR, '%s_%s' % (pair1, satsen22),
'%s_%s_DoubleBias_%s_Launch_%dK.png' % (pair1, satsen22, ch, ref_temp))
else:
# plot latest year
ymd_s = date_s.strftime("%Y%m%d")
ymd_e = date_e.strftime("%Y%m%d")
picPath = os.path.join(DBB_DIR, '%s_%s' % (pair1, satsen22), ymd_s,
'%s_%s_DoubleBias_%s_Year_%s_%dK.png' % (pair1, satsen22, ch, ymd_s, ref_temp))
plot_tbbias(date_D, bias_D, date_M, bias_M, picPath, title, date_s, date_e)
Log.info(u'Success')
def plot_rmd(date_d, data_d, date_m, data_m, std_m, pic_path, date_s, date_e,
sat_name, pair, chan, day_or_night, ref_temp,
xname, xname_l, xunit,
yname, yname_l, yunit,
):
if (np.isnan(data_d)).all():
Log.error('Everything is NaN: %s' % pic_path)
return
plt.style.use(os.path.join(DV_PATH, 'dv_pub_timeseries.mplstyle'))
fig = plt.figure(figsize=(6, 4))
# fig.subplots_adjust(top=0.88, bottom=0.11, left=0.12, right=0.97)
ax1 = plt.subplot2grid((1, 1), (0, 0))
# 设置 title 参数
part1, part2 = pair.split('_')
title = 'Time Series of REF Relative Bias \n{} Minus {} {} {} REF={}'.format(
part1, part2, chan, day_or_night, ref_temp)
# plot timeseries --------------------------------------------------------
timeseries_xmin = pb_time.ymd2date(
'%04d%02d01' % (date_s.year, date_s.month))
timeseries_xmax = date_e
if "FY2" in sat_name:
timeseries_ymin = -20
timeseries_ymax = 20
elif "FY3" in sat_name:
timeseries_ymin = -20
timeseries_ymax = 20
elif "FY4" in sat_name:
timeseries_ymin = -20
timeseries_ymax = 20
else:
timeseries_ymin = None
timeseries_ymax = None
timeseries_axislimit = {
"xlimit": (timeseries_xmin, timeseries_xmax),
"ylimit": (timeseries_ymin, timeseries_ymax),
}
# x y 轴标签
timeseries_label = {}
if xunit != "":
ylabel = 'Relative Bias {}'.format(xunit)
else:
ylabel = "Relative Bias %"
timeseries_label["ylabel"] = ylabel
# x, y 轴大刻度的数量，和小刻度的数量
timeseries_locator = {"locator_x": (None, None), "locator_y": (8, 2)}
# y=0 线配置
timeseries_zeroline = {"line_color": '#808080', "line_width": 1.0}
timeseries_daily = {
"marker": 'x', "color": BLUE, "linewidth": None,
"markersize": 6, "markerfacecolor": None, "markeredgecolor": BLUE,
"alpha": 0.8, "markeredgewidth": 0.3, "label": "Daily",
}
dv_pub_3d.draw_timeseries(
ax1, date_d, data_d, label=timeseries_label,
axislimit=timeseries_axislimit, locator=timeseries_locator,
zeroline=timeseries_zeroline, timeseries=timeseries_daily,
)
timeseries_monthly = {
"marker": 'o-', "color": RED, "linewidth": 0.6,
"markersize": 5, "markerfacecolor": None, "markeredgecolor": RED,
"alpha": 0.8, "markeredgewidth": 0, "label": "Monthly",
}
background_fill_timeseries = {
'x': date_m, 'y': data_m - std_m, 'y1': data_m + std_m, "color": RED,
}
dv_pub_3d.draw_timeseries(
ax1, date_m, data_m, label=timeseries_label,
axislimit=timeseries_axislimit, locator=timeseries_locator,
zeroline=timeseries_zeroline, timeseries=timeseries_monthly,
background_fill=background_fill_timeseries,
)
# --------------------
plt.tight_layout()
fig.suptitle(title, fontsize=11, fontproperties=FONT0)
fig.subplots_adjust(bottom=0.2, top=0.88)
circle1 = mpatches.Circle((74, 15), 6, color=BLUE, ec=EDGE_GRAY, lw=0)
circle2 = mpatches.Circle((164, 15), 6, color=RED, ec=EDGE_GRAY, lw=0)
fig.patches.extend([circle1, circle2])
fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0)
fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0)
ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d')
if ymd_s != ymd_e:
fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0)
else:
fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0)
fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0)
# ---------------
pb_io.make_sure_path_exists(os.path.dirname(pic_path))
plt.savefig(pic_path)
print pic_path
fig.clear()
plt.close()
def run(pair, date_s, date_e):
"""
pair: sat1+sensor1_sat2+sensor2
date_s: datetime of start date
None 处理 从发星 到 有数据的最后一天
date_e: datetime of end date
None 处理 从发星 到 有数据的最后一天
"""
# 提取参数中的卫星信息和传感器信息
part1, part2 = pair.split('_')
sat1, sensor1 = part1.split('+')
sat2, sensor2 = part2.split('+')
# 判断是否从发星开始处理
if date_s is None or date_e is None:
isLaunch = True
elif date_s is not None and date_e is not None:
isLaunch = False
else:
Log.error('Wrong date argument')
return
# 加载绘图配置文件
plt_cfg_file = os.path.join(MAIN_PATH, "cfg", "%s.plt" % pair)
plt_cfg = loadYamlCfg(plt_cfg_file)
if plt_cfg is None:
Log.error("Not find the config file: {}".format(plt_cfg_file))
return
# 设置开始时间和结束时间
if isLaunch:
if sat1 in GLOBAL_CONFIG['LUANCH_DATE']:
date_s = pb_time.ymd2date(str(GLOBAL_CONFIG['LUANCH_DATE'][sat1]))
date_e = datetime.utcnow()
else:
Log.error('%s not in LUANCH_DATE of Cfg, use the first day in txt instead.')
return
ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d')
Log.info(u"----- Start Drawing Regression-Pic, PAIR: {} -----".format(pair))
for each in plt_cfg['time_series']:
# must be in 'all', 'day', 'night'
Day_Night = ['all', 'day', 'night']
if 'time' in plt_cfg[each].keys():
Day_Night = plt_cfg[each]['time']
for i in Day_Night:
if i not in ['all', 'day', 'night']:
Day_Night.remove(i)
# 将每个对通用的属性值放到对循环，每个通道用到的属性值放到通道循环
xname, yname = each.split('-')
xname_l = plt_cfg[each]['x_name']
xunit = plt_cfg[each]['x_unit']
yname_l = plt_cfg[each]['y_name']
yunit = plt_cfg[each]['y_unit']
if 'tbb' in xname and 'tbb' in yname:
o_name = 'TBBCalCoeff'
elif 'ref' in xname and 'ref' in yname:
o_name = 'CorrcCoeff'
elif 'dn' in xname and 'ref' in yname:
o_name = 'CalCoeff'
else:
continue
for DayOrNight in Day_Night:
if DayOrNight == 'all':
DayOrNight = DayOrNight.upper() # all -> ALL
else:
DayOrNight = DayOrNight[0].upper() + DayOrNight[1:]
for i, chan in enumerate(plt_cfg[each]['chan']):
x_range = plt_cfg[each]['x_range'][i]
y_range = plt_cfg[each]['y_range'][i]
# plot slope Intercept count ------------------------
print "plot abc : {} {} {}".format(each, DayOrNight, chan)
abc_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2),
"CABR")
abc_daily_file = os.path.join(abc_path, '%s_%s_%s_%s_%s_Daily.txt' % (
part1, part2, o_name, chan, DayOrNight))
abc_monthly_file = os.path.join(abc_path, '%s_%s_%s_%s_%s_Monthly.txt' % (
part1, part2, o_name, chan, DayOrNight))
abc_day_data = get_cabr_data(abc_daily_file)
abc_month_data = get_cabr_data(abc_monthly_file)
date_D = abc_day_data["date"]
a_D = abc_day_data["slope"]
b_D = abc_day_data["intercept"]
c_D = np.log10(abc_day_data["count"])
date_M = abc_month_data["date"] + relativedelta(days=14)
a_M = abc_month_data["slope"]
b_M = abc_month_data["intercept"]
c_M = np.log10(abc_month_data["count"])
idx_D = np.where(np.logical_and(date_D >= date_s, date_D <= date_e))
idx_M = np.where(np.logical_and(date_M >= pb_time.ymd2date(ymd_s[:6] + '01'), date_M <= date_e))
title = 'Time Series of Slope Intercept & Counts %s %s\n(%s = Slope * %s + Intercept)' % \
(chan, DayOrNight,
part1.replace('+', '_'),
part2.replace('+', '_'))
if isLaunch:
picPath = os.path.join(ABC_DIR, pair,
'%s_%s_ABC_%s_%s_Launch.png' % (pair, o_name, chan, DayOrNight))
else:
picPath = os.path.join(ABC_DIR, pair, ymd_e,
'%s_%s_ABC_%s_%s_Year_%s.png' % (pair, o_name, chan, DayOrNight, ymd_e))
# 系数坐标范围
slope_range = plt_cfg[each]['slope_range'][i]
slope_min, slope_max = slope_range.split('-')
slope_min = float(slope_min)
slope_max = float(slope_max)
plot_abc(date_D[idx_D], a_D[idx_D], b_D[idx_D], c_D[idx_D],
date_M[idx_M], a_M[idx_M], b_M[idx_M], c_M[idx_M],
picPath, title, date_s, date_e, slope_min, slope_max,
each)
# plot MD ----------------------------------------------------
if xname == "ref" or xname == "tbb":
print "plot MD : {} {} {}".format(each, DayOrNight, chan)
# 从 BIAS 文件中读取数据
bias_ref_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2),
"BIAS")
file_name_monthly = os.path.join(
bias_ref_path, '%s_%s_%s_%s_%s_Monthly.txt' % (
part1, part2, xname.upper(), chan, DayOrNight))
file_name_daily = os.path.join(
bias_ref_path, '%s_%s_%s_%s_%s_Daily.txt' % (
part1, part2, xname.upper(), chan, DayOrNight))
bias_d = get_bias_data(file_name_daily)
bias_m = get_bias_data(file_name_monthly)
date_md_d = bias_d["date"]
data_md_d = bias_d["md"]
date_md_m = bias_m["date"] + relativedelta(days=14)
data_md_m = bias_m["md"]
std_md_m = bias_m["md_std"]
idx_d = np.where(np.logical_and(date_md_d >= date_s, date_md_d <= date_e))
idx_m = np.where(
np.logical_and(date_md_m >= pb_time.ymd2date(ymd_s[:6] + '01'), date_md_m <= date_e))
# 根据拟合系数进行绘制
reference_list = plt_cfg[each]['reference'][i]
for ref_temp in reference_list:
if isLaunch:
pic_path = os.path.join(
OMB_DIR, pair, '%s_%s_MD_%s_%s_Launch.png' % (
pair, xname.upper(), chan, DayOrNight))
else:
# plot latest year
pic_path = os.path.join(
OMB_DIR, pair, ymd_e,
'%s_%s_MD_%s_%s_Year_%s.png' % (
pair, xname.upper(), chan, DayOrNight, ymd_e,
))
plot_md(
date_md_d[idx_d], data_md_d[idx_d],
date_md_m[idx_m], data_md_m[idx_m],
std_md_m[idx_m],
pic_path, date_s, date_e,
sat1, pair, chan, DayOrNight, ref_temp,
xname, xname_l, xunit, x_range,
yname, yname_l, yunit, y_range,
)
# plot RMD ----------------------------------------------------
if xname == "ref" and yname == "ref":
print "plot RMD : {} {} {}".format(each, DayOrNight, chan)
# 从 BIAS 文件中读取数据
bias_ref_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2),
"BIAS")
file_name_monthly = os.path.join(
bias_ref_path, '%s_%s_%s_%s_%s_Monthly.txt' % (
part1, part2, xname.upper(), chan, DayOrNight))
file_name_daily = os.path.join(
bias_ref_path, '%s_%s_%s_%s_%s_Daily.txt' % (
part1, part2, xname.upper(), chan, DayOrNight))
bias_d = get_bias_data(file_name_daily)
bias_m = get_bias_data(file_name_monthly)
date_rmd_d = bias_d["date"]
data_rmd_d = bias_d["bias"]
date_rmd_m = bias_m["date"] + relativedelta(days=14)
data_rmd_m = bias_m["bias"]
std_rmd_m = bias_m["bias_std"]
idx_d = np.where(np.logical_and(date_rmd_d >= date_s, date_rmd_d <= date_e))
idx_m = np.where(np.logical_and(date_rmd_m >= pb_time.ymd2date(ymd_s[:6] + '01'), date_rmd_m <= date_e))
# 根据拟合系数进行绘制
reference_list = plt_cfg[each]['reference'][i]
for ref_temp in reference_list:
# ref_temp_f = float(ref_temp)
if isLaunch:
pic_path = os.path.join(
OMB_DIR, pair, '%s_RMD_%s_%s_Launch_%d.png' % (
pair, chan, DayOrNight, ref_temp * 100))
else:
# plot latest year
pic_path = os.path.join(
OMB_DIR, pair, ymd_e,
'%s_RMD_%s_%s_Year_%s_%d.png' % (
pair, chan, DayOrNight, ymd_e,
ref_temp * 100))
plot_rmd(
date_rmd_d[idx_d], data_rmd_d[idx_d],
date_rmd_m[idx_m], data_rmd_m[idx_m],
std_rmd_m[idx_m],
pic_path, date_s, date_e,
sat1, pair, chan, DayOrNight, ref_temp,
xname, xname_l, xunit,
yname, yname_l, yunit,
)
# plot TBBias ------------------------
if xname == 'tbb' and yname == 'tbb':
print "plot TBBias : {} {} {}".format(each, DayOrNight, chan)
# 从 BIAS 文件中读取数据
bias_tbb_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2),
"BIAS")
file_name_monthly = os.path.join(
bias_tbb_path, '%s_%s_%s_%s_%s_Monthly.txt' % (
part1, part2, xname.upper(), chan, DayOrNight))
file_name_daily = os.path.join(
bias_tbb_path, '%s_%s_%s_%s_%s_Daily.txt' % (
part1, part2, xname.upper(), chan, DayOrNight))
tbbias_d = get_bias_data(file_name_daily)
tbbias_m = get_bias_data(file_name_monthly)
date_tbbias_d = tbbias_d["date"]
data_tbbias_d = tbbias_d["bias"]
date_tbbias_m = tbbias_m["date"]
date_tbbias_m = date_tbbias_m + relativedelta(days=14)
data_tbbias_m = tbbias_m["bias"]
std_tbbias_m = tbbias_m["bias_std"]
idx_d = np.where(np.logical_and(date_tbbias_d >= date_s, date_tbbias_d <= date_e))
idx_m = np.where(np.logical_and(date_tbbias_m >= pb_time.ymd2date(ymd_s[:6] + '01'), date_tbbias_m <= date_e))
# 根据拟合系数进行绘制
reference_list = plt_cfg[each]['reference'][i]
for ref_temp in reference_list:
if isLaunch:
pic_path = os.path.join(
OMB_DIR, pair, '%s_TBBias_%s_%s_Launch_%dK.png' % (
pair, chan, DayOrNight, ref_temp))
else:
# plot latest year
pic_path = os.path.join(
OMB_DIR, pair, ymd_e,
'%s_TBBias_%s_%s_Year_%s_%dK.png' % (
pair, chan, DayOrNight, ymd_e,
ref_temp))
plot_tbbias(
date_tbbias_d[idx_d], data_tbbias_d[idx_d],
date_tbbias_m[idx_m], data_tbbias_m[idx_m],
std_tbbias_m[idx_m],
pic_path, date_s, date_e,
sat1, pair, chan, DayOrNight, ref_temp,
xname, xname_l, xunit,
yname, yname_l, yunit,
)
# plot interpolated TBBias img (obs minus backgroud) -------------
print "plot OMB : {} {} {}".format(each, DayOrNight, chan)
title = 'Brightness Temperature Correction\n%s %s %s' % \
(pair, chan, DayOrNight)
if isLaunch:
picPath = os.path.join(OMB_DIR, pair,
'%s_TBBOMB_%s_%s_Launch.png' % (
pair, chan, DayOrNight))
else:
picPath = os.path.join(OMB_DIR, pair, ymd_e,
'%s_TBBOMB_%s_%s_Year_%s.png' % (
pair, chan, DayOrNight, ymd_e))
plot_omb(date_D[idx_D], a_D[idx_D], b_D[idx_D],
picPath, title, date_s, date_e)
def main(sat_sensor, in_file):
"""
对L3数据进行合成
:param sat_sensor: 卫星+传感器
:param in_file: yaml 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
yc = Config(in_file)
log = LogServer(gc.path_out_log)
# 加载全局配置信息
sat_sensor1 = sat_sensor.split('_')[1]
sat_sensor2 = sat_sensor.split('_')[0]
sat1, sensor1 = sat_sensor1.split('+')
sat2, sensor2 = sat_sensor2.split('+')
# 加载卫星配置信息
s_channel1 = sc.name
s_channel2 = sc.name
timseries_channels_config = sc.timeseries_l2_channels
# 加载业务配置信息
# ######################## 开始处理 ###########################
print "-" * 100
print "Start plot verify result."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
all_files = yc.path_ipath
# 加载数据
data_absolute = dict()
data_relative = dict()
date = dict()
ref_s1_all = dict()
ref_s2_all = dict()
amount_all = dict()
date_start = ymd2date(yc.info_ymd_s)
date_end = ymd2date(yc.info_ymd_e)
point_count_min = 2
result = dict()
while date_start <= date_end:
ymd_now = date_start.strftime('%Y%m%d')
in_files = get_one_day_files(all_files=all_files, ymd=ymd_now, ext='.h5',
pattern_ymd=r'.*_(\d{8})')
cross_data = ReadCrossDataL2()
cross_data.read_cross_data(in_files=in_files)
# 循环通道数据
for channel in cross_data.data:
if channel not in data_absolute:
data_absolute[channel] = list()
if channel not in data_relative:
data_relative[channel] = list()
if channel not in date:
date[channel] = list()
if channel not in ref_s1_all:
ref_s1_all[channel] = list()
if channel not in ref_s2_all:
ref_s2_all[channel] = list()
if channel not in amount_all:
amount_all[channel] = list()
if channel not in result:
result[channel] = dict()
if not isinstance(cross_data.data[channel], dict):
continue
ref_s2 = cross_data.data[channel]['MERSI_FovMean']
fine_count = len(ref_s2)
print '---INFO--- {} Points: {}'.format(channel, fine_count)
if fine_count < point_count_min:
print '***WARNING***Dont have enough point to plot: < {}'.format(point_count_min)
continue
ref_s1 = cross_data.data[channel]['MODIS_FovMean']
# 过滤 3 倍std之外的点
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
min_ref_s2 = mean_ref_s2 - 3 * std_ref_s2
max_ref_s2 = mean_ref_s2 + 3 * std_ref_s2
idx = np.logical_and(ref_s2 >= min_ref_s2, ref_s2 <= max_ref_s2)
ref_s2 = ref_s2[idx]
ref_s1 = ref_s1[idx]
# 计算相对偏差和绝对偏差
bias = Bias()
absolute_bias = bias.absolute_deviation(ref_s1, ref_s2)
relative_bias = bias.relative_deviation(ref_s1, ref_s2)
data_absolute[channel].append(np.mean(absolute_bias))
data_relative[channel].append(np.mean(relative_bias))
date[channel].append(date_start)
mean_absolute = np.nanmean(absolute_bias)
std_absolute = np.nanstd(absolute_bias)
amount_absolute = len(absolute_bias)
median_absolute = np.nanmedian(absolute_bias)
rms_absolute = rms(absolute_bias)
mean_relative = np.nanmean(relative_bias)
std_relative = np.nanstd(relative_bias)
amount_relative = len(relative_bias)
median_relative = np.nanmedian(relative_bias)
rms_relative = rms(relative_bias)
mean_ref_s1 = np.nanmean(ref_s1)
std_ref_s1 = np.nanstd(ref_s1)
amount_ref_s1 = len(ref_s1)
median_ref_s1 = np.nanmedian(ref_s1)
rms_ref_s1 = rms(ref_s1)
ref_s1_all[channel].append(mean_ref_s1)
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
amount_ref_s2 = len(ref_s2)
median_ref_s2 = np.nanmedian(ref_s2)
rms_ref_s2 = rms(ref_s2)
ref_s2_all[channel].append(mean_ref_s2)
amount_all[channel].append(amount_ref_s1)
fix_point = sc.plot_scatter_fix_ref
f025_absolute, f025_relative = get_dif_pdif(ref_s1, ref_s2, fix_point)
result_names = ['Dif_mean', 'Dif_std', 'Dif_median', 'Dif_count',
'Dif_rms', 'Dif_025',
'PDif_mean', 'PDif_std', 'PDif_median', 'PDif_count',
'PDif_rms', 'PDif_025',
'{}_MODIS_mean'.format(channel), '{}_MODIS_std'.format(channel), '{}_MODIS_median'.format(channel), '{}_MODIS_count'.format(channel),
'{}_MODIS_rms'.format(channel),
'{}_MERSI_mean'.format(channel), '{}_MERSI_std'.format(channel), '{}_MERSI_median'.format(channel), '{}_MERSI_count'.format(channel),
'{}_MERSI_rms'.format(channel),
'Date']
datas = [mean_absolute, std_absolute, median_absolute, amount_absolute,
rms_absolute, f025_absolute,
mean_relative, std_relative, median_relative, amount_relative,
rms_relative, f025_relative,
mean_ref_s1, std_ref_s1, median_ref_s1, amount_ref_s1,
rms_ref_s1,
mean_ref_s2, std_ref_s2, median_ref_s2, amount_ref_s2,
rms_ref_s2,
ymd_now]
for result_name, data in zip(result_names, datas):
if result_name not in result[channel]:
result[channel][result_name] = list()
else:
result[channel][result_name].append(data)
date_start = date_start + relativedelta(days=1)
for channel in data_absolute:
plot_config = timseries_channels_config[channel]
dif_y_range = plot_config.get('dif_y_range')
pdif_y_range = plot_config.get('pdif_y_range')
ref_s1_y_range = plot_config.get('ref_s1_y_range')
ref_s2_y_range = plot_config.get('ref_s2_y_range')
count_y_range = plot_config.get('count_y_range')
absolute_bias = data_absolute[channel]
if len(absolute_bias) == 0:
print 'Dont have enough point to plot, is 0: {}'.format(channel)
continue
relative_bias = data_relative[channel]
date_channel = date[channel]
ref_s1_channel = ref_s1_all[channel]
ref_s2_channel = ref_s2_all[channel]
amount_channel = amount_all[channel]
# 绘制时间序列图
channel1 = channel
index_channel1 = s_channel1.index(channel1)
channel2 = s_channel2[index_channel1]
title_series = '{}_{} {}_{} Time Series'.format(sat_sensor1, channel1, sat_sensor2,
channel2)
title_ref_s1 = '{}_{} {} Time Series'.format(channel1, sat_sensor1, channel1)
title_ref_s2 = '{}_{} {} Time Series'.format(channel2, sat_sensor2, channel2)
title_amount = '{}_{} {}_{} Matched Points Count Time Series'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
y_label_series_absolute = 'Dif {}-{}'.format(sensor1, sensor2)
y_label_series_relative = 'PDif ({}/{})-1'.format(sensor1, sensor2)
y_label_ref_s1 = '{}'.format(channel1)
y_label_ref_s2 = '{}'.format(channel2)
y_label_amount = 'Count'
picture_path = yc.path_opath
# 孙凌添加,出两张图,限制Y轴坐标的图和不限制Y轴坐标的图,这里是限制Y轴坐标的图
picture_name_absolute = 'Time_Series_Dif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_relative = 'Time_Series_PDif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_ref_s1 = 'Time_Series_{}_{}.png'.format(sat_sensor1, channel1)
picture_name_ref_s2 = 'Time_Series_{}_{}.png'.format(sat_sensor2, channel2)
picture_name_amount = 'Time_Series_Count_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_file_absolute = os.path.join(picture_path, picture_name_absolute)
picture_file_relative = os.path.join(picture_path, picture_name_relative)
picture_file_ref_s1 = os.path.join(picture_path, picture_name_ref_s1)
picture_file_ref_s2 = os.path.join(picture_path, picture_name_ref_s2)
picture_file_amount = os.path.join(picture_path, picture_name_amount)
plot_time_series(day_data_x=date_channel, day_data_y=absolute_bias,
y_range=dif_y_range,
out_file=picture_file_absolute,
title=title_series, y_label=y_label_series_absolute,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=relative_bias,
y_range=pdif_y_range,
out_file=picture_file_relative,
title=title_series, y_label=y_label_series_relative,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=ref_s1_channel,
y_range=ref_s1_y_range,
out_file=picture_file_ref_s1,
title=title_ref_s1, y_label=y_label_ref_s1,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
zero_line=False)
plot_time_series(day_data_x=date_channel, day_data_y=ref_s2_channel,
y_range=ref_s2_y_range,
out_file=picture_file_ref_s2,
title=title_ref_s2, y_label=y_label_ref_s2,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
zero_line=False)
plot_time_series(day_data_x=date_channel, day_data_y=amount_channel,
y_range=count_y_range,
out_file=picture_file_amount,
title=title_amount, y_label=y_label_amount,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
plot_background=False)
# 孙凌添加,出两张图,限制Y轴坐标的图和不限制Y轴坐标的图,这里是不限制Y轴坐标的图
picture_name_absolute = 'Time_Series_Dif_{}_{}_{}_{}_NL.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_relative = 'Time_Series_PDif_{}_{}_{}_{}_NL.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_ref_s1 = 'Time_Series_{}_{}_NL.png'.format(sat_sensor1, channel1)
picture_name_ref_s2 = 'Time_Series_{}_{}_NL.png'.format(sat_sensor2, channel2)
picture_name_amount = 'Time_Series_Count_{}_{}_{}_{}_NL.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_file_absolute = os.path.join(picture_path, picture_name_absolute)
picture_file_relative = os.path.join(picture_path, picture_name_relative)
picture_file_ref_s1 = os.path.join(picture_path, picture_name_ref_s1)
picture_file_ref_s2 = os.path.join(picture_path, picture_name_ref_s2)
picture_file_amount = os.path.join(picture_path, picture_name_amount)
plot_time_series(day_data_x=date_channel, day_data_y=absolute_bias,
out_file=picture_file_absolute,
title=title_series, y_label=y_label_series_absolute,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=relative_bias,
out_file=picture_file_relative,
title=title_series, y_label=y_label_series_relative,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=ref_s1_channel,
out_file=picture_file_ref_s1,
title=title_ref_s1, y_label=y_label_ref_s1,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=ref_s2_channel,
out_file=picture_file_ref_s2,
title=title_ref_s2, y_label=y_label_ref_s2,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=amount_channel,
out_file=picture_file_amount,
title=title_amount, y_label=y_label_amount,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
plot_background=False)
# 输出HDF5
hdf5_name = '{}_{}_Dif_PDif_Daily.HDF'.format(sat_sensor1, sat_sensor2)
out_path = yc.path_opath
out_file_hdf5 = os.path.join(out_path, hdf5_name)
make_sure_path_exists(out_path)
write_hdf5(out_file_hdf5, result)
keys = amount_all.keys()
keys.sort()
for channel in keys:
print 'CHANNEL: {} POINT: {}'.format(channel, np.sum(amount_all[channel]))
print '-' * 100
# 输出月的HDF5
# 加载数据
data_absolute = dict()
data_relative = dict()
date = dict()
ref_s1_all = dict()
ref_s2_all = dict()
amount_all = dict()
date_start = ymd2date(yc.info_ymd_s)
date_end = ymd2date(yc.info_ymd_e)
result = dict()
while date_start <= date_end:
ymd_now = date_start.strftime('%Y%m')
in_files = get_one_day_files(all_files=all_files, ymd=ymd_now, ext='.h5',
pattern_ymd=r'.*_(\d{6})')
cross_data = ReadCrossDataL2()
cross_data.read_cross_data(in_files=in_files)
# 循环通道数据
for channel in cross_data.data:
if channel not in data_absolute:
data_absolute[channel] = list()
if channel not in data_relative:
data_relative[channel] = list()
if channel not in date:
date[channel] = list()
if channel not in ref_s1_all:
ref_s1_all[channel] = list()
if channel not in ref_s2_all:
ref_s2_all[channel] = list()
if channel not in amount_all:
amount_all[channel] = list()
if channel not in result:
result[channel] = dict()
ref_s1 = cross_data.data[channel]['MERSI_FovMean']
if len(ref_s1) == 0:
print '{} {} : Dont have enough point, is 0'.format(ymd_now, channel)
continue
ref_s2 = cross_data.data[channel]['MODIS_FovMean']
# 过滤 3 倍std之外的点
mean_ref_s1 = np.nanmean(ref_s1)
std_ref_s1 = np.nanstd(ref_s1)
min_ref_s1 = mean_ref_s1 - 3 * std_ref_s1
max_ref_s1 = mean_ref_s1 + 3 * std_ref_s1
idx = np.logical_and(ref_s1 >= min_ref_s1, ref_s1 <= max_ref_s1)
ref_s1 = ref_s1[idx]
ref_s2 = ref_s2[idx]
# 计算相对偏差和绝对偏差
bias = Bias()
absolute_bias = bias.absolute_deviation(ref_s1, ref_s2)
relative_bias = bias.relative_deviation(ref_s1, ref_s2)
data_absolute[channel].append(np.mean(absolute_bias))
data_relative[channel].append(np.mean(relative_bias))
date[channel].append(date_start)
mean_absolute = np.nanmean(absolute_bias)
std_absolute = np.nanstd(absolute_bias)
amount_absolute = len(absolute_bias)
median_absolute = np.nanmedian(absolute_bias)
rms_absolute = rms(absolute_bias)
mean_relative = np.nanmean(relative_bias)
std_relative = np.nanstd(relative_bias)
amount_relative = len(relative_bias)
median_relative = np.nanmedian(relative_bias)
rms_relative = rms(relative_bias)
mean_ref_s1 = np.nanmean(ref_s1)
std_ref_s1 = np.nanstd(ref_s1)
amount_ref_s1 = len(ref_s1)
median_ref_s1 = np.nanmedian(ref_s1)
rms_ref_s1 = rms(ref_s1)
ref_s1_all[channel].append(mean_ref_s1)
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
amount_ref_s2 = len(ref_s2)
median_ref_s2 = np.nanmedian(ref_s2)
rms_ref_s2 = rms(ref_s2)
ref_s2_all[channel].append(mean_ref_s2)
amount_all[channel].append(amount_ref_s1)
fix_point = sc.plot_scatter_fix_ref
f025_absolute, f025_relative = get_dif_pdif(ref_s1, ref_s2, fix_point)
result_names = ['Dif_mean', 'Dif_std', 'Dif_median', 'Dif_count',
'Dif_rms', 'Dif_025',
'PDif_mean', 'PDif_std', 'PDif_median', 'PDif_count',
'PDif_rms', 'PDif_025',
'{}_MODIS_mean'.format(channel), '{}_MODIS_std'.format(channel), '{}_MODIS_median'.format(channel), '{}_MODIS_count'.format(channel),
'{}_MODIS_rms'.format(channel),
'{}_MERSI_mean'.format(channel), '{}_MERSI_std'.format(channel), '{}_MERSI_median'.format(channel), '{}_MERSI_count'.format(channel),
'{}_MERSI_rms'.format(channel),
'Date']
datas = [mean_absolute, std_absolute, median_absolute, amount_absolute,
rms_absolute, f025_absolute,
mean_relative, std_relative, median_relative, amount_relative,
rms_relative, f025_relative,
mean_ref_s1, std_ref_s1, median_ref_s1, amount_ref_s1,
rms_ref_s1,
mean_ref_s2, std_ref_s2, median_ref_s2, amount_ref_s2,
rms_ref_s2,
ymd_now]
for result_name, data in zip(result_names, datas):
if result_name not in result[channel]:
result[channel][result_name] = list()
else:
result[channel][result_name].append(data)
date_start = date_start + relativedelta(months=1)
# 输出HDF5
hdf5_name = '{}_{}_Dif_PDif_Monthly.HDF'.format(sat_sensor1, sat_sensor2)
out_path = yc.path_opath
out_file_hdf5 = os.path.join(out_path, hdf5_name)
make_sure_path_exists(out_path)
write_hdf5(out_file_hdf5, result)
print '-' * 100
def main(yamlfile):
"""
对L3产品做长时间序列图
"""
YamlFile = yamlfile
if not os.path.isfile(YamlFile):
print 'Not Found %s' % YamlFile
sys.exit(-1)
with open(YamlFile, 'r') as stream:
cfg = yaml.load(stream)
s_time = cfg["INFO"]['stime']
e_time = cfg["INFO"]['etime']
pair = cfg["INFO"]['pair']
file_list = cfg["PATH"]['ipath']
out_path = cfg["PATH"]['opath']
print "-" * 100
# date_start = ymd2date(s_time)
# date_end = ymd2date(e_time)
h5 = LoadH5()
data_dict = {}
datetime_list = []
for in_file in file_list:
file_name = os.path.basename(in_file)
if 'L3' in pair:
list1 = ['aqua_chl1', 'aqua_kd490', 'aqua_poc', 'aqua_zsd']
list2 = ['fy_chl1', 'fy_kd490', 'fy_poc', 'fy_zsd']
reg = '.*_(\d{8}).*.HDF'
m = re.match(reg, file_name)
ymd = m.group(1)
datetime_list.append(ymd2date(ymd))
elif 'L2' in pair:
list1 = [
'aqua_chl1', 'aqua_kd490', 'aqua_poc', 'aqua_rw412',
'aqua_rw443', 'aqua_a490', 'aqua_rw490'
]
list2 = [
'fy_chl1', 'fy_kd490', 'fy_poc', 'aqua_rw412', 'fy_rw443',
'aqua_a490', 'aqua_rw490'
]
reg = '.*_(\d{14}).*.HDF'
m = re.match(reg, file_name)
ymd = m.group(1)
ymdhms = datetime.strptime('%s' % (ymd), "%Y%m%d%H%M%S")
datetime_list.append(ymdhms)
h5.load(in_file, ymd)
# 在配置中
dic_range = {
'aqua_chl1': [-0.05, 4.00],
'fy_chl1': [-0.05, 4.00],
'aqua_kd490': [-0.001, 1.00],
'fy_kd490': [-0.001, 1.00],
'aqua_poc': [-0.05, 50.00],
'fy_poc': [-0.05, 50.00],
'aqua_zsd': [-0.05, 50.00],
'fy_zsd': [-0.05, 50.00],
'aqua_a490': [-0.05, 50.00],
'fy_a490': [-0.05, 50.00],
'aqua_rw412': [-0.05, 50.00],
'fy_rw412': [-0.05, 50.00],
'aqua_rw443': [-0.05, 50.00],
'fy_rw443': [-0.05, 50.00],
'aqua_rw490': [-0.05, 50.00],
'fy_rw490': [-0.05, 50.00],
'dif_y_range': [-0.3, 0.3]
}
# 绘制每个数据集长时间
for key in h5.data.keys():
if key in ['Longitude', 'ymd', 'Latitude']:
continue
file_name = '%s_%s_%s_%s_time_series.png' % (pair, key, s_time, e_time)
out_file_png = os.path.join(out_path, file_name)
y_data = h5.data[key]
plot_time_series(day_data_x=datetime_list,
day_data_y=y_data,
y_range=dic_range[key],
out_file=out_file_png,
title='time_series %s %s ' % (pair, key),
y_label=key,
ymd_start=s_time,
ymd_end=e_time)
# 绘制差值长时间序列图
for key1, key2 in zip(list1, list2):
print 'bias---'
file_name = '%s_diff_%s_%s_%s_time_series.png' % (
pair, key1.split('_')[1], s_time, e_time)
out_file_png = os.path.join(out_path, file_name)
y_data = h5.data[key]
diff_data = h5.data[key1] - h5.data[key2]
plot_time_series(day_data_x=datetime_list,
day_data_y=diff_data,
y_range=dic_range['dif_y_range'],
out_file=out_file_png,
title='time_series %s %s %s' % (pair, key1, key2),
y_label='%s-%s' % (key1, key2),
ymd_start=s_time,
ymd_end=e_time)
def plot_bias(date_D, bias_D, date_M, bias_M, picPath, title, date_s, date_e,
each, date_type, ylim_min, ylim_max):
"""
画偏差时序折线图
"""
plt.style.use(os.path.join(dvPath, 'dv_pub_timeseries.mplstyle'))
fig = plt.figure(figsize=(6, 4))
# plt.subplots_adjust(left=0.13, right=0.95, bottom=0.11, top=0.97)
plt.plot(date_D,
bias_D,
'x',
ms=6,
markerfacecolor=None,
markeredgecolor=BLUE,
alpha=0.8,
mew=0.3,
label='Daily')
plt.plot(date_M, bias_M, 'o-', ms=5, lw=0.6, c=RED, mew=0, label='Monthly')
plt.grid(True)
plt.ylabel('%s %s' % (each, date_type), fontsize=11, fontproperties=FONT0)
xlim_min = pb_time.ymd2date('%04d%02d01' % (date_s.year, date_s.month))
xlim_max = date_e
plt.plot([xlim_min, xlim_max], [0, 0], '#808080') # 在 y=0 绘制一条深灰色直线
plt.xlim(xlim_min, xlim_max)
plt.ylim(ylim_min, ylim_max)
ax = plt.gca()
# format the ticks
interval = (ylim_max - ylim_min) / 8 # 8 个间隔
minibar = interval / 2.
setXLocator(ax, xlim_min, xlim_max)
set_tick_font(ax)
# 如果范围为浮点数，需要进行一次格式化，否则图像不会显示最后一个刻度
if isinstance(interval, float):
interval = float('%.5f' % interval)
minibar = float('%.5f' % minibar)
ax.yaxis.set_major_locator(MultipleLocator(interval))
ax.yaxis.set_minor_locator(MultipleLocator(minibar))
# title
plt.title(title, fontsize=12, fontproperties=FONT0)
plt.tight_layout()
# --------------------
fig.subplots_adjust(bottom=0.2)
circle1 = mpatches.Circle((74, 15), 6, color=BLUE, ec=EDGE_GRAY, lw=0)
circle2 = mpatches.Circle((164, 15), 6, color=RED, ec=EDGE_GRAY, lw=0)
fig.patches.extend([circle1, circle2])
fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0)
fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0)
ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d')
if ymd_s != ymd_e:
fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0)
else:
fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0)
fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0)
# ---------------
pb_io.make_sure_path_exists(os.path.dirname(picPath))
plt.savefig(picPath)
fig.clear()
plt.close()
