def main(yamlfile): """ 对L3产品做长时间序列图 """ YamlFile = yamlfile if not os.path.isfile(YamlFile): print 'Not Found %s' % YamlFile sys.exit(-1) with open(YamlFile, 'r') as stream: cfg = yaml.load(stream) ymd = cfg["INFO"]['ymd'] pair = cfg["INFO"]['pair'] file_list = cfg["PATH"]['ipath'] out_path = cfg["PATH"]['opath'] print "-" * 100 # date_start = ymd2date(s_time) # date_end = ymd2date(e_time) for in_file in file_list: h5 = LoadH5() data_dict = {} datetime_list = [] file_name = os.path.basename(in_file) if 'L3' in pair: reg = '.*_(\d{8}).*.HDF' m = re.match(reg, file_name) ymd = m.group(1) datetime_list.append(ymd2date(ymd)) elif 'L2' in pair: reg = '.*_(\d{14}).*.HDF' m = re.match(reg, file_name) ymd = m.group(1) ymdhms = datetime.strptime('%s' % (ymd), "%Y%m%d%H%M%S") datetime_list.append(ymdhms) h5.load(in_file, ymd) dict_data = {} for key in h5.data.keys(): if key in ['Longitude', 'ymd', 'Latitude']: continue data = h5.data[key] idx = np.where(data > 0) mean = np.mean(data[idx]) # print key, type(mean) dict_data[key] = mean.reshape(1, 1) # print dict_data file_name = '%s_%s_COMBINE.HDF' % (pair, ymd) ofile = os.path.join(out_path, file_name) write_h5(ofile, dict_data)
def plot_abc(date_D, a_D, b_D, c_D, date_M, a_M, b_M, c_M, picPath, title, date_s, date_e, var): fig = plt.figure(figsize=(6, 6)) ax1 = plt.subplot(311) ax2 = plt.subplot(312, sharex=ax1) ax3 = plt.subplot(313, sharex=ax1) # format the Xticks xlim_min = pb_time.ymd2date('%04d%02d01' % (date_s.year, date_s.month)) xlim_max = date_e ax1.set_xlim(xlim_min, xlim_max) # format the Yticks\ if var == "tbb-tbb" or var == "ref-ref": ax1.set_ylim(0, 2) ax1.yaxis.set_major_locator(MultipleLocator(0.5)) ax1.yaxis.set_minor_locator(MultipleLocator(0.1)) elif var == "dn-ref": ax1.set_ylim(0.0002, 0.0007) ax1.yaxis.set_major_locator(MultipleLocator(0.0001)) ax1.yaxis.set_minor_locator(MultipleLocator(0.00005)) if var == "tbb-tbb": ax2.set_ylim(-30, 30) ax2.yaxis.set_major_locator(MultipleLocator(10)) ax2.yaxis.set_minor_locator(MultipleLocator(2)) elif var == "ref-ref": ax2.set_ylim(-0.1, 0.1) ax2.yaxis.set_major_locator(MultipleLocator(0.02)) ax2.yaxis.set_minor_locator(MultipleLocator(0.01)) elif var == "dn-ref": ax2.set_ylim(-0.08, 0.08) ax2.yaxis.set_major_locator(MultipleLocator(0.02)) ax2.yaxis.set_minor_locator(MultipleLocator(0.01)) ax3.set_ylim(0, 7) ax3.yaxis.set_major_locator(MultipleLocator(1)) ax3.yaxis.set_minor_locator(MultipleLocator(0.2)) # plot ax1 ------------------------------------------------- plt.sca(ax1) plt.plot(date_D, a_D, 'x', ms=5, markerfacecolor=None, markeredgecolor=BLUE, alpha=0.8, mew=0.3, label='Daily') plt.plot(date_M, a_M, 'o-', ms=4, lw=0.6, c=RED, mew=0, label='Monthly') plt.ylabel('Slope', fontsize=11, fontproperties=FONT0) plt.grid(True) plt.title(title, fontsize=12, fontproperties=FONT0) set_tick_font(ax1) plt.setp(ax1.get_xticklabels(), visible=False) # plot ax2 ------------------------------------------------- plt.sca(ax2) plt.plot(date_D, b_D, 'x', ms=5, markerfacecolor=None, markeredgecolor=BLUE, alpha=0.8, mew=0.3, label='Daily') plt.plot(date_M, b_M, 'o-', ms=4, lw=0.6, c=RED, mew=0, label='Monthly') plt.ylabel('Intercept', fontsize=11, fontproperties=FONT0) plt.grid(True) set_tick_font(ax2) plt.setp(ax2.get_xticklabels(), visible=False) # point number ------------------------------------------------- plt.sca(ax3) plt.fill_between(date_D, 0, c_D, edgecolor=BLUE, facecolor=BLUE, alpha=0.6) # plt.fill_between(date_M, 0, c_M, # edgecolor=RED, facecolor=RED, alpha=0.5) # plt.bar(date_M, c_M, width=1, align='edge', # "center", # color=RED, linewidth=0) plt.plot(date_M, c_M, 'o-', ms=4, lw=0.6, c=RED, mew=0, label='Monthly') plt.ylabel('Number of sample points\nlog (base = 10)', fontsize=11, fontproperties=FONT0) plt.grid(True) set_tick_font(ax3) setXLocator(ax3, xlim_min, xlim_max) # circle1 = mpatches.Circle((430, 563), 5, color=BLUE, ec=EDGE_GRAY, lw=0) # circle2 = mpatches.Circle((508, 563), 5, color=RED, ec=EDGE_GRAY, lw=0) # fig.patches.extend([circle1, circle2]) # # fig.text(0.74, 0.93, 'Daily', color=BLUE, fontproperties=FONT0) # fig.text(0.86, 0.93, 'Monthly', color=RED, fontproperties=FONT0) #--------------- plt.tight_layout() fig.subplots_adjust(bottom=0.14) circle1 = mpatches.Circle((74, 18), 6, color=BLUE, ec=EDGE_GRAY, lw=0) circle2 = mpatches.Circle((164, 18), 6, color=RED, ec=EDGE_GRAY, lw=0) fig.patches.extend([circle1, circle2]) fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0) fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0) ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d') if ymd_s != ymd_e: fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0) else: fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0) fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0) #--------------- pb_io.make_sure_path_exists(os.path.dirname(picPath)) fig.savefig(picPath) plt.close() fig.clear
def plot_tbbias(date_D, bias_D, date_M, bias_M, picPath, title, date_s, date_e, satName): ''' 画偏差时序折线图 ''' fig = plt.figure(figsize=(6, 4)) # plt.subplots_adjust(left=0.13, right=0.95, bottom=0.11, top=0.97) if (np.isnan(bias_D)).all(): Log.error('Everything is NaN: %s' % picPath) return plt.plot(date_D, bias_D, 'x', ms=6, markerfacecolor=None, markeredgecolor=BLUE, alpha=0.8, mew=0.3, label='Daily') plt.plot(date_M, bias_M, 'o-', ms=5, lw=0.6, c=RED, mew=0, label='Monthly') plt.grid(True) plt.ylabel('DTB($K$)', fontsize=11, fontproperties=FONT0) xlim_min = pb_time.ymd2date('%04d%02d01' % (date_s.year, date_s.month)) xlim_max = date_e plt.xlim(xlim_min, xlim_max) if "FY2" in satName: plt.ylim(-8, 8) elif "FY3" in satName: plt.ylim(-8, 8) elif "FY4" in satName: plt.ylim(-2, 2) ax = plt.gca() # format the ticks setXLocator(ax, xlim_min, xlim_max) set_tick_font(ax) # title plt.title(title, fontsize=12, fontproperties=FONT0) plt.tight_layout() #-------------------- fig.subplots_adjust(bottom=0.2) circle1 = mpatches.Circle((74, 15), 6, color=BLUE, ec=EDGE_GRAY, lw=0) circle2 = mpatches.Circle((164, 15), 6, color=RED, ec=EDGE_GRAY, lw=0) fig.patches.extend([circle1, circle2]) fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0) fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0) ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d') if ymd_s != ymd_e: fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0) else: fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0) fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0) #--------------- pb_io.make_sure_path_exists(os.path.dirname(picPath)) plt.savefig(picPath) fig.clear() plt.close()
def run(pair, date_s, date_e): ''' pair: sat1+sensor1_sat2+sensor2 date_s: datetime of start date None 处理 从发星 到 有数据的最后一天 date_e: datetime of end date None 处理 从发星 到 有数据的最后一天 ''' isLaunch = False if date_s is None or date_e is None: isLaunch = True part1, part2 = pair.split('_') sat1, sensor1 = part1.split('+') sat2, sensor2 = part2.split('+') # load yaml config file plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2)) plt_cfg = pb_io.loadYamlCfg(plt_cfg_file) if plt_cfg is None: return # must be in 'all', 'day', 'night' Day_Night = ['all', 'day', 'night'] if "day_night" in plt_cfg.keys(): Day_Night = plt_cfg['day_night'] for each in Day_Night: if each not in ['all', 'day', 'night']: Day_Night.remove(each) Bias = ["tbb"] if "bias" in plt_cfg.keys(): Bias = plt_cfg["bias"] co = coeff_abr(pair) if co.error: return for each in plt_cfg['plot']: xname, yname = each.split('-') if 'tbb' in xname and 'tbb' in yname: o_name = 'TBBCalCoeff' elif 'ref' in xname and 'ref' in yname: o_name = 'CorrcCoeff' elif 'dn' in xname and 'ref' in yname: o_name = 'CalCoeff' else: continue for DayOrNight in Day_Night: if DayOrNight == 'all': DayOrNight = DayOrNight.upper() # all -> ALL else: DayOrNight = DayOrNight[0].upper() + DayOrNight[1:] co.loadCoeff(o_name, DayOrNight) if co.error: continue date_D = co.Coeff_D['date'] date_M = co.Coeff_M['date'] if isLaunch: if sat1 in inCfg['LUANCH_DATE']: date_s = pb_time.ymd2date(str(inCfg['LUANCH_DATE'][sat1])) else: Log.error( '%s not in LUANCH_DATE of Cfg, use the first day in txt instead.' ) date_s = date_D[0] date_e = date_D[-1] ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d') Log.info(u"----- Start Drawing ABC TBBias Pic, PAIR: {}, YMD: {}-{}: " \ u"{}-----".format(pair, ymd_s, ymd_e, DayOrNight)) idx_D = np.where(np.logical_and(date_D >= date_s, date_D <= date_e)) idx_M = np.where( np.logical_and(date_M >= pb_time.ymd2date(ymd_s[:6] + '01'), date_M <= date_e)) for i, chan in enumerate(plt_cfg['chan']): a_D = co.Coeff_D['a_%s' % chan] b_D = co.Coeff_D['b_%s' % chan] c_D = np.log10(co.Coeff_D['c_%s' % chan]) a_M = co.Coeff_M['a_%s' % chan] b_M = co.Coeff_M['b_%s' % chan] c_M = np.log10(co.Coeff_M['c_%s' % chan]) # plot slope Intercept count ------------------------ title = 'Time Series of Slope Intercept & Counts %s %s\n(%s = Slope * %s + Intercept)' % \ (chan, DayOrNight, part1.replace('+', '_'), part2.replace('+', '_')) if isLaunch: picPath = os.path.join( ABC_DIR, pair, '%s_%s_ABC_%s_%s_Launch.png' % (pair, o_name, chan, DayOrNight)) else: picPath = os.path.join( ABC_DIR, pair, ymd_e, '%s_%s_ABC_%s_%s_Year_%s.png' % (pair, o_name, chan, DayOrNight, ymd_e)) plot_abc(date_D[idx_D], a_D[idx_D], b_D[idx_D], c_D[idx_D], date_M[idx_M], a_M[idx_M], b_M[idx_M], c_M[idx_M], picPath, title, date_s, date_e, each) # plot TBBias ------------------------ if "tbb" not in Bias: continue reference_list = plt_cfg['tbb']['reference'][i] for ref_temp in reference_list: ref_temp_f = float(ref_temp) # plot since launch # bias_D = ref_temp_f - np.divide((ref_temp_f - b_D), a_D) # bias_M = ref_temp_f - np.divide((ref_temp_f - b_M), a_M) # 20161122 change to FY2 - IASI bias_D = ref_temp_f - (ref_temp_f * a_D + b_D) bias_M = ref_temp_f - (ref_temp_f * a_M + b_M) title = 'Time Series of Brightness Temperature Bias Between\n%s %s %s %sK' % \ (pair, chan, DayOrNight, ref_temp) if isLaunch: picPath = os.path.join( OMB_DIR, pair, '%s_TBBias_%s_%s_Launch_%dK.png' % (pair, chan, DayOrNight, ref_temp)) else: # plot latest year picPath = os.path.join( OMB_DIR, pair, ymd_e, '%s_TBBias_%s_%s_Year_%s_%dK.png' % (pair, chan, DayOrNight, ymd_e, ref_temp)) plot_tbbias(date_D[idx_D], bias_D[idx_D], date_M[idx_M], bias_M[idx_M], picPath, title, date_s, date_e, sat1) # plot interpolated TBBias img (obs minus backgroud) ------------- title = 'Brightness Temperature Correction\n%s %s %s' % \ (pair, chan, DayOrNight) if isLaunch: picPath = os.path.join( OMB_DIR, pair, '%s_TBBOMB_%s_%s_Launch.png' % (pair, chan, DayOrNight)) else: picPath = os.path.join( OMB_DIR, pair, ymd_e, '%s_TBBOMB_%s_%s_Year_%s.png' % (pair, chan, DayOrNight, ymd_e)) plot_omb(date_D[idx_D], a_D[idx_D], b_D[idx_D], picPath, title, date_s, date_e)
def run(pair1, pair2, date_s, date_e): """ pair: sat1+sensor1_sat2+sensor2 date_s: datetime of start date None 处理 从发星 到 有数据的最后一天 date_e: datetime of end date None 处理 从发星 到 有数据的最后一天 """ Log.info(u'开始运行双差统计图绘制程序%s %s-----------------' % (pair1, pair2)) isLaunch = False if date_s is None or date_e is None: isLaunch = True satsen11, satsen12 = pair1.split("_") satsen21, satsen22 = pair2.split("_") if satsen11 != satsen21: Log.error("%s and %s not the same, can't do double bias" % (satsen11, satsen21)) return # 读取传感器对的配置文件 sat11, sen11 = satsen11.split('+') sat12, sen12 = satsen12.split('+') plt_cfg_file = os.path.join(MAIN_PATH, "cfg", '%s_%s_3d.yaml' % (sen11, sen12)) plt_cfg = loadYamlCfg(plt_cfg_file) chans = plt_cfg["rad-rad"]["chan"] # change sensor Name if "VISSR" in satsen11: # VISSR -> SVISSR satsen11 = satsen11.replace("VISSR", "SVISSR") satsen21 = satsen21.replace("VISSR", "SVISSR") if "METOP-" in satsen12: # METOP -> MetOp satsen12 = satsen12.replace("METOP-", "MetOp") if "METOP-" in satsen22: # METOP -> MetOp satsen22 = satsen22.replace("METOP-", "MetOp") flst = [e for e in os.listdir(StdNC_DIR) if os.path.isfile(os.path.join(StdNC_DIR, e))] nc1_path = nc2_path = None for each in flst: if satsen11 in each and satsen12 in each: nc1_path = os.path.join(StdNC_DIR, each) if satsen21 in each and satsen22 in each: nc2_path = os.path.join(StdNC_DIR, each) nc1 = stdNC() if not nc1.LoadData(nc1_path): return nc2 = stdNC() if not nc2.LoadData(nc2_path): return time1 = nc1.time[:, 0] time2 = nc2.time[:, 0] tbbias1 = nc1.tbbias tbbias2 = nc2.tbbias reftmp = nc1.reftmp if date_s is None: # TODO: # timestamp_s = max(time1[0], time2[0]) # date_s = datetime.fromtimestamp(timestamp_s, tz=pytz.utc) sat1, sen1 = satsen11.split("+") date_s = pb_time.ymd2date(GLOBAL_CONFIG["LUANCH_DATE"][sat1]) date_s = pytz.utc.localize(date_s) timestamp_s = calendar.timegm(date_s.timetuple()) if date_e is None: timestamp_e = min(time1[-1], time2[-1]) date_e = datetime.fromtimestamp(timestamp_e, tz=pytz.utc) else: date_e = pytz.utc.localize(date_e) timestamp_e = calendar.timegm(date_e.timetuple()) days1 = tbbias1.shape[0] days2 = tbbias2.shape[0] index1 = [] index2 = [] date_D = [] for i in xrange(days1): if time1[i] < timestamp_s or time1[i] > timestamp_e: continue idxs2 = np.where(time2 == time1[i])[0] if len(idxs2) != 1: continue date_D.append(datetime.fromtimestamp(time1[i])) index1.append(i) index2.append(idxs2[0]) if len(date_D) == 0: return for k, ch in enumerate(chans): ch = chans[k] ref_temp = reftmp[k] tb1 = tbbias1[index1, k] tb2 = tbbias2[index2, k] bias_D = tb1 - tb2 idx = np.logical_or(tb1 < -998, tb2 < -998) bias_D[idx] = None date_M, bias_M = month_mean(date_D, bias_D) title = 'Time Series of Double Bias \n%s_%s Miuns %s_%s %s %dK' % \ (satsen11, satsen12, satsen11, satsen22, ch, ref_temp) if isLaunch: picPath = os.path.join(DBB_DIR, '%s_%s' % (pair1, satsen22), '%s_%s_DoubleBias_%s_Launch_%dK.png' % (pair1, satsen22, ch, ref_temp)) else: # plot latest year ymd_s = date_s.strftime("%Y%m%d") ymd_e = date_e.strftime("%Y%m%d") picPath = os.path.join(DBB_DIR, '%s_%s' % (pair1, satsen22), ymd_s, '%s_%s_DoubleBias_%s_Year_%s_%dK.png' % (pair1, satsen22, ch, ymd_s, ref_temp)) plot_tbbias(date_D, bias_D, date_M, bias_M, picPath, title, date_s, date_e) Log.info(u'Success')
def plot_rmd(date_d, data_d, date_m, data_m, std_m, pic_path, date_s, date_e, sat_name, pair, chan, day_or_night, ref_temp, xname, xname_l, xunit, yname, yname_l, yunit, ): if (np.isnan(data_d)).all(): Log.error('Everything is NaN: %s' % pic_path) return plt.style.use(os.path.join(DV_PATH, 'dv_pub_timeseries.mplstyle')) fig = plt.figure(figsize=(6, 4)) # fig.subplots_adjust(top=0.88, bottom=0.11, left=0.12, right=0.97) ax1 = plt.subplot2grid((1, 1), (0, 0)) # 设置 title 参数 part1, part2 = pair.split('_') title = 'Time Series of REF Relative Bias \n{} Minus {} {} {} REF={}'.format( part1, part2, chan, day_or_night, ref_temp) # plot timeseries -------------------------------------------------------- timeseries_xmin = pb_time.ymd2date( '%04d%02d01' % (date_s.year, date_s.month)) timeseries_xmax = date_e if "FY2" in sat_name: timeseries_ymin = -20 timeseries_ymax = 20 elif "FY3" in sat_name: timeseries_ymin = -20 timeseries_ymax = 20 elif "FY4" in sat_name: timeseries_ymin = -20 timeseries_ymax = 20 else: timeseries_ymin = None timeseries_ymax = None timeseries_axislimit = { "xlimit": (timeseries_xmin, timeseries_xmax), "ylimit": (timeseries_ymin, timeseries_ymax), } # x y 轴标签 timeseries_label = {} if xunit != "": ylabel = 'Relative Bias {}'.format(xunit) else: ylabel = "Relative Bias %" timeseries_label["ylabel"] = ylabel # x, y 轴大刻度的数量,和小刻度的数量 timeseries_locator = {"locator_x": (None, None), "locator_y": (8, 2)} # y=0 线配置 timeseries_zeroline = {"line_color": '#808080', "line_width": 1.0} timeseries_daily = { "marker": 'x', "color": BLUE, "linewidth": None, "markersize": 6, "markerfacecolor": None, "markeredgecolor": BLUE, "alpha": 0.8, "markeredgewidth": 0.3, "label": "Daily", } dv_pub_3d.draw_timeseries( ax1, date_d, data_d, label=timeseries_label, axislimit=timeseries_axislimit, locator=timeseries_locator, zeroline=timeseries_zeroline, timeseries=timeseries_daily, ) timeseries_monthly = { "marker": 'o-', "color": RED, "linewidth": 0.6, "markersize": 5, "markerfacecolor": None, "markeredgecolor": RED, "alpha": 0.8, "markeredgewidth": 0, "label": "Monthly", } background_fill_timeseries = { 'x': date_m, 'y': data_m - std_m, 'y1': data_m + std_m, "color": RED, } dv_pub_3d.draw_timeseries( ax1, date_m, data_m, label=timeseries_label, axislimit=timeseries_axislimit, locator=timeseries_locator, zeroline=timeseries_zeroline, timeseries=timeseries_monthly, background_fill=background_fill_timeseries, ) # -------------------- plt.tight_layout() fig.suptitle(title, fontsize=11, fontproperties=FONT0) fig.subplots_adjust(bottom=0.2, top=0.88) circle1 = mpatches.Circle((74, 15), 6, color=BLUE, ec=EDGE_GRAY, lw=0) circle2 = mpatches.Circle((164, 15), 6, color=RED, ec=EDGE_GRAY, lw=0) fig.patches.extend([circle1, circle2]) fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0) fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0) ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d') if ymd_s != ymd_e: fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0) else: fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0) fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0) # --------------- pb_io.make_sure_path_exists(os.path.dirname(pic_path)) plt.savefig(pic_path) print pic_path fig.clear() plt.close()
def run(pair, date_s, date_e): """ pair: sat1+sensor1_sat2+sensor2 date_s: datetime of start date None 处理 从发星 到 有数据的最后一天 date_e: datetime of end date None 处理 从发星 到 有数据的最后一天 """ # 提取参数中的卫星信息和传感器信息 part1, part2 = pair.split('_') sat1, sensor1 = part1.split('+') sat2, sensor2 = part2.split('+') # 判断是否从发星开始处理 if date_s is None or date_e is None: isLaunch = True elif date_s is not None and date_e is not None: isLaunch = False else: Log.error('Wrong date argument') return # 加载绘图配置文件 plt_cfg_file = os.path.join(MAIN_PATH, "cfg", "%s.plt" % pair) plt_cfg = loadYamlCfg(plt_cfg_file) if plt_cfg is None: Log.error("Not find the config file: {}".format(plt_cfg_file)) return # 设置开始时间和结束时间 if isLaunch: if sat1 in GLOBAL_CONFIG['LUANCH_DATE']: date_s = pb_time.ymd2date(str(GLOBAL_CONFIG['LUANCH_DATE'][sat1])) date_e = datetime.utcnow() else: Log.error('%s not in LUANCH_DATE of Cfg, use the first day in txt instead.') return ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d') Log.info(u"----- Start Drawing Regression-Pic, PAIR: {} -----".format(pair)) for each in plt_cfg['time_series']: # must be in 'all', 'day', 'night' Day_Night = ['all', 'day', 'night'] if 'time' in plt_cfg[each].keys(): Day_Night = plt_cfg[each]['time'] for i in Day_Night: if i not in ['all', 'day', 'night']: Day_Night.remove(i) # 将每个对通用的属性值放到对循环,每个通道用到的属性值放到通道循环 xname, yname = each.split('-') xname_l = plt_cfg[each]['x_name'] xunit = plt_cfg[each]['x_unit'] yname_l = plt_cfg[each]['y_name'] yunit = plt_cfg[each]['y_unit'] if 'tbb' in xname and 'tbb' in yname: o_name = 'TBBCalCoeff' elif 'ref' in xname and 'ref' in yname: o_name = 'CorrcCoeff' elif 'dn' in xname and 'ref' in yname: o_name = 'CalCoeff' else: continue for DayOrNight in Day_Night: if DayOrNight == 'all': DayOrNight = DayOrNight.upper() # all -> ALL else: DayOrNight = DayOrNight[0].upper() + DayOrNight[1:] for i, chan in enumerate(plt_cfg[each]['chan']): x_range = plt_cfg[each]['x_range'][i] y_range = plt_cfg[each]['y_range'][i] # plot slope Intercept count ------------------------ print "plot abc : {} {} {}".format(each, DayOrNight, chan) abc_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2), "CABR") abc_daily_file = os.path.join(abc_path, '%s_%s_%s_%s_%s_Daily.txt' % ( part1, part2, o_name, chan, DayOrNight)) abc_monthly_file = os.path.join(abc_path, '%s_%s_%s_%s_%s_Monthly.txt' % ( part1, part2, o_name, chan, DayOrNight)) abc_day_data = get_cabr_data(abc_daily_file) abc_month_data = get_cabr_data(abc_monthly_file) date_D = abc_day_data["date"] a_D = abc_day_data["slope"] b_D = abc_day_data["intercept"] c_D = np.log10(abc_day_data["count"]) date_M = abc_month_data["date"] + relativedelta(days=14) a_M = abc_month_data["slope"] b_M = abc_month_data["intercept"] c_M = np.log10(abc_month_data["count"]) idx_D = np.where(np.logical_and(date_D >= date_s, date_D <= date_e)) idx_M = np.where(np.logical_and(date_M >= pb_time.ymd2date(ymd_s[:6] + '01'), date_M <= date_e)) title = 'Time Series of Slope Intercept & Counts %s %s\n(%s = Slope * %s + Intercept)' % \ (chan, DayOrNight, part1.replace('+', '_'), part2.replace('+', '_')) if isLaunch: picPath = os.path.join(ABC_DIR, pair, '%s_%s_ABC_%s_%s_Launch.png' % (pair, o_name, chan, DayOrNight)) else: picPath = os.path.join(ABC_DIR, pair, ymd_e, '%s_%s_ABC_%s_%s_Year_%s.png' % (pair, o_name, chan, DayOrNight, ymd_e)) # 系数坐标范围 slope_range = plt_cfg[each]['slope_range'][i] slope_min, slope_max = slope_range.split('-') slope_min = float(slope_min) slope_max = float(slope_max) plot_abc(date_D[idx_D], a_D[idx_D], b_D[idx_D], c_D[idx_D], date_M[idx_M], a_M[idx_M], b_M[idx_M], c_M[idx_M], picPath, title, date_s, date_e, slope_min, slope_max, each) # plot MD ---------------------------------------------------- if xname == "ref" or xname == "tbb": print "plot MD : {} {} {}".format(each, DayOrNight, chan) # 从 BIAS 文件中读取数据 bias_ref_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2), "BIAS") file_name_monthly = os.path.join( bias_ref_path, '%s_%s_%s_%s_%s_Monthly.txt' % ( part1, part2, xname.upper(), chan, DayOrNight)) file_name_daily = os.path.join( bias_ref_path, '%s_%s_%s_%s_%s_Daily.txt' % ( part1, part2, xname.upper(), chan, DayOrNight)) bias_d = get_bias_data(file_name_daily) bias_m = get_bias_data(file_name_monthly) date_md_d = bias_d["date"] data_md_d = bias_d["md"] date_md_m = bias_m["date"] + relativedelta(days=14) data_md_m = bias_m["md"] std_md_m = bias_m["md_std"] idx_d = np.where(np.logical_and(date_md_d >= date_s, date_md_d <= date_e)) idx_m = np.where( np.logical_and(date_md_m >= pb_time.ymd2date(ymd_s[:6] + '01'), date_md_m <= date_e)) # 根据拟合系数进行绘制 reference_list = plt_cfg[each]['reference'][i] for ref_temp in reference_list: if isLaunch: pic_path = os.path.join( OMB_DIR, pair, '%s_%s_MD_%s_%s_Launch.png' % ( pair, xname.upper(), chan, DayOrNight)) else: # plot latest year pic_path = os.path.join( OMB_DIR, pair, ymd_e, '%s_%s_MD_%s_%s_Year_%s.png' % ( pair, xname.upper(), chan, DayOrNight, ymd_e, )) plot_md( date_md_d[idx_d], data_md_d[idx_d], date_md_m[idx_m], data_md_m[idx_m], std_md_m[idx_m], pic_path, date_s, date_e, sat1, pair, chan, DayOrNight, ref_temp, xname, xname_l, xunit, x_range, yname, yname_l, yunit, y_range, ) # plot RMD ---------------------------------------------------- if xname == "ref" and yname == "ref": print "plot RMD : {} {} {}".format(each, DayOrNight, chan) # 从 BIAS 文件中读取数据 bias_ref_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2), "BIAS") file_name_monthly = os.path.join( bias_ref_path, '%s_%s_%s_%s_%s_Monthly.txt' % ( part1, part2, xname.upper(), chan, DayOrNight)) file_name_daily = os.path.join( bias_ref_path, '%s_%s_%s_%s_%s_Daily.txt' % ( part1, part2, xname.upper(), chan, DayOrNight)) bias_d = get_bias_data(file_name_daily) bias_m = get_bias_data(file_name_monthly) date_rmd_d = bias_d["date"] data_rmd_d = bias_d["bias"] date_rmd_m = bias_m["date"] + relativedelta(days=14) data_rmd_m = bias_m["bias"] std_rmd_m = bias_m["bias_std"] idx_d = np.where(np.logical_and(date_rmd_d >= date_s, date_rmd_d <= date_e)) idx_m = np.where(np.logical_and(date_rmd_m >= pb_time.ymd2date(ymd_s[:6] + '01'), date_rmd_m <= date_e)) # 根据拟合系数进行绘制 reference_list = plt_cfg[each]['reference'][i] for ref_temp in reference_list: # ref_temp_f = float(ref_temp) if isLaunch: pic_path = os.path.join( OMB_DIR, pair, '%s_RMD_%s_%s_Launch_%d.png' % ( pair, chan, DayOrNight, ref_temp * 100)) else: # plot latest year pic_path = os.path.join( OMB_DIR, pair, ymd_e, '%s_RMD_%s_%s_Year_%s_%d.png' % ( pair, chan, DayOrNight, ymd_e, ref_temp * 100)) plot_rmd( date_rmd_d[idx_d], data_rmd_d[idx_d], date_rmd_m[idx_m], data_rmd_m[idx_m], std_rmd_m[idx_m], pic_path, date_s, date_e, sat1, pair, chan, DayOrNight, ref_temp, xname, xname_l, xunit, yname, yname_l, yunit, ) # plot TBBias ------------------------ if xname == 'tbb' and yname == 'tbb': print "plot TBBias : {} {} {}".format(each, DayOrNight, chan) # 从 BIAS 文件中读取数据 bias_tbb_path = os.path.join(ABR_DIR, '%s_%s' % (part1, part2), "BIAS") file_name_monthly = os.path.join( bias_tbb_path, '%s_%s_%s_%s_%s_Monthly.txt' % ( part1, part2, xname.upper(), chan, DayOrNight)) file_name_daily = os.path.join( bias_tbb_path, '%s_%s_%s_%s_%s_Daily.txt' % ( part1, part2, xname.upper(), chan, DayOrNight)) tbbias_d = get_bias_data(file_name_daily) tbbias_m = get_bias_data(file_name_monthly) date_tbbias_d = tbbias_d["date"] data_tbbias_d = tbbias_d["bias"] date_tbbias_m = tbbias_m["date"] date_tbbias_m = date_tbbias_m + relativedelta(days=14) data_tbbias_m = tbbias_m["bias"] std_tbbias_m = tbbias_m["bias_std"] idx_d = np.where(np.logical_and(date_tbbias_d >= date_s, date_tbbias_d <= date_e)) idx_m = np.where(np.logical_and(date_tbbias_m >= pb_time.ymd2date(ymd_s[:6] + '01'), date_tbbias_m <= date_e)) # 根据拟合系数进行绘制 reference_list = plt_cfg[each]['reference'][i] for ref_temp in reference_list: if isLaunch: pic_path = os.path.join( OMB_DIR, pair, '%s_TBBias_%s_%s_Launch_%dK.png' % ( pair, chan, DayOrNight, ref_temp)) else: # plot latest year pic_path = os.path.join( OMB_DIR, pair, ymd_e, '%s_TBBias_%s_%s_Year_%s_%dK.png' % ( pair, chan, DayOrNight, ymd_e, ref_temp)) plot_tbbias( date_tbbias_d[idx_d], data_tbbias_d[idx_d], date_tbbias_m[idx_m], data_tbbias_m[idx_m], std_tbbias_m[idx_m], pic_path, date_s, date_e, sat1, pair, chan, DayOrNight, ref_temp, xname, xname_l, xunit, yname, yname_l, yunit, ) # plot interpolated TBBias img (obs minus backgroud) ------------- print "plot OMB : {} {} {}".format(each, DayOrNight, chan) title = 'Brightness Temperature Correction\n%s %s %s' % \ (pair, chan, DayOrNight) if isLaunch: picPath = os.path.join(OMB_DIR, pair, '%s_TBBOMB_%s_%s_Launch.png' % ( pair, chan, DayOrNight)) else: picPath = os.path.join(OMB_DIR, pair, ymd_e, '%s_TBBOMB_%s_%s_Year_%s.png' % ( pair, chan, DayOrNight, ymd_e)) plot_omb(date_D[idx_D], a_D[idx_D], b_D[idx_D], picPath, title, date_s, date_e)
def main(sat_sensor, in_file): """ 对L3数据进行合成 :param sat_sensor: 卫星+传感器 :param in_file: yaml 文件 :return: """ # ######################## 初始化 ########################### # 获取程序所在位置,拼接配置文件 app = InitApp(sat_sensor) if app.error: print "Load config file error." return gc = app.global_config sc = app.sat_config yc = Config(in_file) log = LogServer(gc.path_out_log) # 加载全局配置信息 sat_sensor1 = sat_sensor.split('_')[1] sat_sensor2 = sat_sensor.split('_')[0] sat1, sensor1 = sat_sensor1.split('+') sat2, sensor2 = sat_sensor2.split('+') # 加载卫星配置信息 s_channel1 = sc.name s_channel2 = sc.name timseries_channels_config = sc.timeseries_l2_channels # 加载业务配置信息 # ######################## 开始处理 ########################### print "-" * 100 print "Start plot verify result." if not os.path.isfile(in_file): log.error("File is not exist: {}".format(in_file)) return print "<<< {}".format(in_file) all_files = yc.path_ipath # 加载数据 data_absolute = dict() data_relative = dict() date = dict() ref_s1_all = dict() ref_s2_all = dict() amount_all = dict() date_start = ymd2date(yc.info_ymd_s) date_end = ymd2date(yc.info_ymd_e) point_count_min = 2 result = dict() while date_start <= date_end: ymd_now = date_start.strftime('%Y%m%d') in_files = get_one_day_files(all_files=all_files, ymd=ymd_now, ext='.h5', pattern_ymd=r'.*_(\d{8})') cross_data = ReadCrossDataL2() cross_data.read_cross_data(in_files=in_files) # 循环通道数据 for channel in cross_data.data: if channel not in data_absolute: data_absolute[channel] = list() if channel not in data_relative: data_relative[channel] = list() if channel not in date: date[channel] = list() if channel not in ref_s1_all: ref_s1_all[channel] = list() if channel not in ref_s2_all: ref_s2_all[channel] = list() if channel not in amount_all: amount_all[channel] = list() if channel not in result: result[channel] = dict() if not isinstance(cross_data.data[channel], dict): continue ref_s2 = cross_data.data[channel]['MERSI_FovMean'] fine_count = len(ref_s2) print '---INFO--- {} Points: {}'.format(channel, fine_count) if fine_count < point_count_min: print '***WARNING***Dont have enough point to plot: < {}'.format(point_count_min) continue ref_s1 = cross_data.data[channel]['MODIS_FovMean'] # 过滤 3 倍std之外的点 mean_ref_s2 = np.nanmean(ref_s2) std_ref_s2 = np.nanstd(ref_s2) min_ref_s2 = mean_ref_s2 - 3 * std_ref_s2 max_ref_s2 = mean_ref_s2 + 3 * std_ref_s2 idx = np.logical_and(ref_s2 >= min_ref_s2, ref_s2 <= max_ref_s2) ref_s2 = ref_s2[idx] ref_s1 = ref_s1[idx] # 计算相对偏差和绝对偏差 bias = Bias() absolute_bias = bias.absolute_deviation(ref_s1, ref_s2) relative_bias = bias.relative_deviation(ref_s1, ref_s2) data_absolute[channel].append(np.mean(absolute_bias)) data_relative[channel].append(np.mean(relative_bias)) date[channel].append(date_start) mean_absolute = np.nanmean(absolute_bias) std_absolute = np.nanstd(absolute_bias) amount_absolute = len(absolute_bias) median_absolute = np.nanmedian(absolute_bias) rms_absolute = rms(absolute_bias) mean_relative = np.nanmean(relative_bias) std_relative = np.nanstd(relative_bias) amount_relative = len(relative_bias) median_relative = np.nanmedian(relative_bias) rms_relative = rms(relative_bias) mean_ref_s1 = np.nanmean(ref_s1) std_ref_s1 = np.nanstd(ref_s1) amount_ref_s1 = len(ref_s1) median_ref_s1 = np.nanmedian(ref_s1) rms_ref_s1 = rms(ref_s1) ref_s1_all[channel].append(mean_ref_s1) mean_ref_s2 = np.nanmean(ref_s2) std_ref_s2 = np.nanstd(ref_s2) amount_ref_s2 = len(ref_s2) median_ref_s2 = np.nanmedian(ref_s2) rms_ref_s2 = rms(ref_s2) ref_s2_all[channel].append(mean_ref_s2) amount_all[channel].append(amount_ref_s1) fix_point = sc.plot_scatter_fix_ref f025_absolute, f025_relative = get_dif_pdif(ref_s1, ref_s2, fix_point) result_names = ['Dif_mean', 'Dif_std', 'Dif_median', 'Dif_count', 'Dif_rms', 'Dif_025', 'PDif_mean', 'PDif_std', 'PDif_median', 'PDif_count', 'PDif_rms', 'PDif_025', '{}_MODIS_mean'.format(channel), '{}_MODIS_std'.format(channel), '{}_MODIS_median'.format(channel), '{}_MODIS_count'.format(channel), '{}_MODIS_rms'.format(channel), '{}_MERSI_mean'.format(channel), '{}_MERSI_std'.format(channel), '{}_MERSI_median'.format(channel), '{}_MERSI_count'.format(channel), '{}_MERSI_rms'.format(channel), 'Date'] datas = [mean_absolute, std_absolute, median_absolute, amount_absolute, rms_absolute, f025_absolute, mean_relative, std_relative, median_relative, amount_relative, rms_relative, f025_relative, mean_ref_s1, std_ref_s1, median_ref_s1, amount_ref_s1, rms_ref_s1, mean_ref_s2, std_ref_s2, median_ref_s2, amount_ref_s2, rms_ref_s2, ymd_now] for result_name, data in zip(result_names, datas): if result_name not in result[channel]: result[channel][result_name] = list() else: result[channel][result_name].append(data) date_start = date_start + relativedelta(days=1) for channel in data_absolute: plot_config = timseries_channels_config[channel] dif_y_range = plot_config.get('dif_y_range') pdif_y_range = plot_config.get('pdif_y_range') ref_s1_y_range = plot_config.get('ref_s1_y_range') ref_s2_y_range = plot_config.get('ref_s2_y_range') count_y_range = plot_config.get('count_y_range') absolute_bias = data_absolute[channel] if len(absolute_bias) == 0: print 'Dont have enough point to plot, is 0: {}'.format(channel) continue relative_bias = data_relative[channel] date_channel = date[channel] ref_s1_channel = ref_s1_all[channel] ref_s2_channel = ref_s2_all[channel] amount_channel = amount_all[channel] # 绘制时间序列图 channel1 = channel index_channel1 = s_channel1.index(channel1) channel2 = s_channel2[index_channel1] title_series = '{}_{} {}_{} Time Series'.format(sat_sensor1, channel1, sat_sensor2, channel2) title_ref_s1 = '{}_{} {} Time Series'.format(channel1, sat_sensor1, channel1) title_ref_s2 = '{}_{} {} Time Series'.format(channel2, sat_sensor2, channel2) title_amount = '{}_{} {}_{} Matched Points Count Time Series'.format( sat_sensor1, channel1, sat_sensor2, channel2) y_label_series_absolute = 'Dif {}-{}'.format(sensor1, sensor2) y_label_series_relative = 'PDif ({}/{})-1'.format(sensor1, sensor2) y_label_ref_s1 = '{}'.format(channel1) y_label_ref_s2 = '{}'.format(channel2) y_label_amount = 'Count' picture_path = yc.path_opath # 孙凌添加,出两张图,限制Y轴坐标的图和不限制Y轴坐标的图,这里是限制Y轴坐标的图 picture_name_absolute = 'Time_Series_Dif_{}_{}_{}_{}.png'.format( sat_sensor1, channel1, sat_sensor2, channel2) picture_name_relative = 'Time_Series_PDif_{}_{}_{}_{}.png'.format( sat_sensor1, channel1, sat_sensor2, channel2) picture_name_ref_s1 = 'Time_Series_{}_{}.png'.format(sat_sensor1, channel1) picture_name_ref_s2 = 'Time_Series_{}_{}.png'.format(sat_sensor2, channel2) picture_name_amount = 'Time_Series_Count_{}_{}_{}_{}.png'.format( sat_sensor1, channel1, sat_sensor2, channel2) picture_file_absolute = os.path.join(picture_path, picture_name_absolute) picture_file_relative = os.path.join(picture_path, picture_name_relative) picture_file_ref_s1 = os.path.join(picture_path, picture_name_ref_s1) picture_file_ref_s2 = os.path.join(picture_path, picture_name_ref_s2) picture_file_amount = os.path.join(picture_path, picture_name_amount) plot_time_series(day_data_x=date_channel, day_data_y=absolute_bias, y_range=dif_y_range, out_file=picture_file_absolute, title=title_series, y_label=y_label_series_absolute, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, ) plot_time_series(day_data_x=date_channel, day_data_y=relative_bias, y_range=pdif_y_range, out_file=picture_file_relative, title=title_series, y_label=y_label_series_relative, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, ) plot_time_series(day_data_x=date_channel, day_data_y=ref_s1_channel, y_range=ref_s1_y_range, out_file=picture_file_ref_s1, title=title_ref_s1, y_label=y_label_ref_s1, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, zero_line=False) plot_time_series(day_data_x=date_channel, day_data_y=ref_s2_channel, y_range=ref_s2_y_range, out_file=picture_file_ref_s2, title=title_ref_s2, y_label=y_label_ref_s2, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, zero_line=False) plot_time_series(day_data_x=date_channel, day_data_y=amount_channel, y_range=count_y_range, out_file=picture_file_amount, title=title_amount, y_label=y_label_amount, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, plot_background=False) # 孙凌添加,出两张图,限制Y轴坐标的图和不限制Y轴坐标的图,这里是不限制Y轴坐标的图 picture_name_absolute = 'Time_Series_Dif_{}_{}_{}_{}_NL.png'.format( sat_sensor1, channel1, sat_sensor2, channel2) picture_name_relative = 'Time_Series_PDif_{}_{}_{}_{}_NL.png'.format( sat_sensor1, channel1, sat_sensor2, channel2) picture_name_ref_s1 = 'Time_Series_{}_{}_NL.png'.format(sat_sensor1, channel1) picture_name_ref_s2 = 'Time_Series_{}_{}_NL.png'.format(sat_sensor2, channel2) picture_name_amount = 'Time_Series_Count_{}_{}_{}_{}_NL.png'.format( sat_sensor1, channel1, sat_sensor2, channel2) picture_file_absolute = os.path.join(picture_path, picture_name_absolute) picture_file_relative = os.path.join(picture_path, picture_name_relative) picture_file_ref_s1 = os.path.join(picture_path, picture_name_ref_s1) picture_file_ref_s2 = os.path.join(picture_path, picture_name_ref_s2) picture_file_amount = os.path.join(picture_path, picture_name_amount) plot_time_series(day_data_x=date_channel, day_data_y=absolute_bias, out_file=picture_file_absolute, title=title_series, y_label=y_label_series_absolute, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, ) plot_time_series(day_data_x=date_channel, day_data_y=relative_bias, out_file=picture_file_relative, title=title_series, y_label=y_label_series_relative, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, ) plot_time_series(day_data_x=date_channel, day_data_y=ref_s1_channel, out_file=picture_file_ref_s1, title=title_ref_s1, y_label=y_label_ref_s1, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, ) plot_time_series(day_data_x=date_channel, day_data_y=ref_s2_channel, out_file=picture_file_ref_s2, title=title_ref_s2, y_label=y_label_ref_s2, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, ) plot_time_series(day_data_x=date_channel, day_data_y=amount_channel, out_file=picture_file_amount, title=title_amount, y_label=y_label_amount, ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, plot_background=False) # 输出HDF5 hdf5_name = '{}_{}_Dif_PDif_Daily.HDF'.format(sat_sensor1, sat_sensor2) out_path = yc.path_opath out_file_hdf5 = os.path.join(out_path, hdf5_name) make_sure_path_exists(out_path) write_hdf5(out_file_hdf5, result) keys = amount_all.keys() keys.sort() for channel in keys: print 'CHANNEL: {} POINT: {}'.format(channel, np.sum(amount_all[channel])) print '-' * 100 # 输出月的HDF5 # 加载数据 data_absolute = dict() data_relative = dict() date = dict() ref_s1_all = dict() ref_s2_all = dict() amount_all = dict() date_start = ymd2date(yc.info_ymd_s) date_end = ymd2date(yc.info_ymd_e) result = dict() while date_start <= date_end: ymd_now = date_start.strftime('%Y%m') in_files = get_one_day_files(all_files=all_files, ymd=ymd_now, ext='.h5', pattern_ymd=r'.*_(\d{6})') cross_data = ReadCrossDataL2() cross_data.read_cross_data(in_files=in_files) # 循环通道数据 for channel in cross_data.data: if channel not in data_absolute: data_absolute[channel] = list() if channel not in data_relative: data_relative[channel] = list() if channel not in date: date[channel] = list() if channel not in ref_s1_all: ref_s1_all[channel] = list() if channel not in ref_s2_all: ref_s2_all[channel] = list() if channel not in amount_all: amount_all[channel] = list() if channel not in result: result[channel] = dict() ref_s1 = cross_data.data[channel]['MERSI_FovMean'] if len(ref_s1) == 0: print '{} {} : Dont have enough point, is 0'.format(ymd_now, channel) continue ref_s2 = cross_data.data[channel]['MODIS_FovMean'] # 过滤 3 倍std之外的点 mean_ref_s1 = np.nanmean(ref_s1) std_ref_s1 = np.nanstd(ref_s1) min_ref_s1 = mean_ref_s1 - 3 * std_ref_s1 max_ref_s1 = mean_ref_s1 + 3 * std_ref_s1 idx = np.logical_and(ref_s1 >= min_ref_s1, ref_s1 <= max_ref_s1) ref_s1 = ref_s1[idx] ref_s2 = ref_s2[idx] # 计算相对偏差和绝对偏差 bias = Bias() absolute_bias = bias.absolute_deviation(ref_s1, ref_s2) relative_bias = bias.relative_deviation(ref_s1, ref_s2) data_absolute[channel].append(np.mean(absolute_bias)) data_relative[channel].append(np.mean(relative_bias)) date[channel].append(date_start) mean_absolute = np.nanmean(absolute_bias) std_absolute = np.nanstd(absolute_bias) amount_absolute = len(absolute_bias) median_absolute = np.nanmedian(absolute_bias) rms_absolute = rms(absolute_bias) mean_relative = np.nanmean(relative_bias) std_relative = np.nanstd(relative_bias) amount_relative = len(relative_bias) median_relative = np.nanmedian(relative_bias) rms_relative = rms(relative_bias) mean_ref_s1 = np.nanmean(ref_s1) std_ref_s1 = np.nanstd(ref_s1) amount_ref_s1 = len(ref_s1) median_ref_s1 = np.nanmedian(ref_s1) rms_ref_s1 = rms(ref_s1) ref_s1_all[channel].append(mean_ref_s1) mean_ref_s2 = np.nanmean(ref_s2) std_ref_s2 = np.nanstd(ref_s2) amount_ref_s2 = len(ref_s2) median_ref_s2 = np.nanmedian(ref_s2) rms_ref_s2 = rms(ref_s2) ref_s2_all[channel].append(mean_ref_s2) amount_all[channel].append(amount_ref_s1) fix_point = sc.plot_scatter_fix_ref f025_absolute, f025_relative = get_dif_pdif(ref_s1, ref_s2, fix_point) result_names = ['Dif_mean', 'Dif_std', 'Dif_median', 'Dif_count', 'Dif_rms', 'Dif_025', 'PDif_mean', 'PDif_std', 'PDif_median', 'PDif_count', 'PDif_rms', 'PDif_025', '{}_MODIS_mean'.format(channel), '{}_MODIS_std'.format(channel), '{}_MODIS_median'.format(channel), '{}_MODIS_count'.format(channel), '{}_MODIS_rms'.format(channel), '{}_MERSI_mean'.format(channel), '{}_MERSI_std'.format(channel), '{}_MERSI_median'.format(channel), '{}_MERSI_count'.format(channel), '{}_MERSI_rms'.format(channel), 'Date'] datas = [mean_absolute, std_absolute, median_absolute, amount_absolute, rms_absolute, f025_absolute, mean_relative, std_relative, median_relative, amount_relative, rms_relative, f025_relative, mean_ref_s1, std_ref_s1, median_ref_s1, amount_ref_s1, rms_ref_s1, mean_ref_s2, std_ref_s2, median_ref_s2, amount_ref_s2, rms_ref_s2, ymd_now] for result_name, data in zip(result_names, datas): if result_name not in result[channel]: result[channel][result_name] = list() else: result[channel][result_name].append(data) date_start = date_start + relativedelta(months=1) # 输出HDF5 hdf5_name = '{}_{}_Dif_PDif_Monthly.HDF'.format(sat_sensor1, sat_sensor2) out_path = yc.path_opath out_file_hdf5 = os.path.join(out_path, hdf5_name) make_sure_path_exists(out_path) write_hdf5(out_file_hdf5, result) print '-' * 100
def main(yamlfile): """ 对L3产品做长时间序列图 """ YamlFile = yamlfile if not os.path.isfile(YamlFile): print 'Not Found %s' % YamlFile sys.exit(-1) with open(YamlFile, 'r') as stream: cfg = yaml.load(stream) s_time = cfg["INFO"]['stime'] e_time = cfg["INFO"]['etime'] pair = cfg["INFO"]['pair'] file_list = cfg["PATH"]['ipath'] out_path = cfg["PATH"]['opath'] print "-" * 100 # date_start = ymd2date(s_time) # date_end = ymd2date(e_time) h5 = LoadH5() data_dict = {} datetime_list = [] for in_file in file_list: file_name = os.path.basename(in_file) if 'L3' in pair: list1 = ['aqua_chl1', 'aqua_kd490', 'aqua_poc', 'aqua_zsd'] list2 = ['fy_chl1', 'fy_kd490', 'fy_poc', 'fy_zsd'] reg = '.*_(\d{8}).*.HDF' m = re.match(reg, file_name) ymd = m.group(1) datetime_list.append(ymd2date(ymd)) elif 'L2' in pair: list1 = [ 'aqua_chl1', 'aqua_kd490', 'aqua_poc', 'aqua_rw412', 'aqua_rw443', 'aqua_a490', 'aqua_rw490' ] list2 = [ 'fy_chl1', 'fy_kd490', 'fy_poc', 'aqua_rw412', 'fy_rw443', 'aqua_a490', 'aqua_rw490' ] reg = '.*_(\d{14}).*.HDF' m = re.match(reg, file_name) ymd = m.group(1) ymdhms = datetime.strptime('%s' % (ymd), "%Y%m%d%H%M%S") datetime_list.append(ymdhms) h5.load(in_file, ymd) # 在配置中 dic_range = { 'aqua_chl1': [-0.05, 4.00], 'fy_chl1': [-0.05, 4.00], 'aqua_kd490': [-0.001, 1.00], 'fy_kd490': [-0.001, 1.00], 'aqua_poc': [-0.05, 50.00], 'fy_poc': [-0.05, 50.00], 'aqua_zsd': [-0.05, 50.00], 'fy_zsd': [-0.05, 50.00], 'aqua_a490': [-0.05, 50.00], 'fy_a490': [-0.05, 50.00], 'aqua_rw412': [-0.05, 50.00], 'fy_rw412': [-0.05, 50.00], 'aqua_rw443': [-0.05, 50.00], 'fy_rw443': [-0.05, 50.00], 'aqua_rw490': [-0.05, 50.00], 'fy_rw490': [-0.05, 50.00], 'dif_y_range': [-0.3, 0.3] } # 绘制每个数据集长时间 for key in h5.data.keys(): if key in ['Longitude', 'ymd', 'Latitude']: continue file_name = '%s_%s_%s_%s_time_series.png' % (pair, key, s_time, e_time) out_file_png = os.path.join(out_path, file_name) y_data = h5.data[key] plot_time_series(day_data_x=datetime_list, day_data_y=y_data, y_range=dic_range[key], out_file=out_file_png, title='time_series %s %s ' % (pair, key), y_label=key, ymd_start=s_time, ymd_end=e_time) # 绘制差值长时间序列图 for key1, key2 in zip(list1, list2): print 'bias---' file_name = '%s_diff_%s_%s_%s_time_series.png' % ( pair, key1.split('_')[1], s_time, e_time) out_file_png = os.path.join(out_path, file_name) y_data = h5.data[key] diff_data = h5.data[key1] - h5.data[key2] plot_time_series(day_data_x=datetime_list, day_data_y=diff_data, y_range=dic_range['dif_y_range'], out_file=out_file_png, title='time_series %s %s %s' % (pair, key1, key2), y_label='%s-%s' % (key1, key2), ymd_start=s_time, ymd_end=e_time)
def plot_bias(date_D, bias_D, date_M, bias_M, picPath, title, date_s, date_e, each, date_type, ylim_min, ylim_max): """ 画偏差时序折线图 """ plt.style.use(os.path.join(dvPath, 'dv_pub_timeseries.mplstyle')) fig = plt.figure(figsize=(6, 4)) # plt.subplots_adjust(left=0.13, right=0.95, bottom=0.11, top=0.97) plt.plot(date_D, bias_D, 'x', ms=6, markerfacecolor=None, markeredgecolor=BLUE, alpha=0.8, mew=0.3, label='Daily') plt.plot(date_M, bias_M, 'o-', ms=5, lw=0.6, c=RED, mew=0, label='Monthly') plt.grid(True) plt.ylabel('%s %s' % (each, date_type), fontsize=11, fontproperties=FONT0) xlim_min = pb_time.ymd2date('%04d%02d01' % (date_s.year, date_s.month)) xlim_max = date_e plt.plot([xlim_min, xlim_max], [0, 0], '#808080') # 在 y=0 绘制一条深灰色直线 plt.xlim(xlim_min, xlim_max) plt.ylim(ylim_min, ylim_max) ax = plt.gca() # format the ticks interval = (ylim_max - ylim_min) / 8 # 8 个间隔 minibar = interval / 2. setXLocator(ax, xlim_min, xlim_max) set_tick_font(ax) # 如果范围为浮点数,需要进行一次格式化,否则图像不会显示最后一个刻度 if isinstance(interval, float): interval = float('%.5f' % interval) minibar = float('%.5f' % minibar) ax.yaxis.set_major_locator(MultipleLocator(interval)) ax.yaxis.set_minor_locator(MultipleLocator(minibar)) # title plt.title(title, fontsize=12, fontproperties=FONT0) plt.tight_layout() # -------------------- fig.subplots_adjust(bottom=0.2) circle1 = mpatches.Circle((74, 15), 6, color=BLUE, ec=EDGE_GRAY, lw=0) circle2 = mpatches.Circle((164, 15), 6, color=RED, ec=EDGE_GRAY, lw=0) fig.patches.extend([circle1, circle2]) fig.text(0.15, 0.02, 'Daily', color=BLUE, fontproperties=FONT0) fig.text(0.3, 0.02, 'Monthly', color=RED, fontproperties=FONT0) ymd_s, ymd_e = date_s.strftime('%Y%m%d'), date_e.strftime('%Y%m%d') if ymd_s != ymd_e: fig.text(0.50, 0.02, '%s-%s' % (ymd_s, ymd_e), fontproperties=FONT0) else: fig.text(0.50, 0.02, '%s' % ymd_s, fontproperties=FONT0) fig.text(0.8, 0.02, ORG_NAME, fontproperties=FONT0) # --------------- pb_io.make_sure_path_exists(os.path.dirname(picPath)) plt.savefig(picPath) fig.clear() plt.close()