def run(pair, ymd): """ pair: sat1+sensor1_sat2+sensor2 ymd: str YYYYMMDD """ # 提取参数中的卫星信息和传感器信息 part1, part2 = pair.split("_") sat1, sensor1 = part1.split("+") sat2, sensor2 = part2.split("+") # 判断是静止卫星还是动态卫星 if "FY2" in part1 or "FY4" in part1: Type = "GEOLEO" elif "FY3" in part1: Type = "LEOLEO" else: LOG.error("Cant distinguish the satellite type") return # 加载绘图配置文件 plt_cfg_file = os.path.join(MAIN_PATH, "cfg", "%s.plt" % pair) plt_cfg = loadYamlCfg(plt_cfg_file) if plt_cfg is None: LOG.error("Not find the config file: {}".format(plt_cfg_file)) return PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数 ym = ymd[:6] ymd = ym + '%02d' % PERIOD # 当月最后一天 LOG.info(u"----- Start Drawing Monthly TBBias Analysis Pic, PAIR: {}, YMD: {} -----".format(pair, ymd)) for each in plt_cfg['monthly_staistics']: # Day_Night must be in 'all', 'day', 'night' Day_Night = ['all', 'day', 'night'] # default if 'time' in plt_cfg[each].keys(): Day_Night = plt_cfg[each]['time'] for i in Day_Night: if i not in ['all', 'day', 'night']: Day_Night.remove(i) for idx, chan in enumerate(plt_cfg[each]['chan']): LOG.info(u"Start Drawing {} Channel {}".format(each, chan)) oneHDF5 = ReadHDF5() # load Matched HDF5 num_file = PERIOD for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) nc_name = 'COLLOC+%sIR,%s_C_BABJ_%s.hdf5' % (Type, pair, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, nc_name) if not os.path.isfile(filefullpath): LOG.info(u"HDF5 not found: {}".format(filefullpath)) num_file -= 1 continue if not oneHDF5.LoadData(filefullpath, chan): LOG.error('Error occur when reading %s of %s' % (chan, filefullpath)) if num_file == 0: LOG.error(u"No file found.") continue elif num_file != PERIOD: LOG.info(u"{} of {} file(s) found.".format(num_file, PERIOD)) # 输出目录 cur_path = os.path.join(MBA_DIR, pair, ymd[:6]) # get threhold, unit, names... xname, yname = each.split('-') bias = xname xname_l = xname.upper() xunit = plt_cfg[each]['x_unit'] xlimit = plt_cfg[each]['x_range'][idx] xmin, xmax = xlimit.split('-') xmin = float(xmin) xmax = float(xmax) # delete 0 in std if len(oneHDF5.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值,而导致权重极大,所以 std>0 改成 std>0.0001 deletezeros = np.where(oneHDF5.rad1_std > 0.0001) oneHDF5.rad1_std = oneHDF5.rad1_std[deletezeros] oneHDF5.rad1 = oneHDF5.rad1[deletezeros] if len( oneHDF5.rad1) > 0 else oneHDF5.rad1 oneHDF5.rad2 = oneHDF5.rad2[deletezeros] if len( oneHDF5.rad2) > 0 else oneHDF5.rad2 oneHDF5.tbb1 = oneHDF5.tbb1[deletezeros] if len( oneHDF5.tbb1) > 0 else oneHDF5.tbb1 oneHDF5.tbb2 = oneHDF5.tbb2[deletezeros] if len( oneHDF5.tbb2) > 0 else oneHDF5.tbb2 oneHDF5.time = oneHDF5.time[deletezeros] if len( oneHDF5.time) > 0 else oneHDF5.time oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len( oneHDF5.lon1) > 0 else oneHDF5.lon1 oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len( oneHDF5.lon2) > 0 else oneHDF5.lon2 if len(oneHDF5.ref1_std) > 0.0001: deletezeros = np.where(oneHDF5.ref1_std > 0.0001) oneHDF5.ref1_std = oneHDF5.ref1_std[deletezeros] oneHDF5.ref1 = oneHDF5.ref1[deletezeros] if len( oneHDF5.ref1) > 0 else oneHDF5.ref1 oneHDF5.ref2 = oneHDF5.ref2[deletezeros] if len( oneHDF5.ref2) > 0 else oneHDF5.ref2 oneHDF5.dn1 = oneHDF5.dn1[deletezeros] if len( oneHDF5.dn1) > 0 else oneHDF5.dn1 oneHDF5.dn2 = oneHDF5.dn1[deletezeros] if len( oneHDF5.dn2) > 0 else oneHDF5.dn2 oneHDF5.time = oneHDF5.time[deletezeros] if len( oneHDF5.time) > 0 else oneHDF5.time oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len( oneHDF5.lon1) > 0 else oneHDF5.lon1 oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len( oneHDF5.lon2) > 0 else oneHDF5.lon2 # find out day and night if ('day' in Day_Night or 'night' in Day_Night) and len(oneHDF5.time) > 0: vect_is_day = np.vectorize(is_day_timestamp_and_lon) day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1) night_index = np.logical_not(day_index) else: day_index = None night_index = None # get x dset_name = xname + "1" if hasattr(oneHDF5, dset_name): x = getattr(oneHDF5, dset_name) else: LOG.error("Can't plot, no %s in HDF5 class" % dset_name) continue # get y dset_name = yname + "2" if hasattr(oneHDF5, dset_name): y = getattr(oneHDF5, dset_name) else: LOG.error("Can't plot, no %s in HDF5 class" % dset_name) continue if 'rad' == bias: o_name = 'RadBiasMonthStats' elif 'tbb' == bias: o_name = 'TBBiasMonthStats' elif 'ref' == bias: o_name = 'RefBiasMonthStats' else: o_name = 'DUMMY' if x.size < 10: LOG.error("Not enough match point to draw.") continue # 获取 std weight = None if 'rad' in xname and 'rad' in yname: if len(oneHDF5.rad1_std) > 0: weight = oneHDF5.rad1_std elif 'tbb' in xname and 'tbb' in yname: weight = None elif 'ref' in xname and 'ref' in yname: if len(oneHDF5.ref1_std) > 0: weight = oneHDF5.ref1_std elif 'dn' in xname and 'ref' in yname: weight = None # rad-specified regression starts reference_list = [] if 'reference' in plt_cfg[each]: reference_list = plt_cfg[each]['reference'][idx] if 'all' in Day_Night: o_file = os.path.join(cur_path, '%s_%s_%s_ALL_%s' % (pair, o_name, chan, ym)) print("x_all, y_all", len(x), len(y)) plot(x, y, weight, o_file, part1, part2, chan, ym, 'ALL', reference_list, xname, xname_l, xunit, xmin, xmax) # ------- day ---------- if 'day' in Day_Night: if day_index is not None and np.where(day_index)[0].size > 10: # rad-specified o_file = os.path.join(cur_path, '%s_%s_%s_Day_%s' % (pair, o_name, chan, ym)) x_d = x[day_index] y_d = y[day_index] w_d = weight[day_index] if weight is not None else None print("x_all, y_all", len(x), len(y)) print("x_day, y_day", len(x_d), len(y_d)) plot(x_d, y_d, w_d, o_file, part1, part2, chan, ym, 'Day', reference_list, xname, xname_l, xunit, xmin, xmax) if 'night' in Day_Night: # ---------night ------------ if night_index is not None and np.where(night_index)[0].size > 10: # rad-specified o_file = os.path.join(cur_path, '%s_%s_%s_Night_%s' % (pair, o_name, chan, ym)) x_n = x[night_index] y_n = y[night_index] w_n = weight[day_index] if weight is not None else None print("x_all, y_all", len(x), len(y)) print("x_night, y_night", len(x_n), len(y_n)) plot(x_n, y_n, w_n, o_file, part1, part2, chan, ym, 'Night', reference_list, xname, xname_l, xunit, xmin, xmax)
def run(pair, ymd, is_monthly): """ pair: sat1+sensor1_sat2+sensor2 ymd: str YYYYMMDD """ # 提取参数中的卫星信息和传感器信息 part1, part2 = pair.split("_") sat1, sensor1 = part1.split("+") sat2, sensor2 = part2.split("+") # 判断是静止卫星还是动态卫星 if "FY2" in part1 or "FY4" in part1: Type = "GEOLEO" elif "FY3" in part1: Type = "LEOLEO" else: Log.error("Cant distinguish the satellite type") return # 加载绘图配置文件 plt_cfg_file = os.path.join(MainPath, "%s_%s_3d.yaml" % (sensor1, sensor2)) plt_cfg = loadYamlCfg(plt_cfg_file) if plt_cfg is None: Log.error("Not find the config file: {}".format(plt_cfg_file)) return Log.info(u"----- Start Drawing Regression-Pic, PAIR: {}, YMD: {} -----".format(pair, ymd)) for each in plt_cfg["regression"]: dict_cabr = {} dict_cabr_d = {} dict_cabr_n = {} dict_bias = {} dict_bias_d = {} dict_bias_n = {} # 需要回滚的天数 if is_monthly: PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数 ymd = ymd[:6] + "%02d" % PERIOD # 当月最后一天 else: PERIOD = plt_cfg[each]["days"] # must be in "all", "day", "night" Day_Night = ["all", "day", "night"] if "time" in plt_cfg[each].keys(): Day_Night = plt_cfg[each]["time"] for t in Day_Night: if t not in ["all", "day", "night"]: Day_Night.remove(t) for idx, chan in enumerate(plt_cfg[each]["chan"]): Log.info(u"Start Drawing {} Channel {}".format(each, chan)) oneHDF5 = ReadHDF5() num_file = PERIOD for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) hdf5_name = "COLLOC+%sIR,%s_C_BABJ_%s.hdf5" % (Type, pair, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, hdf5_name) if not os.path.isfile(filefullpath): Log.info(u"File not found: {}".format(filefullpath)) num_file -= 1 continue if not oneHDF5.LoadData(filefullpath, chan): Log.error("Error occur when reading %s of %s" % (chan, filefullpath)) if num_file == 0: Log.error(u"No file found.") continue elif num_file != PERIOD: Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD)) if is_monthly: str_time = ymd[:6] cur_path = os.path.join(MRA_DIR, pair, str_time) else: str_time = ymd cur_path = os.path.join(DRA_DIR, pair, str_time) # delete 0 in std if len(oneHDF5.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值,而导致权重极大,所以 std>0 改成 std>0.0001 deletezeros = np.where(oneHDF5.rad1_std > 0.0001) oneHDF5.rad1_std = oneHDF5.rad1_std[deletezeros] oneHDF5.rad1 = oneHDF5.rad1[deletezeros] if len( oneHDF5.rad1) > 0 else oneHDF5.rad1 oneHDF5.rad2 = oneHDF5.rad2[deletezeros] if len( oneHDF5.rad2) > 0 else oneHDF5.rad2 oneHDF5.tbb1 = oneHDF5.tbb1[deletezeros] if len( oneHDF5.tbb1) > 0 else oneHDF5.tbb1 oneHDF5.tbb2 = oneHDF5.tbb2[deletezeros] if len( oneHDF5.tbb2) > 0 else oneHDF5.tbb2 oneHDF5.time = oneHDF5.time[deletezeros] if len( oneHDF5.time) > 0 else oneHDF5.time oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len( oneHDF5.lon1) > 0 else oneHDF5.lon1 oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len( oneHDF5.lon2) > 0 else oneHDF5.lon2 if len(oneHDF5.ref1_std) > 0.0001: deletezeros = np.where(oneHDF5.ref1_std > 0.0001) oneHDF5.ref1_std = oneHDF5.ref1_std[deletezeros] oneHDF5.ref1 = oneHDF5.ref1[deletezeros] if len( oneHDF5.ref1) > 0 else oneHDF5.ref1 oneHDF5.ref2 = oneHDF5.ref2[deletezeros] if len( oneHDF5.ref2) > 0 else oneHDF5.ref2 oneHDF5.dn1 = oneHDF5.dn1[deletezeros] if len( oneHDF5.dn1) > 0 else oneHDF5.dn1 oneHDF5.dn2 = oneHDF5.dn1[deletezeros] if len( oneHDF5.dn2) > 0 else oneHDF5.dn2 oneHDF5.time = oneHDF5.time[deletezeros] if len( oneHDF5.time) > 0 else oneHDF5.time oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len( oneHDF5.lon1) > 0 else oneHDF5.lon1 oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len( oneHDF5.lon2) > 0 else oneHDF5.lon2 # find out day and night if ("day" in Day_Night or "night" in Day_Night) and len(oneHDF5.time) > 0: vect_is_day = np.vectorize(is_day_timestamp_and_lon) day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1) night_index = np.logical_not(day_index) else: day_index = None night_index = None # 将每个对通用的属性值放到对循环,每个通道用到的属性值放到通道循环 # get threhold, unit, names... xname, yname = each.split("-") xname_l = plt_cfg[each]["x_name"] xunit = plt_cfg[each]["x_unit"] xlimit = plt_cfg[each]["x_range"][idx] xmin, xmax = xlimit.split("-") xmin = float(xmin) xmax = float(xmax) yname_l = plt_cfg[each]["y_name"] yunit = plt_cfg[each]["y_unit"] ylimit = plt_cfg[each]["y_range"][idx] ymin, ymax = ylimit.split("-") ymin = float(ymin) ymax = float(ymax) weight = None if "rad" in xname: x = oneHDF5.rad1 elif "tbb" in xname: x = oneHDF5.tbb1 elif "ref" in xname: x = oneHDF5.ref1 elif "dn" in xname: x = oneHDF5.dn1 else: Log.error("Can't plot %s" % each) continue if "rad" in yname: y = oneHDF5.rad2 elif "tbb" in yname: y = oneHDF5.tbb2 elif "ref" in yname: y = oneHDF5.ref2 else: Log.error("Can't plot %s" % each) continue if "rad" in xname and "rad" in yname: if len(oneHDF5.rad1_std) > 0: weight = oneHDF5.rad1_std o_name = "RadCalCoeff" elif "tbb" in xname and "tbb" in yname: o_name = "TBBCalCoeff" elif "ref" in xname and "ref" in yname: if len(oneHDF5.ref1_std) > 0: weight = oneHDF5.ref1_std o_name = "CorrcCoeff" elif "dn" in xname and "ref" in yname: o_name = "CalCoeff" # 画对角线 if xname == yname: diagonal = True else: diagonal = False if "all" in Day_Night and o_name not in dict_cabr: dict_cabr[o_name] = {} dict_bias[xname] = {} if "day" in Day_Night and o_name not in dict_cabr_d: dict_cabr_d[o_name] = {} dict_bias_d[xname] = {} if "night" in Day_Night and o_name not in dict_cabr_n: dict_cabr_n[o_name] = {} dict_bias_n[xname] = {} # 对样本点数量进行判断,如果样本点少于 100 个,则不进行绘制 if x.size < 100: Log.error("Not enough match point to draw: {}, {}".format(each, chan)) if "all" in Day_Night: dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias[xname][chan] = [np.NaN, np.NaN] if "day" in Day_Night: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_d[xname][chan] = [np.NaN, np.NaN] if "night" in Day_Night: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_n[xname][chan] = [np.NaN, np.NaN] continue # regression starts if "all" in Day_Night: o_file = os.path.join(cur_path, "%s_%s_%s_ALL_%s" % ( pair, o_name, chan, str_time)) print("x_all, y_all", len(x), len(y)) abr, bias = plot(x, y, weight, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal, is_monthly) if abr: dict_cabr[o_name][chan] = abr else: dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if bias: dict_bias[xname][chan] = bias else: dict_bias[xname][chan] = [np.NaN, np.NaN] # ------- day ---------- if "day" in Day_Night: if day_index is not None and np.where(day_index)[0].size > 10: o_file = os.path.join(cur_path, "%s_%s_%s_Day_%s" % ( pair, o_name, chan, str_time)) x_d = x[day_index] y_d = y[day_index] w_d = weight[day_index] if weight is not None else None print("x_all, y_all", len(x), len(y)) print("x_day, y_day", len(x_d), len(y_d)) abr, bias = plot(x_d, y_d, w_d, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal, is_monthly) if abr: dict_cabr_d[o_name][chan] = abr else: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if bias: dict_bias_d[xname][chan] = bias else: dict_bias_d[xname][chan] = [np.NaN, np.NaN] else: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_d[xname][chan] = [np.NaN, np.NaN] # ---------night ------------ if "night" in Day_Night: if night_index is not None and np.where(night_index)[0].size > 10: o_file = os.path.join(cur_path, "%s_%s_%s_Night_%s" % ( pair, o_name, chan, str_time)) x_n = x[night_index] y_n = y[night_index] w_n = weight[night_index] if weight is not None else None print("x_all, y_all", len(x), len(y)) print("x_night, y_night", len(x_n), len(y_n)) abr, bias = plot(x_n, y_n, w_n, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal, is_monthly) if abr: dict_cabr_n[o_name][chan] = abr else: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if bias: dict_bias_n[xname][chan] = bias else: dict_bias_n[xname][chan] = [np.NaN, np.NaN] else: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_n[xname][chan] = [np.NaN, np.NaN] oneHDF5.clear() # write txt lock.acquire() channel = plt_cfg[each]["chan"] if "all" in Day_Night: for o_name in dict_cabr: write_bias(channel, part1, part2, xname, ymd, dict_bias, "ALL") write_cabr(channel, part1, part2, o_name, ymd, dict_cabr, "ALL") if "day" in Day_Night: for o_name in dict_cabr_d: write_bias(channel, part1, part2, xname, ymd, dict_bias_d, "Day") write_cabr(channel, part1, part2, o_name, ymd, dict_cabr_d, "Day") if "night" in Day_Night: for o_name in dict_cabr_n: write_bias(channel, part1, part2, xname, ymd, dict_bias_n, "Night") write_cabr(channel, part1, part2, o_name, ymd, dict_cabr_n, "Night") lock.release()
def proj_dcc(self): # 初始化投影参数 rowMax=None, colMax=None lookup_table = prj_gll(resLat=self.resLat, resLon=self.resLon) newLats, newLons = lookup_table.generateLatsLons() proj_data_num = np.full_like(newLats, 0) row = proj_data_num.shape[0] col = proj_data_num.shape[1] if self.is_monthly: # 如果是月的,需要查找当前自然月所有数据 PERIOD = calendar.monthrange(int(self.ymd[:4]), int(self.ymd[4:6]))[1] # 当前月份天数 _ymd = self.ymd[:6] + '%02d' % PERIOD # 当月最后一天 else: PERIOD = 1 _ymd = self.ymd file_list = [] for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(_ymd, -daydelta) # 判断文件是否存在 file_name = '%s_DCC_SLT_%s.H5' % (self.sat_sensor, cur_ymd) ym = self.ymd[:6] HDF_file = os.path.join(self.ifile, ym, file_name) if not os.path.isfile(HDF_file): print("file no exist : %s" % HDF_file) else: file_list.append(HDF_file) if len(file_list) == 0: return # 开始进行投影 for HDF_file in file_list: h5File = h5py.File(HDF_file, 'r') lons = h5File.get('Longitude')[:] lats = h5File.get('Latitude')[:] h5File.close() lons = lons / 100. lats = lats / 100. ii, jj = lookup_table.lonslats2ij(lons, lats) for i in xrange(row): for j in xrange(col): condition = np.logical_and(ii[:] == i, jj[:] == j) idx = np.where(condition) proj_data_num[i][j] = proj_data_num[i][j] + len(idx[0]) proj_data_num = np.ma.masked_where(proj_data_num == 0, proj_data_num) if self.is_monthly: print("plot: %s" % self.ymd[0:6]) p = dv_map.dv_map() p.easyplot(newLats, newLons, proj_data_num, vmin=0, vmax=10000, ptype=None, markersize=20, marker='s') title_name = '%s_dcc_projection_' % self.sat_sensor + str( self.ymd[0:6]) p.title = u'dcc: ' + str(title_name) + u' (分辨率1度)' opath_fig = os.path.join(self.ofile, 'Monthly', '%s' % self.ymd[:6]) if not os.path.exists(opath_fig): os.makedirs(opath_fig) fig_name = os.path.join( opath_fig, '%s_%s_dcc_monthly.png' % (self.sat_sensor, self.ymd[:6])) p.savefig(fig_name) opath_hdf = os.path.join(self.ofile, 'Monthly', '%s' % self.ymd[:6]) if not os.path.exists(opath_hdf): os.mkdir(opath_hdf) opath_hdf = os.path.join( opath_hdf, '%s_%s_dcc_monthly.hdf' % (self.sat_sensor, self.ymd[0:6])) h5file_W = h5py.File(opath_hdf, 'w') h5file_W.create_dataset('proj_data_nums', dtype='i2', data=proj_data_num, compression='gzip', compression_opts=5, shuffle=True) h5file_W.close() ########################################### # read hdf file and cont values print("write txt: %s" % self.ymd[0:6]) if not os.path.exists(self.ofile_txt): os.mkdir(self.ofile_txt) opath_txt = os.path.join( self.ofile_txt, '%s_dcc_monthly_count.txt' % self.sat_sensor) if self.ifile and len(self.ymd) == 8: hdf = h5py.File(opath_hdf, 'r') data_mat = hdf.get("proj_data_nums")[:] hdf.close() count_value = np.sum(data_mat) self.data_count = '%8s\t%8s\n' % (self.ymd[0:6], count_value) lock.acquire() self.write_txt(opath_txt) lock.release() else: print("plot: %s" % self.ymd) p = dv_map.dv_map() p.easyplot(newLats, newLons, proj_data_num, vmin=0, vmax=10000, ptype=None, markersize=20, marker='s') title_name = '%s_dcc_projection_' % self.sat_sensor + str(self.ymd) p.title = u'dcc: ' + str(title_name) + u' (分辨率1度)' opath_fig = os.path.join(self.ofile, 'Daily', '%s' % self.ymd) if not os.path.exists(opath_fig): os.makedirs(opath_fig) fig_name = os.path.join( opath_fig, '%s_%s_dcc_daily.png' % (self.sat_sensor, self.ymd)) p.savefig(fig_name) opath_hdf = os.path.join(self.ofile, 'Daily', '%s' % self.ymd) if not os.path.exists(opath_hdf): os.mkdir(opath_hdf) opath_hdf = os.path.join( opath_hdf, '%s_%s_dcc_daily.hdf' % (self.sat_sensor, self.ymd)) h5file_W = h5py.File(opath_hdf, 'w') h5file_W.create_dataset('proj_data_nums', dtype='i2', data=proj_data_num, compression='gzip', compression_opts=5, shuffle=True) h5file_W.close() ########################################### # read hdf file and cont values print("write txt: %s" % self.ymd) if not os.path.exists(self.ofile_txt): os.mkdir(self.ofile_txt) opath_txt = os.path.join( self.ofile_txt, '%s_dcc_daily_count.txt' % self.sat_sensor) if self.ifile and len(self.ymd) == 8: hdf = h5py.File(opath_hdf, 'r') data_mat = hdf.get("proj_data_nums")[:] count_value = np.sum(data_mat) self.data_count = '%8s\t%8s\n' % (self.ymd, count_value) lock.acquire() self.write_txt(opath_txt) lock.release()
def run(pair, ymd): ''' pair: sat1+sensor1_sat2+sensor2 ymd: str YYYYMMDD ''' part1, part2 = pair.split('_') sat1, sensor1 = part1.split('+') sat2, sensor2 = part2.split('+') if 'FY2' in part1 or 'FY4' in part1: ReadMatchNC = ReadMatchNC_FY2 Type = "GEOLEO" elif 'FY3' in part1: ReadMatchNC = ReadMatchNC_FY3 Type = "LEOLEO" else: return # load yaml plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2)) plt_cfg = pb_io.loadYamlCfg(plt_cfg_file) if plt_cfg is None: return # Day_Night must be in 'all', 'day', 'night' Day_Night = ['all', 'day', 'night'] # default if "day_night" in plt_cfg.keys(): Day_Night = plt_cfg['day_night'] for each in Day_Night: if each not in ['all', 'day', 'night']: Day_Night.remove(each) Bias = [] if "bias" in plt_cfg.keys(): Bias = plt_cfg["bias"] PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数 ym = ymd[:6] ymd = ym + '%02d' % PERIOD # 当月最后一天 Log.info(u"----- Start Drawing Monthly TBBias Analysis Pic, PAIR: {}, YMD: {}" \ u" -----".format(pair, ymd)) for idx, chan in enumerate(plt_cfg['chan']): Log.info(u"Start Drawing Channel {}".format(chan)) oneNC = ReadMatchNC() # load Matched NC num_file = PERIOD for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) nc_name = 'COLLOC+%sIR,%s_C_BABJ_%s.NC' % (Type, pair, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, nc_name) if not os.path.isfile(filefullpath): Log.info(u"NC not found: {}".format(filefullpath)) num_file -= 1 continue if oneNC.LoadData(filefullpath, chan) == False: Log.error('Error occur when reading %s of %s' % (chan, filefullpath)) if num_file == 0: Log.error(u"No file found.") continue elif num_file != PERIOD: Log.info(u"{} of {} file(s) found.".format(num_file, PERIOD)) # 输出目录 cur_path = os.path.join(MBA_DIR, pair, ymd[:6]) # find out day and night if ('day' in Day_Night or 'night' in Day_Night) and len(oneNC.time) > 0: jd = oneNC.time / 24. / 3600. # jday from 1993/01/01 00:00:00 hour = ((jd - jd.astype('int8')) * 24).astype('int8') day_index = (hour < 10) # utc hour<10 is day night_index = np.logical_not(day_index) else: day_index = None night_index = None for each in Bias: # get threhold, unit, names... xname, yname = each, each # xname_l = plt_cfg[xname]['name'] xname_l = xname.upper() xunit = plt_cfg[xname]['unit'] xlimit = plt_cfg[xname]['thhold'][idx] xmin, xmax = xlimit.split('-') xmin = int(xmin) xmax = int(xmax) # get x dset_name = xname + "1" if hasattr(oneNC, dset_name): x = getattr(oneNC, dset_name) else: Log.error("Can't plot, no %s in NC class" % dset_name) continue # get y dset_name = yname + "2" if hasattr(oneNC, dset_name): y = getattr(oneNC, dset_name) else: Log.error("Can't plot, no %s in NC class" % dset_name) continue if 'rad' == each: o_name = 'RadBiasMonthStats' elif 'tbb' == each: o_name = 'TBBiasMonthStats' elif 'ref' == each: o_name = 'RefBiasMonthStats' else: o_name = 'DUMMY' if x.size < 10: Log.error("Not enough match point to draw.") continue # rad-specified regression starts reference_list = [] if "reference" in plt_cfg[each]: reference_list = plt_cfg[each]['reference'][idx] if 'all' in Day_Night: o_file = os.path.join( cur_path, '%s_%s_%s_ALL_%s' % (pair, o_name, chan, ym)) plot(x, y, o_file, part1, part2, chan, ym, 'ALL', reference_list, xname, xname_l, xunit, xmin, xmax) # ------- day ---------- if 'day' in Day_Night: if day_index is not None and np.where(day_index)[0].size > 10: # rad-specified o_file = os.path.join( cur_path, '%s_%s_%s_Day_%s' % (pair, o_name, chan, ym)) x_d = x[day_index] y_d = y[day_index] plot(x_d, y_d, o_file, part1, part2, chan, ym, 'Day', reference_list, xname, xname_l, xunit, xmin, xmax) if 'night' in Day_Night: # ---------night ------------ if night_index is not None and np.where( night_index)[0].size > 10: # rad-specified o_file = os.path.join( cur_path, '%s_%s_%s_Night_%s' % (pair, o_name, chan, ym)) x_n = x[night_index] y_n = y[night_index] plot(x_n, y_n, o_file, part1, part2, chan, ym, 'Night', reference_list, xname, xname_l, xunit, xmin, xmax)
def run(pair, ymd, isMonthly): ''' pair: sat1+sensor1_sat2+sensor2 ymd: str YYYYMMDD ''' part1, part2 = pair.split('_') sat1, sensor1 = part1.split('+') sat2, sensor2 = part2.split('+') if 'FY2' in part1 or 'FY4' in part1: ReadMatchNC = ReadMatchNC_FY2 Type = "GEOLEO" elif 'FY3' in part1: ReadMatchNC = ReadMatchNC_FY3 Type = "LEOLEO" else: return # load yaml config file plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2)) plt_cfg = pb_io.loadYamlCfg(plt_cfg_file) if plt_cfg is None: return if isMonthly: PERIOD = calendar.monthrange(int(ymd[:4]), int(ymd[4:6]))[1] # 当月天数 ymd = ymd[:6] + '%02d' % PERIOD # 当月最后一天 else: PERIOD = plt_cfg['days'] # must be in 'all', 'day', 'night' Day_Night = ['all', 'day', 'night'] if "day_night" in plt_cfg.keys(): Day_Night = plt_cfg['day_night'] for each in Day_Night: if each not in ['all', 'day', 'night']: Day_Night.remove(each) Log.info(u"----- Start Drawing Regression-Pic, PAIR: {}, YMD: {}, PERIOD: " \ u"{} -----".format(pair, ymd, PERIOD)) dict_cabr = {} dict_cabr_d = {} dict_cabr_n = {} for idx, chan in enumerate(plt_cfg['chan']): Log.info(u"Start Drawing Channel {}".format(chan)) oneNC = ReadMatchNC() num_file = PERIOD for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) nc_name = 'COLLOC+%sIR,%s_C_BABJ_%s.NC' % (Type, pair, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, nc_name) if not os.path.isfile(filefullpath): Log.info(u"File not found: {}".format(filefullpath)) num_file -= 1 continue if oneNC.LoadData(filefullpath, chan) == False: Log.error('Error occur when reading %s of %s' % (chan, filefullpath)) if num_file == 0: Log.error(u"No file found.") continue elif num_file != PERIOD: Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD)) if isMonthly: str_time = ymd[:6] cur_path = os.path.join(MRA_DIR, pair, str_time) else: str_time = ymd cur_path = os.path.join(DRA_DIR, pair, str_time) # delete 0 in std if len(oneNC.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值,而导致权重极大,所以 std>0 改成 std>0.0001 deletezeros = np.where(oneNC.rad1_std > 0.0001) oneNC.rad1_std = oneNC.rad1_std[deletezeros] oneNC.rad1 = oneNC.rad1[deletezeros] if len(oneNC.rad1) > 0 else oneNC.rad1 oneNC.rad2 = oneNC.rad2[deletezeros] if len(oneNC.rad2) > 0 else oneNC.rad2 oneNC.tbb1 = oneNC.tbb1[deletezeros] if len(oneNC.tbb1) > 0 else oneNC.tbb1 oneNC.tbb2 = oneNC.tbb2[deletezeros] if len(oneNC.tbb2) > 0 else oneNC.tbb2 oneNC.time = oneNC.time[deletezeros] if len(oneNC.time) > 0 else oneNC.time if len(oneNC.ref1_std) > 0.0001: deletezeros = np.where(oneNC.ref1_std > 0.0001) oneNC.ref1_std = oneNC.ref1_std[deletezeros] oneNC.ref1 = oneNC.ref1[deletezeros] if len(oneNC.ref1) > 0 else oneNC.ref1 oneNC.ref2 = oneNC.ref2[deletezeros] if len(oneNC.ref2) > 0 else oneNC.ref2 # find out day and night if ('day' in Day_Night or 'night' in Day_Night) and len(oneNC.time) > 0: jd = oneNC.time / 24. / 3600. # jday from 1993/01/01 00:00:00 hour = ((jd - jd.astype('int8')) * 24).astype('int8') day_index = (hour < 10) # utc hour<10 is day night_index = np.logical_not(day_index) else: day_index = None night_index = None for each in plt_cfg['plot']: # get threhold, unit, names... xname, yname = each.split('-') xname_l = plt_cfg[xname]['name'] xunit = plt_cfg[xname]['unit'] xlimit = plt_cfg[xname]['thhold'][idx] xmin, xmax = xlimit.split('-') xmin = float(xmin) xmax = float(xmax) yname_l = plt_cfg[yname]['name'] yunit = plt_cfg[yname]['unit'] ylimit = plt_cfg[yname]['thhold'][idx] ymin, ymax = ylimit.split('-') ymin = float(ymin) ymax = float(ymax) weight = None if 'rad' in xname: x = oneNC.rad1 elif 'tbb' in xname: x = oneNC.tbb1 elif 'ref' in xname: x = oneNC.ref1 elif 'dn' in xname: x = oneNC.dn1 else: Log.error("Can't plot %s" % each) continue if 'rad' in yname: y = oneNC.rad2 elif 'tbb' in yname: y = oneNC.tbb2 elif 'ref' in yname: y = oneNC.ref2 else: Log.error("Can't plot %s" % each) continue if 'rad' in xname and 'rad' in yname: if len(oneNC.rad1_std) > 0: weight = oneNC.rad1_std o_name = 'RadCalCoeff' elif 'tbb' in xname and 'tbb' in yname: o_name = 'TBBCalCoeff' elif 'ref' in xname and 'ref' in yname: if len(oneNC.ref1_std) > 0: weight = oneNC.ref1_std o_name = 'CorrcCoeff' elif 'dn' in xname and 'ref' in yname: o_name = 'CalCoeff' # 画对角线 if xname == yname: diagonal = True else: diagonal = False if 'all' in Day_Night and o_name not in dict_cabr: dict_cabr[o_name] = {} if 'day' in Day_Night and o_name not in dict_cabr_d: dict_cabr_d[o_name] = {} if 'night' in Day_Night and o_name not in dict_cabr_n: dict_cabr_n[o_name] = {} if x.size < 10: Log.error("Not enough match point to draw.") if 'all' in Day_Night: dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if 'day' in Day_Night: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if 'night' in Day_Night: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] continue # regression starts if 'all' in Day_Night: o_file = os.path.join(cur_path, '%s_%s_%s_ALL_%s' % (pair, o_name, chan, str_time)) abr = plot(x, y, weight, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal) if abr: dict_cabr[o_name][chan] = abr else: dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] # ------- day ---------- if 'day' in Day_Night: if day_index is not None and np.where(day_index)[0].size > 10 : # o_file = os.path.join(cur_path, '%s_%s_%s_Day_%s' % (pair, o_name, chan, str_time)) x_d = x[day_index] y_d = y[day_index] w_d = weight[day_index] if weight is not None else None abr = plot(x_d, y_d, w_d, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal) if abr: dict_cabr_d[o_name][chan] = abr else: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] else: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] # ---------night ------------ if 'night' in Day_Night: if night_index is not None and np.where(night_index)[0].size > 10 : # o_file = os.path.join(cur_path, '%s_%s_%s_Night_%s' % (pair, o_name, chan, str_time)) x_n = x[night_index] y_n = y[night_index] w_n = weight[night_index] if weight is not None else None abr = plot(x_n, y_n, w_n, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal) if abr: dict_cabr_n[o_name][chan] = abr else: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] else: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] oneNC.clear() # write txt lock.acquire() if 'all' in Day_Night: for o_name in dict_cabr: writeTxt(plt_cfg, part1, part2, o_name, str_time, dict_cabr, 'ALL', isMonthly) if 'day' in Day_Night: for o_name in dict_cabr_d: writeTxt(plt_cfg, part1, part2, o_name, str_time, dict_cabr_d, 'Day', isMonthly) if 'night' in Day_Night: for o_name in dict_cabr_n: writeTxt(plt_cfg, part1, part2, o_name, str_time, dict_cabr_n, 'Night', isMonthly) lock.release()
def run(pair, ymd): ''' pair: sat1+sensor1_sat2+sensor2 ymd: YYYYMMDD ''' part1, part2 = pair.split('_') sat1, sensor1 = part1.split('+') sat2, sensor2 = part2.split('+') # TODO: for now only support FY2 FY4 if 'FY2' in part1 or 'FY4' in part1: pass else: return # load yaml config file plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2)) plt_cfg = pb_io.loadYamlCfg(plt_cfg_file) if plt_cfg is None: return PERIOD = plt_cfg['days'] Log.info(u"----- Start Drawing Matched Map-Pic, PAIR: {}, YMD: {}, PERIOD: " \ u"{} -----".format(pair, ymd, PERIOD)) fy2_orbit = Sat_Orbit(inCfg['SAT_S2L'][sat1], ymd, ORBIT_DIR) if fy2_orbit.error: return fy2_orbit.get_orbit(ymd) if fy2_orbit.error: return fy2_lon = fy2_orbit.orbit['Lon'][0] fy2_lat = fy2_orbit.orbit['Lat'][0] if len(plt_cfg['chan']) > 3: chan = plt_cfg['chan'][2] # FY2 IR3 has the most matchpoint else: chan = plt_cfg['chan'][0] NC = ReadMatchNC() num_file = PERIOD for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) filename = 'COLLOC+GEOLEOIR,%s_C_BABJ_%s.NC' % (pair, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, filename) if not os.path.isfile(filefullpath): Log.info(u"File not found: {}".format(filefullpath)) num_file -= 1 continue if NC.LoadData(filefullpath, chan, PERIOD - daydelta) == False: Log.error('Error occur when reading %s of %s' % (chan, filefullpath)) if num_file == 0: Log.error(u"No file found.") return elif num_file != PERIOD: Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD)) cur_path = os.path.join(DMS_DIR, pair, ymd) # FY2F+VISSR-MetopA+IASI_MatchedPoints_ALL_20150226.png # find out day and night jd = NC.time / 24. / 3600. # jday from 1993/01/01 00:00:00 hour = ((jd - jd.astype('int8')) * 24).astype('int8') day_index = (hour < 10) # utc hour<10 is day night_index = np.logical_not(day_index) x = NC.lon y = NC.lat d = NC.days.astype('uint8') # trans to int if len(d) == 0: Log.error('No days info in NC.') return # ------- day ---------- if np.where(day_index)[0].size > 0 : o_file = os.path.join(cur_path, '%s-%s_MatchedPoints_Day_%s' % (part1, part2, ymd)) x_d = x[day_index] y_d = y[day_index] d_d = d[day_index] draw_butterfly(part1, part2, cur_ymd, ymd, fy2_lon, fy2_lat, x_d, y_d, d_d, o_file) # ---------night ------------ if np.where(night_index)[0].size > 0 : o_file = os.path.join(cur_path, '%s-%s_MatchedPoints_Night_%s' % (part1, part2, ymd)) x_n = x[night_index] y_n = y[night_index] d_n = d[night_index] draw_butterfly(part1, part2, cur_ymd, ymd, fy2_lon, fy2_lat, x_n, y_n, d_n, o_file) # ------- ALL ---------- if np.where(day_index)[0].size != 0 and np.where(night_index)[0].size != 0: o_file = os.path.join(cur_path, '%s_%s_MatchedPoints_ALL_%s' % (part1, part2, ymd)) draw_butterfly(part1, part2, cur_ymd, ymd, fy2_lon, fy2_lat, x, y, d, o_file) Log.info(u"Success")
def run(pair, ymd): ''' pair: sat1+sensor1_sat2+sensor2 ymd: YYYYMMDD ''' part1, part2 = pair.split('_') sat1, sensor1 = part1.split('+') sat2, sensor2 = part2.split('+') # # TODO: for now only support FY3 if 'FY3' in part1: Type = "LEOLEO" else: return # load yaml config file plt_cfg_file = os.path.join(MainPath, '%s_%s.yaml' % (sensor1, sensor2)) plt_cfg = pb_io.loadYamlCfg(plt_cfg_file) if plt_cfg is None: return # PERIOD = plt_cfg['days'] PERIOD = 1 Log.info(u"----- Start Drawing Matched Map-Pic, PAIR: {}, YMD: {}, PERIOD: " \ u"{} -----".format(pair, ymd, PERIOD)) chan = plt_cfg['chan'][0] NC = ReadMatchNC() num_file = PERIOD for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) filename = "COLLOC+%sIR,%s+%s_%s+%s_C_BABJ_%s.NC" % ( Type, sat1, sensor1, sat2, sensor2, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, filename) if not os.path.isfile(filefullpath): Log.info(u"File not found: {}".format(filefullpath)) num_file -= 1 continue if NC.LoadData(filefullpath, chan) == False: Log.error('Error occur when reading %s of %s' % (chan, filefullpath)) if num_file == 0: Log.error(u"No file found.") return elif num_file != PERIOD: Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD)) cur_path = os.path.join(DMS_DIR, pair, ymd) o_file = os.path.join(cur_path, '%s_%s_MatchedPoints_ALL_%s' % (part1, part2, ymd)) # FY2F+VISSR-MetopA+IASI_MatchedPoints_ALL_20150226.png # find out day and night jd = NC.time / 24. / 3600. # jday from 1993/01/01 00:00:00 hour = ((jd - jd.astype('int8')) * 24).astype('int8') day_index = (hour < 10) # utc hour<10 is day night_index = np.logical_not(day_index) x = NC.lon y = NC.lat # d = NC.days.astype('uint8') # trans to int # if len(d) == 0: # Log.error('No days info in NC.') # return draw_butterfly(part1, part2, cur_ymd, ymd, x, y, o_file) # ------- day ---------- if np.where(day_index)[0].size > 0: o_file = os.path.join( cur_path, '%s-%s_MatchedPoints_Day_%s' % (part1, part2, ymd)) x_d = x[day_index] y_d = y[day_index] # d_d = d[day_index] draw_butterfly(part1, part2, cur_ymd, ymd, x_d, y_d, o_file) # ---------night ------------ if np.where(night_index)[0].size > 0: o_file = os.path.join( cur_path, '%s-%s_MatchedPoints_Night_%s' % (part1, part2, ymd)) x_n = x[night_index] y_n = y[night_index] # d_n = d[night_index] draw_butterfly(part1, part2, cur_ymd, ymd, x_n, y_n, o_file) Log.info(u"Success")
def run(pair, ymd): """ pair: sat1+sensor1_sat2+sensor2 ymd: YYYYMMDD """ # 提取参数中的卫星信息和传感器信息 part1, part2 = pair.split("_") sat1, sensor1 = part1.split("+") sat2, sensor2 = part2.split("+") # 判断是静止卫星还是动态卫星 if "FY2" in part1 or "FY4" in part1: Type = "GEOLEO" elif "FY3" in part1: Type = "LEOLEO" else: Log.error("Cant distinguish the satellite type") return # 加载绘图配置文件 plt_cfg_file = os.path.join(MainPath, "%s_%s_3d.yaml" % (sensor1, sensor2)) plt_cfg = loadYamlCfg(plt_cfg_file) if plt_cfg is None: Log.error("Not find the config file: {}".format(plt_cfg_file)) return # 读取配置文件的信息 PERIOD = plt_cfg['collocation_map']['days'] # 回滚天数 chans = plt_cfg['collocation_map']['chan'] # 通道 maptype = plt_cfg['collocation_map']['maptype'] # 需要绘制的类型 if 'area' in maptype: # 区域块视图 area = plt_cfg['collocation_map']['area'] else: area = None if 'polar' in maptype: # 两极视图 polar = plt_cfg['collocation_map']['polar'] else: polar = None # 读取范围配置 if not area and not polar: return else: map_range = (polar, area) Log.info( u"----- Start Drawing Matched Map-Pic, PAIR: {}, YMD: {}, PERIOD: {} -----" .format(pair, ymd, PERIOD)) # 读取 HDF5 文件数据 oneHDF5 = ReadHDF5() num_file = PERIOD cur_ymd = pb_time.ymd_plus(ymd, 1) # 回滚天数,现在为 1 for daydelta in xrange(PERIOD): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) filename = "COLLOC+%sIR,%s+%s_%s+%s_C_BABJ_%s.hdf5" % ( Type, sat1, sensor1, sat2, sensor2, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, filename) if not os.path.isfile(filefullpath): Log.info(u"File not found: {}".format(filefullpath)) num_file -= 1 continue if not oneHDF5.LoadData(filefullpath, chans): Log.error('Error occur when reading %s of %s' % (chans, filefullpath)) if num_file == 0: Log.error(u"No file found.") return elif num_file != PERIOD: Log.error(u"{} of {} file(s) found.".format(num_file, PERIOD)) cur_path = os.path.join(DMS_DIR, pair, ymd) o_file = os.path.join(cur_path, '%s_%s_MatchedPoints_ALL_%s' % (part1, part2, ymd)) # find out day and night vect_is_day = np.vectorize(is_day_timestamp_and_lon) day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1) night_index = np.logical_not(day_index) x = oneHDF5.lon1 # 经度数据 y = oneHDF5.lat1 # 维度数据 print 'date: {}, x_all: {} y_all: {} '.format(ymd, len(x), len(y)) draw_butterfly(part1, part2, cur_ymd, ymd, x, y, o_file, map_range) # ------- day ---------- if np.where(day_index)[0].size > 0: o_file = os.path.join( cur_path, '%s-%s_MatchedPoints_Day_%s' % (part1, part2, ymd)) x_d = x[day_index] y_d = y[day_index] print 'date: {}, x_day: {} y_day: {} '.format(ymd, len(x_d), len(y_d)) draw_butterfly(part1, part2, cur_ymd, ymd, x_d, y_d, o_file, map_range) # ---------night ------------ if np.where(night_index)[0].size > 0: o_file = os.path.join( cur_path, '%s-%s_MatchedPoints_Night_%s' % (part1, part2, ymd)) x_n = x[night_index] y_n = y[night_index] print 'date: {}, x_night: {} y_night: {} '.format( ymd, len(x_n), len(y_n)) draw_butterfly(part1, part2, cur_ymd, ymd, x_n, y_n, o_file, map_range) Log.info(u"Success")
def runLEO_LEO_SNOX(s1, s2, sat_dist, timeGap, outPath, Log, day_counts=1): ''' 极轨对极轨 交叉点计算 ''' zangle = 180 # 天顶角阈值 80改为180 2018/6/15 drew_points = [] output_list = [] for day_num in range(0, day_counts): ymd = ymd_plus(s1.ymd, day_num) s1.get_orbit(ymd) if s1.error: continue s2.get_orbit(ymd) if s2.error: continue if len(s1.orbit) == 0 or len(s2.orbit) == 0: continue s1.divide_by_lat0() s2.divide_by_lat0() day_list = [] # 输出用 for i in xrange(len(s1.divide)): index1 = s1.divide[i] for j in xrange(len(s2.divide)): index2 = s2.divide[j] timeDiff = index2 - index1 if abs(timeDiff) > timeGap * 60: continue dist = distance_GreatCircle(s1.orbit['Lat'][index1], s1.orbit['Lon'][index1], s2.orbit['Lat'][index2], s2.orbit['Lon'][index2]) if dist > sat_dist: continue # TODO: 只保留太阳天顶角<80°的点(白天) yy, mm, dd = s1.orbit['date'][index1].split(".") hh = int(s1.orbit['time'][index1][0:2]) sz = solar_zen(yy, mm, dd, hh, s1.orbit['Lon'][index1], s1.orbit['Lat'][index1]) if sz > zangle: continue day_list.append([ s1.orbit['date'][index1].replace('.', ''), s1.orbit['time'][index1][0:8], s1.orbit['Lat'][index1], s1.orbit['Lon'][index1], s2.orbit['time'][index2][0:8], s2.orbit['Lat'][index2], s2.orbit['Lon'][index2], dist, timeDiff ]) # 画图所需信息 st = max(index1 - 240, 0) et = min(index1 + 80, len(s1.orbit['Lon'])) trails1 = [s1.orbit['Lon'][st:et], s1.orbit['Lat'][st:et]] st = max(index2 - 240, 0) et = min(index2 + 80, len(s2.orbit['Lon'])) trails2 = [s2.orbit['Lon'][st:et], s2.orbit['Lat'][st:et]] drew_points.append( (s1.orbit['Lon'][index1], s1.orbit['Lat'][index1], s2.orbit['Lon'][index2], s2.orbit['Lat'][index2], trails1, trails2)) output_list.extend(day_list) # ----------------------- if len(output_list) == 0: Log.info('No SNOX!') return wout_fig_file = outPath.replace('.txt', '_worldmap.png') ymd_s = ymd2y_m_d(s1.ymd) # 画交叉点图像 if len(drew_points) > 0: draw_world(s1.sat, s2.sat, ymd_s, ymd2y_m_d(ymd), drew_points, wout_fig_file) folder = os.path.dirname(outPath) if not os.path.isdir(folder): os.makedirs(folder) # 输出交叉点文件 fid_Result = open(outPath, 'w') # 写十行信息 fid_Result.write('Sat1: %s \n' % (s1.sat)) fid_Result.write('Sat2: %s \n' % (s2.sat)) if ymd_s == ymd: fid_Result.write('Time: %s \n' % ymd_s) else: fid_Result.write('Time: %s-%s \n' % (ymd_s, ymd)) fid_Result.write('Time Gap MAX = %d min\n' % timeGap) fid_Result.write('Dist MAX = %d (km)\n' % sat_dist) fid_Result.write('Solar Zenith MAX = %s (deg)\n' % str(zangle)) fid_Result.write('Calc time : %s \n' % get_local_time().strftime('%Y.%m.%d %H:%M:%S')) fid_Result.write('\n') title_line = 'YMD HMS(%s) Lat,Lon(%s) HMS(%s) Lat,Lon(%s) Distance(km) Time_Diff(sec)\n' % \ (s1.sat, s1.sat, s2.sat, s2.sat) fid_Result.write(title_line) fid_Result.write('-' * len(title_line) + '\n') for eachline in output_list: fid_Result.write( '%s %s %6.2f %-7.2f %s %6.2f %-7.2f %7.2f %4d\n' % tuple(eachline)) fid_Result.close() s1.clear() Log.info('Success')
def runLEO_LEO(s1, s2, sat_dist, timeGap_high, timeGap_low, outPath, Log, day_counts=1): ''' 极轨对极轨 交叉点计算 ''' drew_points = [] output_list = [] for day_num in range(0, day_counts): ymd = ymd_plus(s1.ymd, day_num) s1.get_orbit(ymd) if s1.error: continue s2.get_orbit(ymd) if s2.error: continue if len(s1.orbit) == 0 or len(s2.orbit) == 0: continue s1.divide_orbit() s2.divide_orbit() day_list = [] # 输出用 # day_points = [] # 画图用 for i in xrange(len(s1.divide)): if i == 0: continue ii_s = s1.divide[i - 1] ii_e = s1.divide[i] if abs(ii_e - ii_s) < 2: continue line1 = LineString( zip(s1.orbit['Lon'][ii_s:ii_e], s1.orbit['Lat'][ii_s:ii_e])) for j in xrange(len(s2.divide)): if j == 0: continue jj_s = s2.divide[j - 1] jj_e = s2.divide[j] if abs(jj_e - jj_s) < 2: continue if abs(ii_s + ii_e - jj_s - jj_e) > timeGap_low * 60 * 2 * 4: # 中间点4倍时间范围粗筛 continue line2 = LineString( zip(s2.orbit['Lon'][jj_s:jj_e], s2.orbit['Lat'][jj_s:jj_e])) inters = line1.intersection(line2) if inters.geom_type == 'Point': pass elif inters.geom_type == 'MultiPoint': inters = np.array(inters) if (np.abs(np.diff(inters[:, 0])) < 1.).all() and \ (np.abs(np.diff(inters[:, 1])) < 1.).all(): inters = [np.mean(inters[:, 0]), np.mean(inters[:, 1])] else: continue else: continue inters = np.array(inters) # 转成array if len(inters) > 0: index1 = ii_s + \ getIndex(s1.orbit['Lon'][ii_s:ii_e], inters[0]) index2 = jj_s + \ getIndex(s2.orbit['Lon'][jj_s:jj_e], inters[0]) timeDiff = index2 - index1 if abs(timeDiff) <= timeGap_low * 60: # 60度以上高纬 timeGap过滤 if abs(s1.orbit['Lat'][index1]) > 60. and \ abs(timeDiff) > timeGap_high * 60: continue dd = distance_GreatCircle(s1.orbit['Lat'][index1], s1.orbit['Lon'][index1], s2.orbit['Lat'][index2], s2.orbit['Lon'][index2]) if dd > sat_dist: continue day_list.append([ s1.orbit['date'][index1].replace('.', ''), s1.orbit['time'][index1][0:8], s1.orbit['Lat'][index1], s1.orbit['Lon'][index1], s2.orbit['time'][index2][0:8], s2.orbit['Lat'][index2], s2.orbit['Lon'][index2], dd, timeDiff ]) # 画图所需信息 st = max(index1 - 240, 0) et = min(index1 + 80, len(s1.orbit['Lon'])) trails1 = [ s1.orbit['Lon'][st:et], s1.orbit['Lat'][st:et] ] st = max(index2 - 240, 0) et = min(index2 + 80, len(s2.orbit['Lon'])) trails2 = [ s2.orbit['Lon'][st:et], s2.orbit['Lat'][st:et] ] drew_points.append( (s1.orbit['Lon'][index1], s1.orbit['Lat'][index1], s2.orbit['Lon'][index2], s2.orbit['Lat'][index2], trails1, trails2)) # ------- 交叉点多 隔3个 取一个 -------- output_list.extend(day_list[::3]) # drew_points.extend(day_points[::3]) # ----------------------- if len(output_list) == 0: # 20160708 黄叶建huangyj modify Log.info('No SNO!') return pout_fig_file = outPath.replace('.txt', '_polarmap.png') wout_fig_file = outPath.replace('.txt', '_worldmap.png') ymd_s = ymd2y_m_d(s1.ymd) # 画交叉点图像 if len(drew_points) > 0: draw_polar(s1.sat, s2.sat, ymd_s, ymd2y_m_d(ymd), drew_points, pout_fig_file) draw_world(s1.sat, s2.sat, ymd_s, ymd2y_m_d(ymd), drew_points, wout_fig_file) folder = os.path.dirname(outPath) if not os.path.isdir(folder): os.makedirs(folder) # 输出交叉点文件 fid_Result = open(outPath, 'w') fid_Result.write('Sat1: %s \n' % (s1.sat)) fid_Result.write('Sat2: %s \n' % (s2.sat)) if ymd_s == ymd: fid_Result.write('Time: %s \n' % ymd_s) else: fid_Result.write('Time: %s-%s \n' % (ymd_s, ymd)) fid_Result.write('When |lat|<=60: Time Gap MAX = %d min\n' % timeGap_low) fid_Result.write('When |lat|>60: Time Gap MAX = %d min\n' % timeGap_high) fid_Result.write('Dist MAX = %d (km)\n' % sat_dist) fid_Result.write('Calc time : %s \n' % get_local_time().strftime('%Y.%m.%d %H:%M:%S')) fid_Result.write('\n') title_line = 'YMD HMS(%s) Lat,Lon(%s) HMS(%s) Lat,Lon(%s) Distance(km) Time_Diff(sec)\n' % \ (s1.sat, s1.sat, s2.sat, s2.sat) fid_Result.write(title_line) fid_Result.write('-' * len(title_line) + '\n') for eachline in output_list: fid_Result.write( '%s %s %6.2f %-7.2f %s %6.2f %-7.2f %7.2f %4d\n' % tuple(eachline)) fid_Result.close() s1.clear() Log.info('Success')
def runSatPassingFixedPoint(s1, fix_list, fix_dist, outPath, FIX_DICT, Log, day_counts=1): ''' s1 : Sat_Orbit的实例 fix_list: 固定点组名list fix_dist: 距离阈值 outPath: 输出txt全路径 day_counts: 算几天 ''' out_fig = outPath.replace('.txt', '.png') fixDict = {} drew_points = [] output_list = [] # 去重 for fix_group in fix_list: # 循环不同组 for fix_point in FIX_DICT[fix_group].keys(): # 循环每个组中的键 if fix_point not in fixDict.keys(): # 如果键不存在字典中,则将该键和键值进行存入 fixDict[fix_point] = FIX_DICT[fix_group][fix_point] for fix_point in fixDict.keys(): fixed_lon, fixed_lat = [float(e) for e in fixDict[fix_point]] # 提取站点的经度和维度 # log.debug('%s and %s of %s' % (fixed_point[0], sat2, date)) for day_num in range(0, day_counts): ymd = ymd_plus(s1.ymd, day_num) # 将进行进行进行往后推 s1.get_orbit(ymd) if s1.error: continue if len(s1.orbit) == 0: continue l = len(s1.orbit) lat_fixed_array = np.array([fixed_lat] * l) lon_fixed_array = np.array([fixed_lon] * l) with warnings.catch_warnings(): warnings.simplefilter("ignore") # Suppressing RuntimeWarning deltaDist_list = distance_GreatCircle_np( lat_fixed_array, lon_fixed_array, s1.orbit['Lat'], s1.orbit['Lon']) y_growth_flips = np.where( np.diff(np.diff(deltaDist_list) > 0))[0] + 1 i_old = 0 cross_list = [] for i in y_growth_flips: if (deltaDist_list[i] < fix_dist and (i - i_old > 10) and # 去掉间隔秒数在10秒以内的点 (deltaDist_list[i] - deltaDist_list[i_old] < 0)): # 每行2个元素:index, deltaDist_list cross_list.append((i, deltaDist_list[i])) i_old = i # 排序 按deltaDist从小到大 cross_list.sort(cmp_byDeltaDist) if (len(cross_list) == 0): continue j = 0 for each in cross_list: cross_index = each[0] # TODO:过滤夜晚数据 ------------------- # 以太阳天顶角 大于85度作为夜晚的判断。 # 用这种方法,'Greenland' 'Dome_C'就不需要特别考虑极夜极昼了 ymd_sol = s1.orbit['date'][cross_index] yy = int(ymd_sol[:4]) mm = int(ymd_sol[5:7]) dd = int(ymd_sol[8:10]) hh = int(s1.orbit['time'][cross_index][0:2]) if solar_zen(yy, mm, dd, hh, s1.orbit['Lon'][cross_index], s1.orbit['Lat'][cross_index]) >= 85.: continue # ------------------------------ output_list.append([ s1.orbit['date'][cross_index].replace('.', ''), s1.orbit['time'][cross_index][0:8], fix_point, fixed_lat, fixed_lon, s1.orbit['Lat'][cross_index], s1.orbit['Lon'][cross_index], deltaDist_list[cross_index] ]) # 保留画图信息 st = max(cross_index - 240, 0) et = min(cross_index + 80, len(s1.orbit['Lon'])) drew_points.append( (fixed_lon, fixed_lat, s1.orbit['Lon'][cross_index], s1.orbit['Lat'][cross_index], [None, None], [s1.orbit['Lon'][st:et], s1.orbit['Lat'][st:et]])) j = j + 1 if fix_point in ['Greenland', 'Dome_C']: pass # 冰雪目标全部保留 elif j >= 1: # 其他目标只保留最近点 break if (len(output_list) == 0): Log.info('No SNO!') return # 排序 output_list.sort(cmp_col0_col1) ymd_s = ymd2y_m_d(s1.ymd) # 画图 if len(drew_points) > 0: draw_fixed(s1.sat, ymd_s, ymd2y_m_d(ymd), drew_points, out_fig) folder = os.path.dirname(outPath) if not os.path.isdir(folder): os.makedirs(folder) # 写文件 fid_Result = open(outPath, 'w') fid_Result.write('Sat: %s \n' % s1.sat) if ymd_s == ymd: fid_Result.write('Time: %s \n' % ymd_s) else: fid_Result.write('Time: %s-%s \n' % (ymd_s, ymd)) fid_Result.write('Dist MAX = %s (km) \n' % fix_dist) fid_Result.write(u'Solar Zenith MAX = 85° \n') fid_Result.write(u' 对于Greenland和Dome_C,保留所有符合阈值的每轨最近点\n') fid_Result.write(u' 对于其它固定点,只保留符合阈值的全天最近点\n') fid_Result.write('Calc time : %s \n' % get_local_time().strftime('%Y.%m.%d %H:%M:%S')) fid_Result.write('\n') title_line = 'YMD HMS(%s) %-24s %-16s %-19s %s \n' % \ (s1.sat, 'Fixed_Point', 'Lat,Lon(Fixed)', 'Lat,Lon(%s)' % s1.sat, 'Distance(km)') fid_Result.write(title_line) fid_Result.write('-' * len(title_line) + '\n') for eachline in output_list: fid_Result.write( '%s %s %-24s %6.2f %-7.2f %6.2f %-7.2f %7.2f\n' % tuple(eachline)) fid_Result.close() # clean s1.clear() Log.info('Success')