def main(sat_sensor, in_file): """ :param sat_sensor: (str) 卫星对 :param in_file: (str) 输入文件 :return: """ # ######################## 初始化 ########################### # 获取程序所在位置,拼接配置文件 app = InitApp(sat_sensor) if app.error: print "Load config file error." return gc = app.global_config sc = app.sat_config log = LogServer(gc.path_out_log) # 全局配置接口 out_path = gc.path_mid_ncep # 程序配置接口 # 卫星配置接口 suffix = sc.ncep2byte_filename_suffix ncep_table = sc.ncep2byte_ncep_table ######################### 开始处理 ########################### print "-" * 100 print "Start ncep to byte." if not os.path.isfile(in_file): log.error("File is not exist: {}".format(in_file)) return print "<<< {}".format(in_file) ymdhm = _get_ymdhm(in_file) out_file = _get_out_file(in_file, out_path, suffix) # 判断是否使用新命令进行处理 if int(ymdhm) >= 201501221200: new = True else: new = False if pb_io.is_none(in_file, out_file, ncep_table): log.error("Error: {}".format(in_file)) return ncep2byte = Ncep2Byte(in_file, out_file, new=new, ncep_table=ncep_table) ncep2byte.ncep2byte() if not ncep2byte.error: print ">>> {}".format(ncep2byte.out_file) else: log.error("Error: {}".format(in_file)) print "-" * 100
def combine(self): if self.error: return # 如果输出文件已经存在,跳过 elif os.path.isfile(self.ofile): self.error = True print "Error: File is already exist, skip it: {}".format( self.ofile) return # 合成日数据 elif pb_io.is_none(self.ifile, self.ofile): self.error = True print "Error: Is None: ifile or ofile: {}".format(self.yaml_file) return elif len(self.ifile) < 1: self.error = True print "Error: File count lower than 1: {}".format(self.yaml_file) return for in_file in self.ifile: if os.path.isfile(in_file): print "<<< {}".format(in_file) else: print "Warning: File is not exist: {}".format(in_file) continue self.one_in_file = in_file # 日合成 with time_block("One combine time:", switch=TIME_TEST): self._combine_2d() # 计算数据的平均值 with time_block("Calculate mean time:", switch=TIME_TEST): print "Start calculate." self._2d_data_calculate() # 输出数据集有效数据的数量 self._print_data_count()
def main(sat_sensor, in_file): """ 绘制 HDF5.dataset 的快视图。支持真彩图和灰度图 :param sat_sensor: 卫星+传感器 :param in_file: HDF5 文件 :return: """ # ######################## 初始化 ########################### # 获取程序所在位置,拼接配置文件 app = InitApp(sat_sensor) if app.error: print "Load config file error." return gc = app.global_config sc = app.sat_config log = LogServer(gc.path_out_log) log = LogServer(gc.path_out_log) # 加载全局配置信息 # 加载程序配置信息 # 加载卫星配置信息 dataset = sc.plt_quick_view_rgb_dataset rgb_suffix = sc.plt_quick_view_rgb_suffix colorbar_range = sc.plt_quick_view_img_colorbar_range log10_set = sc.plt_quick_view_img_log10_set log10_ticks = sc.plt_quick_view_img_log10_ticks log10_tick_labels = sc.plt_quick_view_img_log10_tick_labels # ######################## 开始处理 ########################### print '-' * 100 print "Start plot quick view picture." if not os.path.isfile(in_file): log.error("File not exist: {}".format(in_file)) return print "<<< {}".format(in_file) in_file_name = os.path.splitext(in_file)[0] # 绘制真彩图 out_picture = "{}_{}.{}".format(in_file_name, rgb_suffix, "png") # 如果文件已经存在,跳过 if not os.path.isfile(out_picture): r_set, g_set, b_set = dataset rgb = RGB(in_file, r_set, g_set, b_set, out_picture) rgb.plot() if not rgb.error: print ">>> {}".format(out_picture) print '-' * 100 else: print "Error: Plot RGB error: {}".format(in_file) else: print "File is already exist, skip it: {}".format(out_picture) # 绘制热度图 for legend in colorbar_range: dataset_name = legend[0] # 数据集名称 vmax = float(legend[1]) # color bar 范围 最大值 vmin = float(legend[2]) # color bar 范围 最小值 out_picture = "{}_{}.{}".format(in_file_name, dataset_name.replace("Aod", "AOD"), "png") # 如果文件已经存在,跳过 if os.path.isfile(out_picture): print "File is already exist, skip it: {}".format(out_picture) continue heat_map = { "vmin": vmin, "vmax": vmax, "cmap": "jet", "fill_value": -32767, } if dataset_name in log10_set: if dataset_name == "Ocean_TSM": log10_ticks.append(2.00) log10_tick_labels.append("100") if dataset_name == "Ocean_YS443": log10_ticks = log10_ticks[:-3] log10_tick_labels = log10_tick_labels[:-3] heat_map["colorbar_ticks"] = log10_ticks heat_map["colorbar_tick_label"] = log10_tick_labels lats, lons = get_lats_lons(in_file) lat_lon_text = _get_lat_lon_text(lats, lons) if is_none(lats, lons): lat_lon_line = None else: lat_lon_line = { "lats": lats, # 经度数据集名称 "lons": lons, # 维度数据集名称 "step": 5.0, # 线密度 "line_width": 0.01, "text": lat_lon_text, } quick_view = QuickView(in_file, dataset_name, out_picture, main_view=heat_map, lat_lon_line=lat_lon_line) quick_view.plot() if not quick_view.error: print ">>> {}".format(quick_view.out_picture) else: print "Error: Plot heat view error: {}".format(in_file) print '-' * 100
def main(pair, ymd): """ pair: sat1+sensor1_sat2+sensor2 ymd: str YYYYMMDD """ ######################### 初始化 ########################### # 提取参数中的卫星信息和传感器信息 part1, part2 = pair.split("_") sat1, sensor1 = part1.split("+") sat2, sensor2 = part2.split("+") sensor_pair = "{}_{}".format(sensor1, sensor2) # 判断是静止卫星还是动态卫星 if "FY2" in part1 or "FY4" in part1: sat_type = "GEOLEO" elif "FY3" in part1: sat_type = "LEOLEO" else: LOG.error("Cant distinguish the satellite type") return # 加载绘图配置文件 plt_cfg_file = os.path.join(MAIN_PATH, "%s_%s_3d.yaml" % (sensor1, sensor2)) plt_cfg = loadYamlCfg(plt_cfg_file) if plt_cfg is None: LOG.error("Not find the config file: {}".format(plt_cfg_file)) return else: # 加载配置信息 try: physics_pair = plt_cfg["regression"] # 需要处理的物理量对 if pb_io.is_none(physics_pair): LOG.error("Yaml args is not completion. : {}".format(plt_cfg_file)) return except ValueError: LOG.error("Load yaml config file error, please check it. : {}".format(plt_cfg_file)) return ######################### 开始处理 ########################### print '-' * 100 print 'Start regression daily.' # 循环处理每个物理量对 for each in physics_pair: dict_cabr = {} dict_cabr_d = {} dict_cabr_n = {} dict_bias = {} dict_bias_d = {} dict_bias_n = {} period = plt_cfg[each]["days"] # 回滚天数 # must be in "all", "day", "night" day_night = ["all", "day", "night"] if "time" in plt_cfg[each].keys(): day_night = plt_cfg[each]["time"] for t in day_night: if t not in ["all", "day", "night"]: day_night.remove(t) for idx, chan in enumerate(plt_cfg[each]["chan"]): LOG.info(u"Start Drawing {} Channel {}".format(each, chan)) oneHDF5 = ReadHDF5() num_file = period for daydelta in xrange(period): cur_ymd = pb_time.ymd_plus(ymd, -daydelta) hdf5_name = "COLLOC+%sIR,%s_C_BABJ_%s.hdf5" % (sat_type, pair, cur_ymd) filefullpath = os.path.join(MATCH_DIR, pair, hdf5_name) if not os.path.isfile(filefullpath): LOG.info(u"File not found: {}".format(filefullpath)) num_file -= 1 continue if not oneHDF5.LoadData(filefullpath, chan): LOG.error("Error occur when reading %s of %s" % (chan, filefullpath)) if num_file == 0: LOG.error(u"No file found.") continue elif num_file != period: LOG.error(u"{} of {} file(s) found.".format(num_file, period)) if is_monthly: str_time = ymd[:6] cur_path = os.path.join(MRA_DIR, pair, str_time) else: str_time = ymd cur_path = os.path.join(DRA_DIR, pair, str_time) # delete 0 in std if len(oneHDF5.rad1_std) > 0.0001: # TODO: 有些极小的std可能是异常值,而导致权重极大,所以 std>0 改成 std>0.0001 deletezeros = np.where(oneHDF5.rad1_std > 0.0001) oneHDF5.rad1_std = oneHDF5.rad1_std[deletezeros] oneHDF5.rad1 = oneHDF5.rad1[deletezeros] if len( oneHDF5.rad1) > 0 else oneHDF5.rad1 oneHDF5.rad2 = oneHDF5.rad2[deletezeros] if len( oneHDF5.rad2) > 0 else oneHDF5.rad2 oneHDF5.tbb1 = oneHDF5.tbb1[deletezeros] if len( oneHDF5.tbb1) > 0 else oneHDF5.tbb1 oneHDF5.tbb2 = oneHDF5.tbb2[deletezeros] if len( oneHDF5.tbb2) > 0 else oneHDF5.tbb2 oneHDF5.time = oneHDF5.time[deletezeros] if len( oneHDF5.time) > 0 else oneHDF5.time oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len( oneHDF5.lon1) > 0 else oneHDF5.lon1 oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len( oneHDF5.lon2) > 0 else oneHDF5.lon2 if len(oneHDF5.ref1_std) > 0.0001: deletezeros = np.where(oneHDF5.ref1_std > 0.0001) oneHDF5.ref1_std = oneHDF5.ref1_std[deletezeros] oneHDF5.ref1 = oneHDF5.ref1[deletezeros] if len( oneHDF5.ref1) > 0 else oneHDF5.ref1 oneHDF5.ref2 = oneHDF5.ref2[deletezeros] if len( oneHDF5.ref2) > 0 else oneHDF5.ref2 oneHDF5.dn1 = oneHDF5.dn1[deletezeros] if len( oneHDF5.dn1) > 0 else oneHDF5.dn1 oneHDF5.dn2 = oneHDF5.dn1[deletezeros] if len( oneHDF5.dn2) > 0 else oneHDF5.dn2 oneHDF5.time = oneHDF5.time[deletezeros] if len( oneHDF5.time) > 0 else oneHDF5.time oneHDF5.lon1 = oneHDF5.lon1[deletezeros] if len( oneHDF5.lon1) > 0 else oneHDF5.lon1 oneHDF5.lon2 = oneHDF5.lon2[deletezeros] if len( oneHDF5.lon2) > 0 else oneHDF5.lon2 # find out day and night if ("day" in day_night or "night" in day_night) and len(oneHDF5.time) > 0: vect_is_day = np.vectorize(is_day_timestamp_and_lon) day_index = vect_is_day(oneHDF5.time, oneHDF5.lon1) night_index = np.logical_not(day_index) else: day_index = None night_index = None # 将每个对通用的属性值放到对循环,每个通道用到的属性值放到通道循环 # get threhold, unit, names... xname, yname = each.split("-") xname_l = plt_cfg[each]["x_name"] xunit = plt_cfg[each]["x_unit"] xlimit = plt_cfg[each]["x_range"][idx] xmin, xmax = xlimit.split("-") xmin = float(xmin) xmax = float(xmax) yname_l = plt_cfg[each]["y_name"] yunit = plt_cfg[each]["y_unit"] ylimit = plt_cfg[each]["y_range"][idx] ymin, ymax = ylimit.split("-") ymin = float(ymin) ymax = float(ymax) weight = None if "rad" in xname: x = oneHDF5.rad1 elif "tbb" in xname: x = oneHDF5.tbb1 elif "ref" in xname: x = oneHDF5.ref1 elif "dn" in xname: x = oneHDF5.dn1 else: LOG.error("Can't plot %s" % each) continue if "rad" in yname: y = oneHDF5.rad2 elif "tbb" in yname: y = oneHDF5.tbb2 elif "ref" in yname: y = oneHDF5.ref2 else: LOG.error("Can't plot %s" % each) continue if "rad" in xname and "rad" in yname: if len(oneHDF5.rad1_std) > 0: weight = oneHDF5.rad1_std o_name = "RadCalCoeff" elif "tbb" in xname and "tbb" in yname: o_name = "TBBCalCoeff" elif "ref" in xname and "ref" in yname: if len(oneHDF5.ref1_std) > 0: weight = oneHDF5.ref1_std o_name = "CorrcCoeff" elif "dn" in xname and "ref" in yname: o_name = "CalCoeff" # 画对角线 if xname == yname: diagonal = True else: diagonal = False if "all" in day_night and o_name not in dict_cabr: dict_cabr[o_name] = {} dict_bias[xname] = {} if "day" in day_night and o_name not in dict_cabr_d: dict_cabr_d[o_name] = {} dict_bias_d[xname] = {} if "night" in day_night and o_name not in dict_cabr_n: dict_cabr_n[o_name] = {} dict_bias_n[xname] = {} # 对样本点数量进行判断,如果样本点少于 100 个,则不进行绘制 if x.size < 100: LOG.error("Not enough match point to draw: {}, {}".format(each, chan)) if "all" in day_night: dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias[xname][chan] = [np.NaN, np.NaN] if "day" in day_night: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_d[xname][chan] = [np.NaN, np.NaN] if "night" in day_night: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_n[xname][chan] = [np.NaN, np.NaN] continue # regression starts if "all" in day_night: o_file = os.path.join(cur_path, "%s_%s_%s_ALL_%s" % ( pair, o_name, chan, str_time)) print("x_all, y_all", len(x), len(y)) abr, bias = plot(x, y, weight, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal, is_monthly) if abr: dict_cabr[o_name][chan] = abr else: dict_cabr[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if bias: dict_bias[xname][chan] = bias else: dict_bias[xname][chan] = [np.NaN, np.NaN] # ------- day ---------- if "day" in day_night: if day_index is not None and np.where(day_index)[0].size > 10: o_file = os.path.join(cur_path, "%s_%s_%s_Day_%s" % ( pair, o_name, chan, str_time)) x_d = x[day_index] y_d = y[day_index] w_d = weight[day_index] if weight is not None else None print("x_all, y_all", len(x), len(y)) print("x_day, y_day", len(x_d), len(y_d)) abr, bias = plot(x_d, y_d, w_d, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal, is_monthly) if abr: dict_cabr_d[o_name][chan] = abr else: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if bias: dict_bias_d[xname][chan] = bias else: dict_bias_d[xname][chan] = [np.NaN, np.NaN] else: dict_cabr_d[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_d[xname][chan] = [np.NaN, np.NaN] # ---------night ------------ if "night" in day_night: if night_index is not None and np.where(night_index)[0].size > 10: o_file = os.path.join(cur_path, "%s_%s_%s_Night_%s" % ( pair, o_name, chan, str_time)) x_n = x[night_index] y_n = y[night_index] w_n = weight[night_index] if weight is not None else None print("x_all, y_all", len(x), len(y)) print("x_night, y_night", len(x_n), len(y_n)) abr, bias = plot(x_n, y_n, w_n, o_file, num_file, part1, part2, chan, str_time, xname, xname_l, xunit, xmin, xmax, yname, yname_l, yunit, ymin, ymax, diagonal, is_monthly) if abr: dict_cabr_n[o_name][chan] = abr else: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] if bias: dict_bias_n[xname][chan] = bias else: dict_bias_n[xname][chan] = [np.NaN, np.NaN] else: dict_cabr_n[o_name][chan] = [0, np.NaN, np.NaN, np.NaN] dict_bias_n[xname][chan] = [np.NaN, np.NaN] oneHDF5.clear() # write txt lock.acquire() channel = plt_cfg[each]["chan"] if "all" in day_night: for o_name in dict_cabr: write_bias(channel, part1, part2, xname, ymd, dict_bias, "ALL") write_cabr(channel, part1, part2, o_name, ymd, dict_cabr, "ALL") if "day" in day_night: for o_name in dict_cabr_d: write_bias(channel, part1, part2, xname, ymd, dict_bias_d, "Day") write_cabr(channel, part1, part2, o_name, ymd, dict_cabr_d, "Day") if "night" in day_night: for o_name in dict_cabr_n: write_bias(channel, part1, part2, xname, ymd, dict_bias_n, "Night") write_cabr(channel, part1, part2, o_name, ymd, dict_cabr_n, "Night") lock.release()
def combine(self): if self.error: return # 如果输出文件已经存在,跳过 elif os.path.isfile(self.ofile): self.error = True print "File is already exist, skip it: {}".format(self.ofile) return # 合成日数据 elif pb_io.is_none(self.ifile, self.pfile, self.ofile): self.error = True print "Is None: ifile or pfile or ofile: {}".format(self.yaml_file) return elif len(self.ifile) < 1: self.error = True print "File count lower than 1: {}".format(self.yaml_file) fill_value = -32767 for file_idx, in_file in enumerate(self.ifile): proj_file = self.pfile[file_idx] if os.path.isfile(in_file) and os.path.isfile(proj_file): print "*" * 100 print "Start combining file:" print "<<< {}\n<<< {}".format(in_file, proj_file) else: print "File is not exist: {} OR {}".format(in_file, proj_file) continue # 加载 proj 数据 self.load_proj_data(proj_file) # 日合成 with time_block("One combine time:", switch=TIME_TEST): try: with h5py.File(in_file, 'r') as h5: for k in h5.keys(): # 记录属性信息 if k not in self.attrs: self.attrs[k] = pb_io.attrs2dict( h5.get(k).attrs) if k == "Longitude" or k == "Latitude": continue elif k not in self.out_data: if k == "Ocean_Flag": self.out_data[k] = np.full( (self.row, self.col), fill_value, dtype='i4') else: self.out_data[k] = np.full( (self.row, self.col), fill_value, dtype='i2') # 合并一个数据 proj_data = h5.get(k)[:] self.out_data[k][self.lut_ii, self.lut_jj] = proj_data[ self.data_ii, self.data_jj] except Exception as why: print why print "Can't combine file, some error exist: {}".format( in_file) with time_block("Grid to lons and lats time:", switch=TIME_TEST): if "Longitude" not in self.out_data: lookup_table = prj_core(self.cmd, self.res, unit="deg", row=self.row, col=self.col) lookup_table.grid_lonslats() self.out_data["Longitude"] = lookup_table.lons self.out_data["Latitude"] = lookup_table.lats # 输出数据集有效数据的数量 keys = [x for x in self.out_data] keys.sort() for k in keys: if self.out_data[k] is None: print k continue idx = np.where(self.out_data[k] > 0) print "{:30} : {}".format(k, len(idx[0]))
def combine(self): all_time_start = datetime.utcnow() if self.error: return # 如果输出文件已经存在,跳过 elif os.path.isfile(self.ofile): self.error = True print "File is already exist, skip it: {}".format(self.ofile) return # 合成日数据 elif pb_io.is_none(self.ifile, self.pfile, self.ofile): self.error = True print "Is None: ifile or pfile or ofile: {}".format(self.yaml_file) return elif len(self.ifile) < 1: self.error = True print "File count lower than 1: {}".format(self.yaml_file) self.load_proj_files(self.pfile) # 读取投影文件 fill_value = -32767 with h5py.File(self.ifile[0], 'r') as h5: data_names = h5.keys() data_names.sort() for k in h5.keys(): # 记录属性信息 if k not in self.attrs: self.attrs[k] = pb_io.attrs2dict(h5.get(k).attrs) file_count = len(self.ifile) data_count = list() date_start = datetime.utcnow() for data_name in data_names: if data_name == "Longitude" or data_name == "Latitude" or data_name == "Ocean_Flag": continue if 'Azimuth' in data_name or 'Zenith' in data_name or data_name == 'chl': continue data = list() count = 0 # 读取所有的数据 for in_file in self.ifile: filename = os.path.basename(in_file) k = filename.split('_1000M')[0] data_ii, data_jj = self.data_ij[k] with h5py.File(in_file, 'r') as hdf5: data_read = hdf5.get(data_name)[:][data_ii, data_jj] data = np.append(data, data_read) date_end = datetime.utcnow() - date_start count += 1 print 'Combine {} {}/{} {}'.format(data_name, count, file_count, date_end) condition = np.logical_and(data != 32767, data != -32767) condition = np.logical_and(condition, data != 0) lut_ii = self.lut_ii[condition] lut_jj = self.lut_jj[condition] data = data[condition] data_count.append((data_name, len(data))) if not len(data) == 0: ijd = dict() date_start = datetime.utcnow() for i, j, d in zip(lut_ii, lut_jj, data): local = (i, j) if local not in ijd: ijd[local] = [d] else: data_list = ijd[local] data_list.append(d) date_end = datetime.utcnow() - date_start print date_end local = ijd.keys() local = np.array(local) lut_ii = local[:, 0].astype(np.int16) lut_jj = local[:, 1].astype(np.int16) data = ijd.values() date_start = datetime.utcnow() mean_ = np.vectorize(mean_data) data = mean_(data) date_end = datetime.utcnow() - date_start print date_end out_data = np.full((self.row, self.col), fill_value, dtype='i2') data = data.reshape(-1) out_data[lut_ii, lut_jj] = data else: out_data = np.full((self.row, self.col), fill_value, dtype='i2') self.write(out_data, data_name, self.ofile) # 输出经纬度数据集 lookup_table = prj_core(self.cmd, self.res, unit="deg", row=self.row, col=self.col) lookup_table.grid_lonslats() longitude = lookup_table.lons latitude = lookup_table.lats data_name = "Longitude" self.write(longitude, data_name, self.ofile) data_name = "Latitude" self.write(latitude, data_name, self.ofile) # 输出海陆标记和方位角 for data_name in [ 'SensorAzimuth', 'SensorZenith', 'SolarAzimuth', 'SolarZenith', 'Ocean_Flag', 'chl' ]: out_data = np.full((self.row, self.col), fill_value, dtype='i4') data = list() for in_file in self.ifile: filename = os.path.basename(in_file) k = filename.split('_1000M')[0] data_ii, data_jj = self.data_ij[k] with h5py.File(in_file, 'r') as hdf5: data_read = hdf5.get(data_name)[:][data_ii, data_jj] data = np.append(data, data_read) data = np.reshape(data, -1) lut_ii = self.lut_ii.astype(np.int16) lut_jj = self.lut_jj.astype(np.int16) out_data[lut_ii, lut_jj] = data self.write(out_data, data_name, self.ofile) # 输出数据集有效数据的数量 for count in data_count: print "{:30} : {}".format(count[0], count[1]) all_time_end = datetime.utcnow() - all_time_start print all_time_end