def job_0311(job_exe, sat_pair, date_s, date_e, job_id): ''' :水色反演 ''' Log.info(u'%s: %s 水色反演处理开始...' % (job_id, job_exe)) # 去掉路径信息 in_path = cfg_body['PATH']['MID']['granule'] out_path = in_path arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_out_path = pb_io.path_replace_ymd(out_path, ymd) if not os.path.isdir(use_out_path): os.makedirs(use_out_path) use_reg = '.*_%s_.*.HDF' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = 'idl -rt=%s -args %s %s/' % (job_exe, in_file, use_out_path) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0710(job_exe, sat_pair, date_s, date_e, job_id): ''' :月合成 ''' Log.info(u'%s: %s 月合成处理开始...' % (job_id, job_exe)) cfg_path = cfg_body['PATH']['MID']['incfg'] daily_path = cfg_body['PATH']['OUT']['daily'] month_path = cfg_body['PATH']['OUT']['monthly'] # 清理配置 if os.path.isdir(cfg_path): shutil.rmtree(cfg_path) arg_list = [] # 月首和月末 调整 ymd1 = date_s.strftime('%Y%m%d') ymd2 = date_e.strftime('%Y%m%d') lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1] date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d') date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d') while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') # 输出 com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAM_5000M.HDF' % ( cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd) month_path_use = pb_io.path_replace_ymd(month_path, ymd) com_out_file = os.path.join(month_path_use, com_filename) # 输入 reg = '.*_%s.*.HDF' % ymd[0:6] daily_path_use = pb_io.path_replace_ymd(daily_path, ymd) data_list_use = pb_io.find_file(daily_path_use, reg) if len(data_list_use) > 0: com_dict = { 'PATH': { 'ipath': data_list_use, 'opath': com_out_file } } cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) date_s = date_s + relativedelta(months=1) reg = '.*.yaml' FileLst = pb_io.find_file(cfg_path, reg) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0610(job_exe, sat_pair, date_s, date_e, job_id): ''' :日合成 ''' Log.info(u'%s: %s 日合成处理开始...' % (job_id, job_exe)) granule_path = cfg_body['PATH']['MID']['granule'] proj_path = cfg_body['PATH']['MID']['projection'] cfg_path = cfg_body['PATH']['MID']['incfg'] daily_path = cfg_body['PATH']['OUT']['daily'] # 清理配置 if os.path.isdir(cfg_path): shutil.rmtree(cfg_path) arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % ( cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd) daily_path_use = pb_io.path_replace_ymd(daily_path, ymd) com_out_file = os.path.join(daily_path_use, com_filename) granule_path_use = pb_io.path_replace_ymd(granule_path, ymd) reg = '.*_%s_.*.HDF' % ymd granule_lst = pb_io.find_file(granule_path_use, reg) # reg = '.*_%s_.*.HDF' % ymd proj_path_use = pb_io.path_replace_ymd(proj_path, ymd) granule_pro_lst = pb_io.find_file(proj_path_use, reg) if len(granule_pro_lst) > 0: com_dict = { 'PATH': { 'ipath': granule_lst, 'ppath': granule_pro_lst, 'opath': com_out_file } } cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) date_s = date_s + relativedelta(days=1) reg = '.*.yaml' FileLst = pb_io.find_file(cfg_path, reg) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0711(job_exe, sat_pair, date_s, date_e, job_id): ''' :月成后文件绘图 ''' Log.info(u'%s: %s 月合成出图处理开始...' % (job_id, job_exe)) ipath = cfg_body['PATH']['OUT']['monthly'] FileLst = [] arg_list = [] # 月首和月末 调整 ymd1 = date_s.strftime('%Y%m%d') ymd2 = date_e.strftime('%Y%m%d') lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1] date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d') date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d') while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') path_use = pb_io.path_replace_ymd(ipath, ymd) reg = '.*_%s_.*.HDF' % ymd mlist = pb_io.find_file(path_use, reg) FileLst.extend(mlist) date_s = date_s + relativedelta(months=1) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0310(job_exe, sat_pair, date_s, date_e, job_id): ''' :反演 ''' Log.info(u'%s: %s 反演预处理开始...' % (job_id, job_exe)) cfg = 'aerosol.cfg' # 去掉路径信息 job_exe = os.path.basename(job_exe) in_path = cfg_body['PATH']['MID']['calibrate'] reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = '.*_%s_.*.HDF' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = './%s %s %s ' % (job_exe, in_file, cfg) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0810(job_exe, sat_pair, date_s, date_e, job_id): ''' :年合成 ''' Log.info(u'%s: %s 年合成处理开始...' % (job_id, job_exe)) cfg_path = cfg_body['PATH']['MID']['incfg'] month_path = cfg_body['PATH']['OUT']['monthly'] yearly_path = cfg_body['PATH']['OUT']['yearly'] # 清理配置 if os.path.isdir(cfg_path): shutil.rmtree(cfg_path) arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') # 输出 com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_YEAR_5000M.HDF' % ( cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd) yearly_path_use = pb_io.path_replace_ymd(yearly_path, ymd) com_out_file = os.path.join(yearly_path_use, com_filename) # 输入 reg = '.*_%s.*.HDF' % ymd[0:4] month_path_use = pb_io.path_replace_ymd(month_path, ymd) data_list_use = pb_io.find_file(month_path_use, reg) if len(data_list_use) > 0: com_dict = { 'PATH': { 'ipath': data_list_use, 'opath': com_out_file } } cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) date_s = date_s + relativedelta(years=1) reg = '.*.yaml' FileLst = pb_io.find_file(cfg_path, reg) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def _get_obc_file(m1000_file, obc_path): """ 通过 1KM 文件路径生成 OBC 文件的路径 :param m1000_file: :param obc_path: :return: """ ymd = _get_ymd(m1000_file) hm = _get_hm(m1000_file) time = ymd + hm obc_path = path_replace_ymd(obc_path, ymd) obc_files = get_files_by_ymd(obc_path, time, time, ext='.HDF', pattern_ymd=r".*_(\d{8})_(\d{4})_") obc_file = obc_files[0] return obc_file
def job_0110(job_exe, sat_pair, date_s, date_e, job_id): """ ncep处理的输入接口 """ Log.info(u'%s: %s ncep处理开始...' % (job_id, job_exe)) in_path = cfg_body['PATH']['IN']['ncep'] reg = 'fnl_%s_.*' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = reg % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0210(job_exe, sat_pair, date_s, date_e, job_id): """ L1数据预处理的输入接口 """ Log.info(u'%s: %s L1数据预处理开始...' % (job_id, job_exe)) in_path = cfg_body['PATH']['IN']['l1'] # reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = '.*_%s_.*.HDF$' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def _get_geo_file(m1000_file, geo_path): """ 通过 1KM 文件路径生成 GEO 文件的路径 :param m1000_file: :param geo_path: :return: """ ymd = _get_ymd(m1000_file) hm = _get_hm(m1000_file) time = ymd + hm geo_path = path_replace_ymd(geo_path, ymd) print geo_path print time geo_files = get_files_by_ymd(geo_path, time, time, ext='.HDF', pattern_ymd=r".*_(\d{8})_(\d{4})_") geo_file = geo_files[0] return geo_file
def job_0611(job_exe, sat_pair, date_s, date_e, job_id): ''' :合成后文件绘图 ''' Log.info(u'%s: %s 日合成出图处理开始...' % (job_id, job_exe)) ipath = cfg_body['PATH']['OUT']['daily'] FileLst = [] arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') path_use = pb_io.path_replace_ymd(ipath, ymd) reg = '.*_%s_.*.HDF' % ymd dlist = pb_io.find_file(path_use, reg) FileLst.extend(dlist) date_s = date_s + relativedelta(days=1) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0410(job_exe, sat_pair, date_s, date_e, job_id): ''' :反演后的快视图 ''' Log.info(u'%s: %s 反演轨道产品快视图处理开始...' % (job_id, job_exe)) in_path = cfg_body['PATH']['MID']['granule'] # reg = 'FY3[A-Z]_MERSI_ORBT_L2_\w{3}_MLT_NUL_%s_(\d{4})_1000M.HDF' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = '.*_%s_.*.HDF' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def _get_out_file(in_file, out_path, suffix): """ 通过输入的文件名和输出路径获取输出文件的完整路径 :return: """ try: if isinstance(in_file, str): ymd = pb_time.get_ymd(in_file) out_path = pb_io.path_replace_ymd(out_path, ymd) _name = os.path.basename(in_file) name = _name.replace("_c", "") + suffix out_file = os.path.join(out_path, name) if not os.path.isdir(os.path.dirname(out_file)): try: os.makedirs(os.path.dirname(out_file)) except OSError as why: print why return out_file except Exception as why: print why return
def job_0211(job_exe, sat_pair, date_s, date_e, job_id, reload=None): if reload is None: Log.info(u'%s: %s 交叉匹配(use pre data)处理开始...' % (job_id, job_exe)) else: print 'reload' # 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] DATA_DIR = cfg_body['PATH']['IN']['data'] CALI_DIR = cfg_body['PATH']['MID']['calibrate'] jobCfg = cfg_body['PATH']['IN']['jobCfg'] match_path = cfg_body['PATH']['MID']['match'] L1_data_dir = cfg_body['PATH']['IN']['l1'] coeff_path = cfg_body['PATH']['IN']['coeff'] # 存放分发列表 arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') jjj = date_s.strftime('%j') coeff_file = os.path.join(coeff_path, '%s.txt' % ymd[:4]) print ymd # 存放俩颗卫星的原始数据目录位置 if reload is None: # inpath1 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' % # (sat1, sensor1), ymd[:6]) FINAL_DIR = pb_io.path_replace_ymd(CALI_DIR, ymd) inpath1 = os.path.join(FINAL_DIR) else: inpath1 = pb_io.path_replace_ymd(L1_data_dir, ymd) inpath2 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' % ( sat2, sensor2, ), ymd[:6]) print "inpath1", inpath1 print "inpath2", inpath2 sat11 = cfg_body['SAT_S2L'][sat1] sat22 = cfg_body['SAT_S2L'][sat2] # 读取交叉点上的俩颗卫星的交叉时间,1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间 timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd) print 'cross', len(timeList) reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd reg2 = 'MYD021KM.A%s%s.*.hdf' % (ymd[0:4], jjj) file_list1 = pb_io.find_file(inpath1, reg1) file_list2 = pb_io.find_file(inpath2, reg2) # 根据交叉点时间,找到数据列表中需要的数据 select File for crossTime in timeList: Lat = crossTime[0] Lon = crossTime[1] ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S') s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1)) e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1)) s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2)) e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2)) # 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据 list1 = Find_data_FromCrossTime(file_list1, s_cross_time1, e_cross_time1) list2 = Find_data_FromCrossTime(file_list2, s_cross_time2, e_cross_time2) print 'fy', len(list1) print 'mo', len(list2) # 存放匹配信息的yaml配置文件存放位置 yaml_file3 = os.path.join( jobCfg, sat_pair, job_id, ymdhms[:8], '%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2)) filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms) # 输出完整路径 # FY3B+MERSI_AQUA+MODIS_L1 和 FY3B+MERSI_AQUA+MODIS 两种支持 wangpeng # add 2018-09-14 full_filename3 = os.path.join(match_path, sat_pair, ymdhms[:6], filename3) # if reload is None: # full_filename3 = os.path.join( # match_path, sat_pair, ymdhms[:6], filename3) # else: # full_filename3 = os.path.join( # match_path, sat_pair + '_L1', ymdhms[:6], filename3) # 投影参数 cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % ( Lat, Lon) if len(list1) > 0 and len(list2) > 0: print '111111' row = 128 col = 128 res = 8000 dict3 = { 'INFO': { 'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd': ymdhms }, 'PATH': { 'opath': full_filename3, 'ipath1': list1, 'ipath2': list2, 'ipath_coeff': coeff_file }, 'PROJ': { 'cmd': cmd, 'row': row, 'col': col, 'res': res } } Log.info('%s %s create collocation cfg success' % (sat_pair, ymdhms)) CreateYamlCfg(dict3, yaml_file3) if reload is None: cmd = '%s %s %s 0' % (python, job_exe, yaml_file3) else: cmd = '%s %s %s 1' % (python, job_exe, yaml_file3) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def main(sat_sensor, in_file): """ 对L2数据进行投影,记录投影后的位置信息和数据信息 :param sat_sensor: 卫星+传感器 :param in_file: HDF5 文件 :return: """ # ######################## 初始化 ########################### # 获取程序所在位置,拼接配置文件 app = InitApp(sat_sensor) if app.error: print "Load config file error." return gc = app.global_config sc = app.sat_config log = LogServer(gc.path_out_log) # 加载全局配置信息 out_path = gc.path_mid_projection # 加载程序配置信息 # 加载卫星配置信息 res = sc.project_res half_res = deg2meter(res) / 2. cmd = sc.project_cmd % (half_res, half_res) row = sc.project_row col = sc.project_col mesh_size = sc.project_mesh_size # ######################## 开始处理 ########################### print "-" * 100 print "Start projection." if not os.path.isfile(in_file): log.error("File is not exist: {}".format(in_file)) return print "<<< {}".format(in_file) with time_block("One project time:", switch=TIME_TEST): # 生成输出文件的文件名 ymd = _get_ymd(in_file) hm = _get_hm(in_file) sat, sensor = sat_sensor.split('+') out_name = "{}_{}_ORBT_L2_OCC_MLT_NUL_{}_{}_{}.HDF".format( sat, sensor, ymd, hm, mesh_size.upper()) out_path = pb_io.path_replace_ymd(out_path, ymd) out_file = os.path.join(out_path, out_name) # 如果输出文件已经存在,跳过 if os.path.isfile(out_file): print "File is already exist, skip it: {}".format(out_file) return # 开始创建投影查找表 projection = Projection(cmd=cmd, row=row, col=col, res=res) projection.project(in_file, out_file) if not projection.error: print ">>> {}".format(out_file) else: print "Error: Projection error: {}".format(in_file) print "-" * 100
def job_0911(job_exe, sat_pair, date_s, date_e, job_id): """ 基于aqua(modis)和fy3b(mersi)的l3数据匹配结果 """ # date_s, date_e = time.split("-") # date_s = datetime.strptime(date_s, '%Y%m%d%H%M') # date_e = datetime.strptime(date_e, '%Y%m%d%H%M') Log.info(u'%s: %s 检验l3产品 开始...' % (job_id, job_exe)) reg_fy3b = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' reg_aqua_chl1 = 'L3m_%s__GLOB_4_GSM-MOD_%s_DAY_00.nc' reg_aqua = 'L3m_%s__GLOB_4_AV-MOD_%s_DAY_00.nc' fy3b_l3 = cfg_body['PATH']['OUT']['daily'] # fy3b_l3 apth modis_l3 = cfg_body['PATH']['IN']['modis_l3'] # modis_l3 jobcfg_path = cfg_body['PATH']['IN']['jobCfg'] out_path = cfg_body['PATH']['MID']['match'] arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') # 查找fy3b_mersi文件 match_fy3b_file = [] fy3b_l3_path_use = pb_io.path_replace_ymd(fy3b_l3, ymd) file_fy3b = pb_io.find_file(fy3b_l3_path_use, reg_fy3b % (ymd)) match_fy3b_file.extend(file_fy3b) # 检查是否已经生成产品 pruc_names = ['KD490', 'CHL1', 'POC', 'ZSD'] match_aqua = [] for pruc_name in pruc_names: # outfile = out_path + pruc_name + os.sep + \ # ymd + '_' + pruc_name + "_5000.hdf" # if os.path.isfile(outfile): # print u'产品已经存在' # pass # else: # 查找对应的aqua_modis产品文件 if pruc_name != 'CHL1': file_aqua = pb_io.find_file(modis_l3, reg_aqua % (ymd, pruc_name)) match_aqua.extend(file_aqua) else: file_aqua = pb_io.find_file(modis_l3, reg_aqua % (ymd, pruc_name)) if len(file_aqua) > 0: match_aqua.extend(file_aqua) else: file_aqua1 = pb_io.find_file( modis_l3, reg_aqua_chl1 % (ymd, pruc_name)) match_aqua.extend(file_aqua1) # 判断当前五分钟的数据是否都存在 if len(match_fy3b_file) > 0 and len(match_aqua) > 0: com_dict = { 'PATH': { 'ipath1': match_fy3b_file, 'ipath2': match_aqua, 'out_path': out_path }, 'INFO': { 'time': ymd, 'pair': sat_pair }, } cfgFile = os.path.join(jobcfg_path, sat_pair, job_id, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) # check_fy_aqual2 = CheckFyAqual2() # check_fy_aqual2.main( # match_fy3b_file, match_iop_file, match_oc_file, ymd_HM, out_path) cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def job_0910(job_exe, sat_pair, date_s, date_e, job_id): """ 基于aqua(modis)和fy3b(mersi)的l2数据匹配结果 """ # date_s, date_e = time.split("-") # date_s = datetime.strptime(date_s, '%Y%m%d%H%M') # date_e = datetime.strptime(date_e, '%Y%m%d%H%M') Log.info(u'%s: %s 检验l2产品 开始...' % (job_id, job_exe)) # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] reg_fy3b = 'FY3B_MERSI_ORBT_L2_OCC_MLT_NUL_.*._1000M.HDF' reg_aqua_iop = 'A.*.L2_LAC_IOP.nc' reg_aqua_oc = 'A.*.L2_LAC_OC.nc' fy3b_l2 = cfg_body['PATH']['MID']['granule'] # fy3b_l2 apth modis_l2 = cfg_body['PATH']['IN']['data'] # modis_l2 jobcfg_path = cfg_body['PATH']['IN']['jobCfg'] out_path = cfg_body['PATH']['MID']['match'] arg_list = [] while date_s <= date_e: # ymd_HM = date_s.strftime('%Y%m%d_%H%M') ymd = date_s.strftime('%Y%m%d') # 解析mathcing: FY3A+MERSI_AQUA+MODIS FY3B+MERSI_AQUA+MODIS_check sat1 = (sat_pair.split('_')[0]).split('+')[0] sat2 = (sat_pair.split('_')[1]).split('+')[0] fy3b_l2_path_use = pb_io.path_replace_ymd(fy3b_l2, ymd) modis_l2_path_use = modis_l2 + '/AQUA/MODIS/L2/ORBIT' + \ os.sep + date_s.strftime('%Y%m') file_list1 = pb_io.find_file(fy3b_l2_path_use, reg_fy3b) file_list2 = pb_io.find_file(modis_l2_path_use, reg_aqua_iop) # file_list3 = pb_io.find_file(modis_l2_path_use, reg_aqua_oc) timeList = ReadCrossFile_LEO_LEO('FENGYUN-3B', 'AQUA', ymd) # print timeList, CROSS_DIR, sat1, sat2, ymd for crossTime in timeList: ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S') s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1)) e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1)) s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2)) e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2)) # 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据 match_fy3b_file = Find_data_fy_FromCrossTime( file_list1, s_cross_time1, e_cross_time1) match_iop_file = Find_data_aqua_FromCrossTime( file_list2, s_cross_time2, e_cross_time2) # match_oc_file = Find_data_aqua_FromCrossTime( # file_list3, s_cross_time2, e_cross_time2) # 判断当前五分钟的数据是否都存在 if len(match_fy3b_file) > 0 and len(match_iop_file) > 0: for match_fy3b in match_fy3b_file: for match_iop in match_iop_file: com_dict = { 'PATH': { 'ipath1': [match_fy3b], 'ipath2': [match_iop], 'out_path': out_path }, 'INFO': { 'time': ymdhms, 'pair': sat_pair, 'lat': crossTime[0], 'lon': crossTime[1] } } cfgFile = os.path.join(jobcfg_path, sat_pair, job_id, '%s.yaml' % ymdhms) CreateYamlCfg(com_dict, cfgFile) # check_fy_aqual2 = CheckFyAqual2() # check_fy_aqual2.main( # match_fy3b_file, match_iop_file, match_oc_file, ymd_HM, # out_path) cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def job_0215(job_exe, sat_pair, date_s, date_e, job_id): Log.info(u'%s: %s 水色L2产品检验处理开始...' % (job_id, job_exe)) # 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] in_path1 = cfg_body['PATH']['MID']['granule'] in_path2 = cfg_body['PATH']['IN']['data'] out_path_cfg = cfg_body['PATH']['IN']['jobCfg'] out_path_match = cfg_body['PATH']['MID']['match'] # 存放分发列表 arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') jjj = date_s.strftime('%j') # 存放俩颗卫星的原始数据目录位置 full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd) full_in_path2 = os.path.join(in_path2, '%s/%s/L2/ORBIT' % ( sat2, sensor2, ), ymd[:6]) print "inpath1", full_in_path1 print "inpath2", full_in_path2 sat11 = cfg_body['SAT_S2L'][sat1] sat22 = cfg_body['SAT_S2L'][sat2] # 读取交叉点上的俩颗卫星的交叉时间,1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间 timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd) print 'cross', len(timeList) reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd reg2 = 'A%s%s.*.nc' % (ymd[0:4], jjj) file_list1 = pb_io.find_file(full_in_path1, reg1) file_list2 = pb_io.find_file(full_in_path2, reg2) # 根据交叉点时间,找到数据列表中需要的数据 select File for crossTime in timeList: Lat = crossTime[0] Lon = crossTime[1] ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S') s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1)) e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1)) s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2)) e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2)) # 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据 list1 = Find_data_FromCrossTime(file_list1, s_cross_time1, e_cross_time1) list2 = Find_data_FromCrossTime(file_list2, s_cross_time2, e_cross_time2) print 'fy', len(list1) print 'mo', len(list2) # 存放匹配信息的yaml配置文件存放位置 yaml_file3 = os.path.join( out_path_cfg, sat_pair, job_id, ymdhms[:8], '%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2)) filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms) # 输出完整路径 full_filename3 = os.path.join(out_path_match, sat_pair, ymdhms[:6], filename3) # 投影参数 cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % ( Lat, Lon) if len(list1) > 0 and len(list2) > 0: print '111111' row = 128 col = 128 res = 8000 dict3 = { 'INFO': { 'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd': ymdhms, 'pair': sat_pair }, 'PATH': { 'opath': full_filename3, 'ipath1': list1, 'ipath2': list2 }, 'PROJ': { 'cmd': cmd, 'row': row, 'col': col, 'res': res } } Log.info('%s %s create collocation cfg success' % (sat_pair, ymdhms)) CreateYamlCfg(dict3, yaml_file3) cmd = '%s %s %s' % (python, job_exe, yaml_file3) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def job_0216(job_exe, sat_pair, date_s, date_e, job_id): Log.info(u'%s: %s 水色L3产品检验处理开始...' % (job_id, job_exe)) # 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] in_path1 = cfg_body['PATH']['OUT']['daily'] in_path2 = cfg_body['PATH']['IN']['data'] out_path_cfg = cfg_body['PATH']['IN']['jobCfg'] out_path_match = cfg_body['PATH']['MID']['match'] # 存放分发列表 arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') jjj = date_s.strftime('%j') # 存放俩颗卫星的原始数据目录位置 full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd) full_in_path2 = os.path.join(in_path2, '%s/%s/L3' % (sat2, sensor2)) print "inpath1", full_in_path1 print "inpath2", full_in_path2 sat11 = cfg_body['SAT_S2L'][sat1] sat22 = cfg_body['SAT_S2L'][sat2] # 读取交叉点上的俩颗卫星的交叉时间,1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间 reg1 = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % ymd reg2 = 'L3m_%s__GLOB_4_AV-MOD_KD490_DAY_00.nc' % (ymd) file_list1 = pb_io.find_file(full_in_path1, reg1) file_list2 = pb_io.find_file(full_in_path2, reg2) # 存放匹配信息的yaml配置文件存放位置 yaml_file3 = os.path.join(out_path_cfg, sat_pair, job_id, ymd, '%s_%s_%s.yaml' % (ymd, sensor1, sensor2)) filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymd) # 输出完整路径 full_filename3 = os.path.join(out_path_match, sat_pair, ymd[:6], filename3) # 投影参数 deg = 0.05 res = deg2meter(deg) half_res = deg2meter(res) / 2. cmd = '+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=-%f +y_0=%f +datum=WGS84' % ( half_res, half_res) if len(file_list1) > 0 and len(file_list2) > 0: print '111111' dict3 = { 'INFO': { 'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd': ymd, 'pair': sat_pair }, 'PATH': { 'opath': full_filename3, 'ipath1': file_list1, 'ipath2': file_list2 }, 'PROJ': { 'cmd': cmd, 'res': deg } } Log.info('%s %s create collocation cfg success' % (sat_pair, ymd)) CreateYamlCfg(dict3, yaml_file3) cmd = '%s %s %s' % (python, job_exe, yaml_file3) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def main(sat_sensor, in_file): """ 使用矫正系数对 MERSI L1 的产品进行定标预处理 :param sat_sensor: (str) 卫星对 :param in_file: (str) 输入文件 :return: """ # ######################## 初始化 ########################### # 获取程序所在位置,拼接配置文件 app = InitApp(sat_sensor) if app.error: print "Load config file error." return gc = app.global_config sc = app.sat_config log = LogServer(gc.path_out_log) # 全局配置接口 obc_path = gc.path_in_obc geo_path = gc.path_in_geo coeff_path = gc.path_in_coeff out_path = gc.path_mid_calibrate # 程序配置接口 # 卫星配置接口 launch_date = sc.launch_date probe_count = sc.calibrate_probe_count probe = sc.calibrate_probe slide_step = sc.calibrate_slide_step plot = sc.calibrate_plot # ######################## 开始处理 ########################### print '-' * 100 print 'Start calibration' # 不同的卫星对使用不同的处理类和流程 if "fy3b" in sat_sensor.lower() and 'mersi' in sat_sensor.lower(): # 获取 M1000 文件和对应 OBC 文件 l1_1000m = in_file obc_1000m = _get_obc_file(l1_1000m, obc_path) if not os.path.isfile(l1_1000m): log.error("File is not exist: {}".format(l1_1000m)) return elif not os.path.isfile(obc_1000m): log.error("File is not exist: {}".format(obc_1000m)) return else: print "<<< {}".format(l1_1000m) print "<<< {}".format(obc_1000m) ymd = _get_ymd(l1_1000m) # 获取 coefficient 水色波段系统定标系数, 2013年以前和2013年以后不同 coeff_file = os.path.join(coeff_path, '{}.txt'.format(ymd[0:4])) if not os.path.isfile(coeff_file): log.error("File is not exist: {}".format(coeff_file)) return else: print "<<< {}".format(coeff_file) # 获取输出文件 out_path = pb_io.path_replace_ymd(out_path, ymd) _name = os.path.basename(l1_1000m) out_file = os.path.join(out_path, _name) # 如果输出文件已经存在,跳过预处理 if os.path.isfile(out_file): print "File is already exist, skip it: {}".format(out_file) return # 初始化一个预处理实例 calibrate = CalibrateFY3B(l1_1000m=l1_1000m, obc_1000m=obc_1000m, coeff_file=coeff_file, out_file=out_file, launch_date=launch_date) # 对 OBC 文件进行 SV 提取 calibrate.obc_sv_extract_fy3b(probe=probe, probe_count=probe_count, slide_step=slide_step) # 重新定标 L1 数据 calibrate.calibrate() # 将新数据写入 HDF5 文件 calibrate.write() if not calibrate.error: print ">>> {}".format(calibrate.out_file) # 对原数据和处理后的数据各出一张真彩图 if plot == "on": if not calibrate.error: picture_suffix = "650_565_490" file_name = os.path.splitext(out_file)[0] out_pic_old = "{}_{}_old.{}".format( file_name, picture_suffix, "png") out_pic_new = "{}_{}_new.{}".format( file_name, picture_suffix, "png") # 如果输出文件已经存在,跳过 if os.path.isfile(out_pic_old): print "File is already exist, skip it: {}".format( out_pic_old) return else: _plot_rgb_fy3b_old(in_file, out_pic_old) if os.path.isfile(out_pic_new): print "File is already exist, skip it: {}".format( out_pic_new) return else: _plot_rgb_fy3b_new(out_file, out_pic_new) else: print "Error: Calibrate error".format(in_file) elif "fy3d" in sat_sensor.lower() and 'mersi' in sat_sensor.lower(): # 获取 M1000 文件和对应 GEO 文件 l1_1000m = in_file geo_1000m = _get_geo_file(l1_1000m, geo_path) if not os.path.isfile(l1_1000m): log.error("File is not exist: {}".format(l1_1000m)) return elif not os.path.isfile(geo_1000m): log.error("File is not exist: {}".format(geo_1000m)) return else: print "<<< {}".format(l1_1000m) print "<<< {}".format(geo_1000m) ymd = _get_ymd(l1_1000m) # 获取 coefficient 定标系数 coeff_file = os.path.join(coeff_path, '{}.txt'.format(ymd[0:4])) if not os.path.isfile(coeff_file): log.error("File is not exist: {}".format(coeff_file)) coeff_file = None else: print "<<< {}".format(coeff_file) # 获取输出文件 out_path = pb_io.path_replace_ymd(out_path, ymd) _name = os.path.basename(l1_1000m) out_file = os.path.join(out_path, _name) # 如果输出文件已经存在,跳过预处理 if os.path.isfile(out_file): print "File is already exist, skip it: {}".format(out_file) return # 初始化一个预处理实例 calibrate = CalibrateFY3D() # 重新定标 L1 数据 calibrate.calibrate(l1_1000m, geo_1000m, coeff_file) # 将新数据写入 HDF5 文件 calibrate.write(out_file) if not calibrate.error: print ">>> {}".format(out_file) if plot == "on": if not calibrate.error: picture_suffix = "670_555_490" file_name = os.path.splitext(out_file)[0] out_pic_old = "{}_{}_old.{}".format( file_name, picture_suffix, "png") out_pic_new = "{}_{}_new.{}".format( file_name, picture_suffix, "png") # 如果输出文件已经存在,跳过 if os.path.isfile(out_pic_old): print "File is already exist, skip it: {}".format( out_pic_old) return else: pass # _plot_rgb_fy3b_old(in_file, out_pic_old) if os.path.isfile(out_pic_new): print "File is already exist, skip it: {}".format( out_pic_new) return else: print out_file _plot_rgb_fy3d_new(out_file, out_pic_new) else: print "***Error***: Calibrate error".format(in_file) else: print "***Error***: Don`t handle this sat and sensor: {}".format( sat_sensor) return print '-' * 100