def job_0710(job_exe, sat_pair, date_s, date_e, job_id): ''' :月合成 ''' Log.info(u'%s: %s 月合成处理开始...' % (job_id, job_exe)) cfg_path = cfg_body['PATH']['MID']['incfg'] daily_path = cfg_body['PATH']['OUT']['daily'] month_path = cfg_body['PATH']['OUT']['monthly'] # 清理配置 if os.path.isdir(cfg_path): shutil.rmtree(cfg_path) arg_list = [] # 月首和月末 调整 ymd1 = date_s.strftime('%Y%m%d') ymd2 = date_e.strftime('%Y%m%d') lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1] date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d') date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d') while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') # 输出 com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAM_5000M.HDF' % ( cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd) month_path_use = pb_io.path_replace_ymd(month_path, ymd) com_out_file = os.path.join(month_path_use, com_filename) # 输入 reg = '.*_%s.*.HDF' % ymd[0:6] daily_path_use = pb_io.path_replace_ymd(daily_path, ymd) data_list_use = pb_io.find_file(daily_path_use, reg) if len(data_list_use) > 0: com_dict = { 'PATH': { 'ipath': data_list_use, 'opath': com_out_file } } cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) date_s = date_s + relativedelta(months=1) reg = '.*.yaml' FileLst = pb_io.find_file(cfg_path, reg) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0610(job_exe, sat_pair, date_s, date_e, job_id): ''' :日合成 ''' Log.info(u'%s: %s 日合成处理开始...' % (job_id, job_exe)) granule_path = cfg_body['PATH']['MID']['granule'] proj_path = cfg_body['PATH']['MID']['projection'] cfg_path = cfg_body['PATH']['MID']['incfg'] daily_path = cfg_body['PATH']['OUT']['daily'] # 清理配置 if os.path.isdir(cfg_path): shutil.rmtree(cfg_path) arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % ( cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd) daily_path_use = pb_io.path_replace_ymd(daily_path, ymd) com_out_file = os.path.join(daily_path_use, com_filename) granule_path_use = pb_io.path_replace_ymd(granule_path, ymd) reg = '.*_%s_.*.HDF' % ymd granule_lst = pb_io.find_file(granule_path_use, reg) # reg = '.*_%s_.*.HDF' % ymd proj_path_use = pb_io.path_replace_ymd(proj_path, ymd) granule_pro_lst = pb_io.find_file(proj_path_use, reg) if len(granule_pro_lst) > 0: com_dict = { 'PATH': { 'ipath': granule_lst, 'ppath': granule_pro_lst, 'opath': com_out_file } } cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) date_s = date_s + relativedelta(days=1) reg = '.*.yaml' FileLst = pb_io.find_file(cfg_path, reg) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0311(job_exe, sat_pair, date_s, date_e, job_id): ''' :水色反演 ''' Log.info(u'%s: %s 水色反演处理开始...' % (job_id, job_exe)) # 去掉路径信息 in_path = cfg_body['PATH']['MID']['granule'] out_path = in_path arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_out_path = pb_io.path_replace_ymd(out_path, ymd) if not os.path.isdir(use_out_path): os.makedirs(use_out_path) use_reg = '.*_%s_.*.HDF' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = 'idl -rt=%s -args %s %s/' % (job_exe, in_file, use_out_path) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0310(job_exe, sat_pair, date_s, date_e, job_id): ''' :反演 ''' Log.info(u'%s: %s 反演预处理开始...' % (job_id, job_exe)) cfg = 'aerosol.cfg' # 去掉路径信息 job_exe = os.path.basename(job_exe) in_path = cfg_body['PATH']['MID']['calibrate'] reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = '.*_%s_.*.HDF' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = './%s %s %s ' % (job_exe, in_file, cfg) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0711(job_exe, sat_pair, date_s, date_e, job_id): ''' :月成后文件绘图 ''' Log.info(u'%s: %s 月合成出图处理开始...' % (job_id, job_exe)) ipath = cfg_body['PATH']['OUT']['monthly'] FileLst = [] arg_list = [] # 月首和月末 调整 ymd1 = date_s.strftime('%Y%m%d') ymd2 = date_e.strftime('%Y%m%d') lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1] date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d') date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d') while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') path_use = pb_io.path_replace_ymd(ipath, ymd) reg = '.*_%s_.*.HDF' % ymd mlist = pb_io.find_file(path_use, reg) FileLst.extend(mlist) date_s = date_s + relativedelta(months=1) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0810(job_exe, sat_pair, date_s, date_e, job_id): ''' :年合成 ''' Log.info(u'%s: %s 年合成处理开始...' % (job_id, job_exe)) cfg_path = cfg_body['PATH']['MID']['incfg'] month_path = cfg_body['PATH']['OUT']['monthly'] yearly_path = cfg_body['PATH']['OUT']['yearly'] # 清理配置 if os.path.isdir(cfg_path): shutil.rmtree(cfg_path) arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') # 输出 com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_YEAR_5000M.HDF' % ( cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd) yearly_path_use = pb_io.path_replace_ymd(yearly_path, ymd) com_out_file = os.path.join(yearly_path_use, com_filename) # 输入 reg = '.*_%s.*.HDF' % ymd[0:4] month_path_use = pb_io.path_replace_ymd(month_path, ymd) data_list_use = pb_io.find_file(month_path_use, reg) if len(data_list_use) > 0: com_dict = { 'PATH': { 'ipath': data_list_use, 'opath': com_out_file } } cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) date_s = date_s + relativedelta(years=1) reg = '.*.yaml' FileLst = pb_io.find_file(cfg_path, reg) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0912(job_exe, sat_pair, date_s, date_e, job_id): Log.info(u'%s: %s occ检验日合成...' % (job_id, job_exe)) # 解析mathcing: FY3A+MERSI_AQUA+MODIS_check ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] jobcfg_path = cfg_body['PATH']['IN']['jobCfg'] match_path = cfg_body['PATH']['MID']['match'] # 存放分发列表 arg_list = [] all_file_list = [] stime = date_s.strftime('%Y%m%d') etime = date_e.strftime('%Y%m%d') while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_match_path = os.path.join(match_path, sat_pair, ymd) use_path_out = os.path.join(match_path, sat_pair) reg1 = '.*_%s.*.HDF' % ymd print use_match_path file_list = pb_io.find_file(use_match_path, reg1) date_s = date_s + relativedelta(days=1) dict = { 'INFO': { 'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'pair': sat_pair, 'ymd': ymd }, 'PATH': { 'opath': use_path_out, 'ipath': file_list } } if len(file_list) > 0: cfgFile = os.path.join(jobcfg_path, sat_pair, job_id, '%s.yaml' % (ymd)) CreateYamlCfg(dict, cfgFile) cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile) arg_list.append(cmd) return arg_list
def get_cmd_list(job_exe, job_name, job_id, date_s, date_e, g_path_interface): cmd_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') date_s = date_s + relativedelta(days=1) path_yaml = os.path.join(g_path_interface, job_name, job_id, ymd) if os.path.isdir(path_yaml): file_list_yaml = pb_io.find_file(path_yaml, '.*.yaml') for file_yaml in file_list_yaml: cmd = '%s %s %s' % (python, job_exe, file_yaml) cmd_list.append(cmd) return cmd_list
def job_0213(job_exe, sat_pair, date_s, date_e, job_id): Log.info(u'%s: %s 交叉匹配结果绘图处理开始...' % (job_id, job_exe)) # 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] cfg_path = cfg_body['PATH']['MID']['incfg'] match_path = cfg_body['PATH']['MID']['match'] # 清理配置 if os.path.isdir(cfg_path): shutil.rmtree(cfg_path) # 存放分发列表 arg_list = [] all_file_list = [] stime = date_s.strftime('%Y%m%d') etime = date_e.strftime('%Y%m%d') while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_match_path = os.path.join(match_path, sat_pair, ymd[0:6]) reg1 = '.*_%s.*.H5' % ymd print use_match_path file_list = pb_io.find_file(use_match_path, reg1) all_file_list.extend(file_list) date_s = date_s + relativedelta(days=1) ofile_yaml = os.path.join(cfg_path, '%s_%s.yaml' % (stime, etime)) opath = os.path.join( match_path, '%s_map' % sat_pair, '%s_%s' % (stime, etime)) if not os.path.isdir(opath): os.makedirs(opath) dict = {'INFO': {'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd_s': stime, 'ymd_e': etime}, 'PATH': {'opath': opath, 'ipath': all_file_list}} if len(all_file_list) > 0: Log.info('%s %s-%s create collocation cfg success' % (sat_pair, stime, etime)) CreateYamlCfg(dict, ofile_yaml) cmd = '%s %s %s %s' % (python, job_exe, sat_pair, ofile_yaml) arg_list.append(cmd) return arg_list
def job_0110(job_exe, sat_pair, date_s, date_e, job_id): """ ncep处理的输入接口 """ Log.info(u'%s: %s ncep处理开始...' % (job_id, job_exe)) in_path = cfg_body['PATH']['IN']['ncep'] reg = 'fnl_%s_.*' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = reg % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0210(job_exe, sat_pair, date_s, date_e, job_id): """ L1数据预处理的输入接口 """ Log.info(u'%s: %s L1数据预处理开始...' % (job_id, job_exe)) in_path = cfg_body['PATH']['IN']['l1'] # reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = '.*_%s_.*.HDF$' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0611(job_exe, sat_pair, date_s, date_e, job_id): ''' :合成后文件绘图 ''' Log.info(u'%s: %s 日合成出图处理开始...' % (job_id, job_exe)) ipath = cfg_body['PATH']['OUT']['daily'] FileLst = [] arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') path_use = pb_io.path_replace_ymd(ipath, ymd) reg = '.*_%s_.*.HDF' % ymd dlist = pb_io.find_file(path_use, reg) FileLst.extend(dlist) date_s = date_s + relativedelta(days=1) for in_file in FileLst: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) return arg_list
def job_0410(job_exe, sat_pair, date_s, date_e, job_id): ''' :反演后的快视图 ''' Log.info(u'%s: %s 反演轨道产品快视图处理开始...' % (job_id, job_exe)) in_path = cfg_body['PATH']['MID']['granule'] # reg = 'FY3[A-Z]_MERSI_ORBT_L2_\w{3}_MLT_NUL_%s_(\d{4})_1000M.HDF' arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') use_in_path = pb_io.path_replace_ymd(in_path, ymd) use_reg = '.*_%s_.*.HDF' % ymd file_list = pb_io.find_file(use_in_path, use_reg) for in_file in file_list: cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file) arg_list.append(cmd_list) date_s = date_s + relativedelta(days=1) return arg_list
def job_0211(job_exe, sat_pair, date_s, date_e, job_id, reload=None): if reload is None: Log.info(u'%s: %s 交叉匹配(use pre data)处理开始...' % (job_id, job_exe)) else: print 'reload' # 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] DATA_DIR = cfg_body['PATH']['IN']['data'] CALI_DIR = cfg_body['PATH']['MID']['calibrate'] jobCfg = cfg_body['PATH']['IN']['jobCfg'] match_path = cfg_body['PATH']['MID']['match'] L1_data_dir = cfg_body['PATH']['IN']['l1'] coeff_path = cfg_body['PATH']['IN']['coeff'] # 存放分发列表 arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') jjj = date_s.strftime('%j') coeff_file = os.path.join(coeff_path, '%s.txt' % ymd[:4]) print ymd # 存放俩颗卫星的原始数据目录位置 if reload is None: # inpath1 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' % # (sat1, sensor1), ymd[:6]) FINAL_DIR = pb_io.path_replace_ymd(CALI_DIR, ymd) inpath1 = os.path.join(FINAL_DIR) else: inpath1 = pb_io.path_replace_ymd(L1_data_dir, ymd) inpath2 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' % ( sat2, sensor2, ), ymd[:6]) print "inpath1", inpath1 print "inpath2", inpath2 sat11 = cfg_body['SAT_S2L'][sat1] sat22 = cfg_body['SAT_S2L'][sat2] # 读取交叉点上的俩颗卫星的交叉时间,1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间 timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd) print 'cross', len(timeList) reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd reg2 = 'MYD021KM.A%s%s.*.hdf' % (ymd[0:4], jjj) file_list1 = pb_io.find_file(inpath1, reg1) file_list2 = pb_io.find_file(inpath2, reg2) # 根据交叉点时间,找到数据列表中需要的数据 select File for crossTime in timeList: Lat = crossTime[0] Lon = crossTime[1] ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S') s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1)) e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1)) s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2)) e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2)) # 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据 list1 = Find_data_FromCrossTime(file_list1, s_cross_time1, e_cross_time1) list2 = Find_data_FromCrossTime(file_list2, s_cross_time2, e_cross_time2) print 'fy', len(list1) print 'mo', len(list2) # 存放匹配信息的yaml配置文件存放位置 yaml_file3 = os.path.join( jobCfg, sat_pair, job_id, ymdhms[:8], '%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2)) filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms) # 输出完整路径 # FY3B+MERSI_AQUA+MODIS_L1 和 FY3B+MERSI_AQUA+MODIS 两种支持 wangpeng # add 2018-09-14 full_filename3 = os.path.join(match_path, sat_pair, ymdhms[:6], filename3) # if reload is None: # full_filename3 = os.path.join( # match_path, sat_pair, ymdhms[:6], filename3) # else: # full_filename3 = os.path.join( # match_path, sat_pair + '_L1', ymdhms[:6], filename3) # 投影参数 cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % ( Lat, Lon) if len(list1) > 0 and len(list2) > 0: print '111111' row = 128 col = 128 res = 8000 dict3 = { 'INFO': { 'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd': ymdhms }, 'PATH': { 'opath': full_filename3, 'ipath1': list1, 'ipath2': list2, 'ipath_coeff': coeff_file }, 'PROJ': { 'cmd': cmd, 'row': row, 'col': col, 'res': res } } Log.info('%s %s create collocation cfg success' % (sat_pair, ymdhms)) CreateYamlCfg(dict3, yaml_file3) if reload is None: cmd = '%s %s %s 0' % (python, job_exe, yaml_file3) else: cmd = '%s %s %s 1' % (python, job_exe, yaml_file3) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def job_0215(job_exe, sat_pair, date_s, date_e, job_id): Log.info(u'%s: %s 水色L2产品检验处理开始...' % (job_id, job_exe)) # 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] in_path1 = cfg_body['PATH']['MID']['granule'] in_path2 = cfg_body['PATH']['IN']['data'] out_path_cfg = cfg_body['PATH']['IN']['jobCfg'] out_path_match = cfg_body['PATH']['MID']['match'] # 存放分发列表 arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') jjj = date_s.strftime('%j') # 存放俩颗卫星的原始数据目录位置 full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd) full_in_path2 = os.path.join(in_path2, '%s/%s/L2/ORBIT' % ( sat2, sensor2, ), ymd[:6]) print "inpath1", full_in_path1 print "inpath2", full_in_path2 sat11 = cfg_body['SAT_S2L'][sat1] sat22 = cfg_body['SAT_S2L'][sat2] # 读取交叉点上的俩颗卫星的交叉时间,1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间 timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd) print 'cross', len(timeList) reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd reg2 = 'A%s%s.*.nc' % (ymd[0:4], jjj) file_list1 = pb_io.find_file(full_in_path1, reg1) file_list2 = pb_io.find_file(full_in_path2, reg2) # 根据交叉点时间,找到数据列表中需要的数据 select File for crossTime in timeList: Lat = crossTime[0] Lon = crossTime[1] ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S') s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1)) e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1)) s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2)) e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2)) # 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据 list1 = Find_data_FromCrossTime(file_list1, s_cross_time1, e_cross_time1) list2 = Find_data_FromCrossTime(file_list2, s_cross_time2, e_cross_time2) print 'fy', len(list1) print 'mo', len(list2) # 存放匹配信息的yaml配置文件存放位置 yaml_file3 = os.path.join( out_path_cfg, sat_pair, job_id, ymdhms[:8], '%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2)) filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms) # 输出完整路径 full_filename3 = os.path.join(out_path_match, sat_pair, ymdhms[:6], filename3) # 投影参数 cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % ( Lat, Lon) if len(list1) > 0 and len(list2) > 0: print '111111' row = 128 col = 128 res = 8000 dict3 = { 'INFO': { 'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd': ymdhms, 'pair': sat_pair }, 'PATH': { 'opath': full_filename3, 'ipath1': list1, 'ipath2': list2 }, 'PROJ': { 'cmd': cmd, 'row': row, 'col': col, 'res': res } } Log.info('%s %s create collocation cfg success' % (sat_pair, ymdhms)) CreateYamlCfg(dict3, yaml_file3) cmd = '%s %s %s' % (python, job_exe, yaml_file3) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def job_0911(job_exe, sat_pair, date_s, date_e, job_id): """ 基于aqua(modis)和fy3b(mersi)的l3数据匹配结果 """ # date_s, date_e = time.split("-") # date_s = datetime.strptime(date_s, '%Y%m%d%H%M') # date_e = datetime.strptime(date_e, '%Y%m%d%H%M') Log.info(u'%s: %s 检验l3产品 开始...' % (job_id, job_exe)) reg_fy3b = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' reg_aqua_chl1 = 'L3m_%s__GLOB_4_GSM-MOD_%s_DAY_00.nc' reg_aqua = 'L3m_%s__GLOB_4_AV-MOD_%s_DAY_00.nc' fy3b_l3 = cfg_body['PATH']['OUT']['daily'] # fy3b_l3 apth modis_l3 = cfg_body['PATH']['IN']['modis_l3'] # modis_l3 jobcfg_path = cfg_body['PATH']['IN']['jobCfg'] out_path = cfg_body['PATH']['MID']['match'] arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') # 查找fy3b_mersi文件 match_fy3b_file = [] fy3b_l3_path_use = pb_io.path_replace_ymd(fy3b_l3, ymd) file_fy3b = pb_io.find_file(fy3b_l3_path_use, reg_fy3b % (ymd)) match_fy3b_file.extend(file_fy3b) # 检查是否已经生成产品 pruc_names = ['KD490', 'CHL1', 'POC', 'ZSD'] match_aqua = [] for pruc_name in pruc_names: # outfile = out_path + pruc_name + os.sep + \ # ymd + '_' + pruc_name + "_5000.hdf" # if os.path.isfile(outfile): # print u'产品已经存在' # pass # else: # 查找对应的aqua_modis产品文件 if pruc_name != 'CHL1': file_aqua = pb_io.find_file(modis_l3, reg_aqua % (ymd, pruc_name)) match_aqua.extend(file_aqua) else: file_aqua = pb_io.find_file(modis_l3, reg_aqua % (ymd, pruc_name)) if len(file_aqua) > 0: match_aqua.extend(file_aqua) else: file_aqua1 = pb_io.find_file( modis_l3, reg_aqua_chl1 % (ymd, pruc_name)) match_aqua.extend(file_aqua1) # 判断当前五分钟的数据是否都存在 if len(match_fy3b_file) > 0 and len(match_aqua) > 0: com_dict = { 'PATH': { 'ipath1': match_fy3b_file, 'ipath2': match_aqua, 'out_path': out_path }, 'INFO': { 'time': ymd, 'pair': sat_pair }, } cfgFile = os.path.join(jobcfg_path, sat_pair, job_id, '%s.yaml' % ymd) CreateYamlCfg(com_dict, cfgFile) # check_fy_aqual2 = CheckFyAqual2() # check_fy_aqual2.main( # match_fy3b_file, match_iop_file, match_oc_file, ymd_HM, out_path) cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def job_0910(job_exe, sat_pair, date_s, date_e, job_id): """ 基于aqua(modis)和fy3b(mersi)的l2数据匹配结果 """ # date_s, date_e = time.split("-") # date_s = datetime.strptime(date_s, '%Y%m%d%H%M') # date_e = datetime.strptime(date_e, '%Y%m%d%H%M') Log.info(u'%s: %s 检验l2产品 开始...' % (job_id, job_exe)) # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] reg_fy3b = 'FY3B_MERSI_ORBT_L2_OCC_MLT_NUL_.*._1000M.HDF' reg_aqua_iop = 'A.*.L2_LAC_IOP.nc' reg_aqua_oc = 'A.*.L2_LAC_OC.nc' fy3b_l2 = cfg_body['PATH']['MID']['granule'] # fy3b_l2 apth modis_l2 = cfg_body['PATH']['IN']['data'] # modis_l2 jobcfg_path = cfg_body['PATH']['IN']['jobCfg'] out_path = cfg_body['PATH']['MID']['match'] arg_list = [] while date_s <= date_e: # ymd_HM = date_s.strftime('%Y%m%d_%H%M') ymd = date_s.strftime('%Y%m%d') # 解析mathcing: FY3A+MERSI_AQUA+MODIS FY3B+MERSI_AQUA+MODIS_check sat1 = (sat_pair.split('_')[0]).split('+')[0] sat2 = (sat_pair.split('_')[1]).split('+')[0] fy3b_l2_path_use = pb_io.path_replace_ymd(fy3b_l2, ymd) modis_l2_path_use = modis_l2 + '/AQUA/MODIS/L2/ORBIT' + \ os.sep + date_s.strftime('%Y%m') file_list1 = pb_io.find_file(fy3b_l2_path_use, reg_fy3b) file_list2 = pb_io.find_file(modis_l2_path_use, reg_aqua_iop) # file_list3 = pb_io.find_file(modis_l2_path_use, reg_aqua_oc) timeList = ReadCrossFile_LEO_LEO('FENGYUN-3B', 'AQUA', ymd) # print timeList, CROSS_DIR, sat1, sat2, ymd for crossTime in timeList: ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S') s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1)) e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1)) s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2)) e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2)) # 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据 match_fy3b_file = Find_data_fy_FromCrossTime( file_list1, s_cross_time1, e_cross_time1) match_iop_file = Find_data_aqua_FromCrossTime( file_list2, s_cross_time2, e_cross_time2) # match_oc_file = Find_data_aqua_FromCrossTime( # file_list3, s_cross_time2, e_cross_time2) # 判断当前五分钟的数据是否都存在 if len(match_fy3b_file) > 0 and len(match_iop_file) > 0: for match_fy3b in match_fy3b_file: for match_iop in match_iop_file: com_dict = { 'PATH': { 'ipath1': [match_fy3b], 'ipath2': [match_iop], 'out_path': out_path }, 'INFO': { 'time': ymdhms, 'pair': sat_pair, 'lat': crossTime[0], 'lon': crossTime[1] } } cfgFile = os.path.join(jobcfg_path, sat_pair, job_id, '%s.yaml' % ymdhms) CreateYamlCfg(com_dict, cfgFile) # check_fy_aqual2 = CheckFyAqual2() # check_fy_aqual2.main( # match_fy3b_file, match_iop_file, match_oc_file, ymd_HM, # out_path) cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list
def job_0216(job_exe, sat_pair, date_s, date_e, job_id): Log.info(u'%s: %s 水色L3产品检验处理开始...' % (job_id, job_exe)) # 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名 sat1 = (sat_pair.split('_')[0]).split('+')[0] sensor1 = (sat_pair.split('_')[0]).split('+')[1] sat2 = (sat_pair.split('_')[1]).split('+')[0] sensor2 = (sat_pair.split('_')[1]).split('+')[1] # 解析global.cfg中的信息 sec1 = cfg_body['PAIRS'][sat_pair]['sec1'] sec2 = cfg_body['PAIRS'][sat_pair]['sec2'] in_path1 = cfg_body['PATH']['OUT']['daily'] in_path2 = cfg_body['PATH']['IN']['data'] out_path_cfg = cfg_body['PATH']['IN']['jobCfg'] out_path_match = cfg_body['PATH']['MID']['match'] # 存放分发列表 arg_list = [] while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') jjj = date_s.strftime('%j') # 存放俩颗卫星的原始数据目录位置 full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd) full_in_path2 = os.path.join(in_path2, '%s/%s/L3' % (sat2, sensor2)) print "inpath1", full_in_path1 print "inpath2", full_in_path2 sat11 = cfg_body['SAT_S2L'][sat1] sat22 = cfg_body['SAT_S2L'][sat2] # 读取交叉点上的俩颗卫星的交叉时间,1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间 reg1 = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % ymd reg2 = 'L3m_%s__GLOB_4_AV-MOD_KD490_DAY_00.nc' % (ymd) file_list1 = pb_io.find_file(full_in_path1, reg1) file_list2 = pb_io.find_file(full_in_path2, reg2) # 存放匹配信息的yaml配置文件存放位置 yaml_file3 = os.path.join(out_path_cfg, sat_pair, job_id, ymd, '%s_%s_%s.yaml' % (ymd, sensor1, sensor2)) filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymd) # 输出完整路径 full_filename3 = os.path.join(out_path_match, sat_pair, ymd[:6], filename3) # 投影参数 deg = 0.05 res = deg2meter(deg) half_res = deg2meter(res) / 2. cmd = '+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=-%f +y_0=%f +datum=WGS84' % ( half_res, half_res) if len(file_list1) > 0 and len(file_list2) > 0: print '111111' dict3 = { 'INFO': { 'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd': ymd, 'pair': sat_pair }, 'PATH': { 'opath': full_filename3, 'ipath1': file_list1, 'ipath2': file_list2 }, 'PROJ': { 'cmd': cmd, 'res': deg } } Log.info('%s %s create collocation cfg success' % (sat_pair, ymd)) CreateYamlCfg(dict3, yaml_file3) cmd = '%s %s %s' % (python, job_exe, yaml_file3) arg_list.append(cmd) date_s = date_s + relativedelta(days=1) return arg_list