def job_0710(job_exe, sat_pair, date_s, date_e, job_id):
'''
:月合成
'''
Log.info(u'%s: %s 月合成处理开始...' % (job_id, job_exe))
cfg_path = cfg_body['PATH']['MID']['incfg']
daily_path = cfg_body['PATH']['OUT']['daily']
month_path = cfg_body['PATH']['OUT']['monthly']
# 清理配置
if os.path.isdir(cfg_path):
shutil.rmtree(cfg_path)
arg_list = []
# 月首和月末 调整
ymd1 = date_s.strftime('%Y%m%d')
ymd2 = date_e.strftime('%Y%m%d')
lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1]
date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d')
date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d')
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
# 输出
com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAM_5000M.HDF' % (
cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd)
month_path_use = pb_io.path_replace_ymd(month_path, ymd)
com_out_file = os.path.join(month_path_use, com_filename)
# 输入
reg = '.*_%s.*.HDF' % ymd[0:6]
daily_path_use = pb_io.path_replace_ymd(daily_path, ymd)
data_list_use = pb_io.find_file(daily_path_use, reg)
if len(data_list_use) > 0:
com_dict = {
'PATH': {
'ipath': data_list_use,
'opath': com_out_file
}
}
cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
date_s = date_s + relativedelta(months=1)
reg = '.*.yaml'
FileLst = pb_io.find_file(cfg_path, reg)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0610(job_exe, sat_pair, date_s, date_e, job_id):
'''
:日合成
'''
Log.info(u'%s: %s 日合成处理开始...' % (job_id, job_exe))
granule_path = cfg_body['PATH']['MID']['granule']
proj_path = cfg_body['PATH']['MID']['projection']
cfg_path = cfg_body['PATH']['MID']['incfg']
daily_path = cfg_body['PATH']['OUT']['daily']
# 清理配置
if os.path.isdir(cfg_path):
shutil.rmtree(cfg_path)
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % (
cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd)
daily_path_use = pb_io.path_replace_ymd(daily_path, ymd)
com_out_file = os.path.join(daily_path_use, com_filename)
granule_path_use = pb_io.path_replace_ymd(granule_path, ymd)
reg = '.*_%s_.*.HDF' % ymd
granule_lst = pb_io.find_file(granule_path_use, reg)
# reg = '.*_%s_.*.HDF' % ymd
proj_path_use = pb_io.path_replace_ymd(proj_path, ymd)
granule_pro_lst = pb_io.find_file(proj_path_use, reg)
if len(granule_pro_lst) > 0:
com_dict = {
'PATH': {
'ipath': granule_lst,
'ppath': granule_pro_lst,
'opath': com_out_file
}
}
cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
date_s = date_s + relativedelta(days=1)
reg = '.*.yaml'
FileLst = pb_io.find_file(cfg_path, reg)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0311(job_exe, sat_pair, date_s, date_e, job_id):
'''
:水色反演
'''
Log.info(u'%s: %s 水色反演处理开始...' % (job_id, job_exe))
# 去掉路径信息
in_path = cfg_body['PATH']['MID']['granule']
out_path = in_path
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_out_path = pb_io.path_replace_ymd(out_path, ymd)
if not os.path.isdir(use_out_path):
os.makedirs(use_out_path)
use_reg = '.*_%s_.*.HDF' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = 'idl -rt=%s -args %s %s/' % (job_exe, in_file,
use_out_path)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0310(job_exe, sat_pair, date_s, date_e, job_id):
'''
:反演
'''
Log.info(u'%s: %s 反演预处理开始...' % (job_id, job_exe))
cfg = 'aerosol.cfg'
# 去掉路径信息
job_exe = os.path.basename(job_exe)
in_path = cfg_body['PATH']['MID']['calibrate']
reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = '.*_%s_.*.HDF' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = './%s %s %s ' % (job_exe, in_file, cfg)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0711(job_exe, sat_pair, date_s, date_e, job_id):
'''
:月成后文件绘图
'''
Log.info(u'%s: %s 月合成出图处理开始...' % (job_id, job_exe))
ipath = cfg_body['PATH']['OUT']['monthly']
FileLst = []
arg_list = []
# 月首和月末 调整
ymd1 = date_s.strftime('%Y%m%d')
ymd2 = date_e.strftime('%Y%m%d')
lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1]
date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d')
date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d')
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
path_use = pb_io.path_replace_ymd(ipath, ymd)
reg = '.*_%s_.*.HDF' % ymd
mlist = pb_io.find_file(path_use, reg)
FileLst.extend(mlist)
date_s = date_s + relativedelta(months=1)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0810(job_exe, sat_pair, date_s, date_e, job_id):
'''
:年合成
'''
Log.info(u'%s: %s 年合成处理开始...' % (job_id, job_exe))
cfg_path = cfg_body['PATH']['MID']['incfg']
month_path = cfg_body['PATH']['OUT']['monthly']
yearly_path = cfg_body['PATH']['OUT']['yearly']
# 清理配置
if os.path.isdir(cfg_path):
shutil.rmtree(cfg_path)
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
# 输出
com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_YEAR_5000M.HDF' % (
cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd)
yearly_path_use = pb_io.path_replace_ymd(yearly_path, ymd)
com_out_file = os.path.join(yearly_path_use, com_filename)
# 输入
reg = '.*_%s.*.HDF' % ymd[0:4]
month_path_use = pb_io.path_replace_ymd(month_path, ymd)
data_list_use = pb_io.find_file(month_path_use, reg)
if len(data_list_use) > 0:
com_dict = {
'PATH': {
'ipath': data_list_use,
'opath': com_out_file
}
}
cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
date_s = date_s + relativedelta(years=1)
reg = '.*.yaml'
FileLst = pb_io.find_file(cfg_path, reg)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0912(job_exe, sat_pair, date_s, date_e, job_id):
Log.info(u'%s: %s occ检验日合成...' % (job_id, job_exe))
# 解析mathcing: FY3A+MERSI_AQUA+MODIS_check ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
jobcfg_path = cfg_body['PATH']['IN']['jobCfg']
match_path = cfg_body['PATH']['MID']['match']
# 存放分发列表
arg_list = []
all_file_list = []
stime = date_s.strftime('%Y%m%d')
etime = date_e.strftime('%Y%m%d')
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_match_path = os.path.join(match_path, sat_pair, ymd)
use_path_out = os.path.join(match_path, sat_pair)
reg1 = '.*_%s.*.HDF' % ymd
print use_match_path
file_list = pb_io.find_file(use_match_path, reg1)
date_s = date_s + relativedelta(days=1)
dict = {
'INFO': {
'sat1': sat1,
'sensor1': sensor1,
'sat2': sat2,
'sensor2': sensor2,
'pair': sat_pair,
'ymd': ymd
},
'PATH': {
'opath': use_path_out,
'ipath': file_list
}
}
if len(file_list) > 0:
cfgFile = os.path.join(jobcfg_path, sat_pair, job_id,
'%s.yaml' % (ymd))
CreateYamlCfg(dict, cfgFile)
cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile)
arg_list.append(cmd)
return arg_list
def get_cmd_list(job_exe, job_name, job_id, date_s, date_e, g_path_interface):
cmd_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
date_s = date_s + relativedelta(days=1)
path_yaml = os.path.join(g_path_interface, job_name, job_id, ymd)
if os.path.isdir(path_yaml):
file_list_yaml = pb_io.find_file(path_yaml, '.*.yaml')
for file_yaml in file_list_yaml:
cmd = '%s %s %s' % (python, job_exe, file_yaml)
cmd_list.append(cmd)
return cmd_list
def job_0213(job_exe, sat_pair, date_s, date_e, job_id):
Log.info(u'%s: %s 交叉匹配结果绘图处理开始...' % (job_id, job_exe))
# 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
cfg_path = cfg_body['PATH']['MID']['incfg']
match_path = cfg_body['PATH']['MID']['match']
# 清理配置
if os.path.isdir(cfg_path):
shutil.rmtree(cfg_path)
# 存放分发列表
arg_list = []
all_file_list = []
stime = date_s.strftime('%Y%m%d')
etime = date_e.strftime('%Y%m%d')
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_match_path = os.path.join(match_path, sat_pair, ymd[0:6])
reg1 = '.*_%s.*.H5' % ymd
print use_match_path
file_list = pb_io.find_file(use_match_path, reg1)
all_file_list.extend(file_list)
date_s = date_s + relativedelta(days=1)
ofile_yaml = os.path.join(cfg_path, '%s_%s.yaml' % (stime, etime))
opath = os.path.join(
match_path, '%s_map' % sat_pair, '%s_%s' % (stime, etime))
if not os.path.isdir(opath):
os.makedirs(opath)
dict = {'INFO': {'sat1': sat1, 'sensor1': sensor1, 'sat2': sat2, 'sensor2': sensor2, 'ymd_s': stime, 'ymd_e': etime},
'PATH': {'opath': opath, 'ipath': all_file_list}}
if len(all_file_list) > 0:
Log.info('%s %s-%s create collocation cfg success' %
(sat_pair, stime, etime))
CreateYamlCfg(dict, ofile_yaml)
cmd = '%s %s %s %s' % (python, job_exe, sat_pair, ofile_yaml)
arg_list.append(cmd)
return arg_list
def job_0110(job_exe, sat_pair, date_s, date_e, job_id):
"""
ncep处理的输入接口
"""
Log.info(u'%s: %s ncep处理开始...' % (job_id, job_exe))
in_path = cfg_body['PATH']['IN']['ncep']
reg = 'fnl_%s_.*'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = reg % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0210(job_exe, sat_pair, date_s, date_e, job_id):
"""
L1数据预处理的输入接口
"""
Log.info(u'%s: %s L1数据预处理开始...' % (job_id, job_exe))
in_path = cfg_body['PATH']['IN']['l1']
# reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = '.*_%s_.*.HDF$' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0611(job_exe, sat_pair, date_s, date_e, job_id):
'''
:合成后文件绘图
'''
Log.info(u'%s: %s 日合成出图处理开始...' % (job_id, job_exe))
ipath = cfg_body['PATH']['OUT']['daily']
FileLst = []
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
path_use = pb_io.path_replace_ymd(ipath, ymd)
reg = '.*_%s_.*.HDF' % ymd
dlist = pb_io.find_file(path_use, reg)
FileLst.extend(dlist)
date_s = date_s + relativedelta(days=1)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0410(job_exe, sat_pair, date_s, date_e, job_id):
'''
:反演后的快视图
'''
Log.info(u'%s: %s 反演轨道产品快视图处理开始...' % (job_id, job_exe))
in_path = cfg_body['PATH']['MID']['granule']
# reg = 'FY3[A-Z]_MERSI_ORBT_L2_\w{3}_MLT_NUL_%s_(\d{4})_1000M.HDF'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = '.*_%s_.*.HDF' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0211(job_exe, sat_pair, date_s, date_e, job_id, reload=None):
if reload is None:
Log.info(u'%s: %s 交叉匹配(use pre data)处理开始...' % (job_id, job_exe))
else:
print 'reload'
# 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
DATA_DIR = cfg_body['PATH']['IN']['data']
CALI_DIR = cfg_body['PATH']['MID']['calibrate']
jobCfg = cfg_body['PATH']['IN']['jobCfg']
match_path = cfg_body['PATH']['MID']['match']
L1_data_dir = cfg_body['PATH']['IN']['l1']
coeff_path = cfg_body['PATH']['IN']['coeff']
# 存放分发列表
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
jjj = date_s.strftime('%j')
coeff_file = os.path.join(coeff_path, '%s.txt' % ymd[:4])
print ymd
# 存放俩颗卫星的原始数据目录位置
if reload is None:
# inpath1 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' %
# (sat1, sensor1), ymd[:6])
FINAL_DIR = pb_io.path_replace_ymd(CALI_DIR, ymd)
inpath1 = os.path.join(FINAL_DIR)
else:
inpath1 = pb_io.path_replace_ymd(L1_data_dir, ymd)
inpath2 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' % (
sat2,
sensor2,
), ymd[:6])
print "inpath1", inpath1
print "inpath2", inpath2
sat11 = cfg_body['SAT_S2L'][sat1]
sat22 = cfg_body['SAT_S2L'][sat2]
# 读取交叉点上的俩颗卫星的交叉时间，1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间
timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd)
print 'cross', len(timeList)
reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd
reg2 = 'MYD021KM.A%s%s.*.hdf' % (ymd[0:4], jjj)
file_list1 = pb_io.find_file(inpath1, reg1)
file_list2 = pb_io.find_file(inpath2, reg2)
# 根据交叉点时间，找到数据列表中需要的数据 select File
for crossTime in timeList:
Lat = crossTime[0]
Lon = crossTime[1]
ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S')
s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1))
e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1))
s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2))
e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2))
# 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据
list1 = Find_data_FromCrossTime(file_list1, s_cross_time1,
e_cross_time1)
list2 = Find_data_FromCrossTime(file_list2, s_cross_time2,
e_cross_time2)
print 'fy', len(list1)
print 'mo', len(list2)
# 存放匹配信息的yaml配置文件存放位置
yaml_file3 = os.path.join(
jobCfg, sat_pair, job_id, ymdhms[:8],
'%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2))
filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms)
# 输出完整路径
# FY3B+MERSI_AQUA+MODIS_L1 和 FY3B+MERSI_AQUA+MODIS 两种支持 wangpeng
# add 2018-09-14
full_filename3 = os.path.join(match_path, sat_pair, ymdhms[:6],
filename3)
# if reload is None:
# full_filename3 = os.path.join(
# match_path, sat_pair, ymdhms[:6], filename3)
# else:
# full_filename3 = os.path.join(
# match_path, sat_pair + '_L1', ymdhms[:6], filename3)
# 投影参数
cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % (
Lat, Lon)
if len(list1) > 0 and len(list2) > 0:
print '111111'
row = 128
col = 128
res = 8000
dict3 = {
'INFO': {
'sat1': sat1,
'sensor1': sensor1,
'sat2': sat2,
'sensor2': sensor2,
'ymd': ymdhms
},
'PATH': {
'opath': full_filename3,
'ipath1': list1,
'ipath2': list2,
'ipath_coeff': coeff_file
},
'PROJ': {
'cmd': cmd,
'row': row,
'col': col,
'res': res
}
}
Log.info('%s %s create collocation cfg success' %
(sat_pair, ymdhms))
CreateYamlCfg(dict3, yaml_file3)
if reload is None:
cmd = '%s %s %s 0' % (python, job_exe, yaml_file3)
else:
cmd = '%s %s %s 1' % (python, job_exe, yaml_file3)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0215(job_exe, sat_pair, date_s, date_e, job_id):
Log.info(u'%s: %s 水色L2产品检验处理开始...' % (job_id, job_exe))
# 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
in_path1 = cfg_body['PATH']['MID']['granule']
in_path2 = cfg_body['PATH']['IN']['data']
out_path_cfg = cfg_body['PATH']['IN']['jobCfg']
out_path_match = cfg_body['PATH']['MID']['match']
# 存放分发列表
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
jjj = date_s.strftime('%j')
# 存放俩颗卫星的原始数据目录位置
full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd)
full_in_path2 = os.path.join(in_path2, '%s/%s/L2/ORBIT' % (
sat2,
sensor2,
), ymd[:6])
print "inpath1", full_in_path1
print "inpath2", full_in_path2
sat11 = cfg_body['SAT_S2L'][sat1]
sat22 = cfg_body['SAT_S2L'][sat2]
# 读取交叉点上的俩颗卫星的交叉时间，1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间
timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd)
print 'cross', len(timeList)
reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd
reg2 = 'A%s%s.*.nc' % (ymd[0:4], jjj)
file_list1 = pb_io.find_file(full_in_path1, reg1)
file_list2 = pb_io.find_file(full_in_path2, reg2)
# 根据交叉点时间，找到数据列表中需要的数据 select File
for crossTime in timeList:
Lat = crossTime[0]
Lon = crossTime[1]
ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S')
s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1))
e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1))
s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2))
e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2))
# 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据
list1 = Find_data_FromCrossTime(file_list1, s_cross_time1,
e_cross_time1)
list2 = Find_data_FromCrossTime(file_list2, s_cross_time2,
e_cross_time2)
print 'fy', len(list1)
print 'mo', len(list2)
# 存放匹配信息的yaml配置文件存放位置
yaml_file3 = os.path.join(
out_path_cfg, sat_pair, job_id, ymdhms[:8],
'%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2))
filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms)
# 输出完整路径
full_filename3 = os.path.join(out_path_match, sat_pair, ymdhms[:6],
filename3)
# 投影参数
cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % (
Lat, Lon)
if len(list1) > 0 and len(list2) > 0:
print '111111'
row = 128
col = 128
res = 8000
dict3 = {
'INFO': {
'sat1': sat1,
'sensor1': sensor1,
'sat2': sat2,
'sensor2': sensor2,
'ymd': ymdhms,
'pair': sat_pair
},
'PATH': {
'opath': full_filename3,
'ipath1': list1,
'ipath2': list2
},
'PROJ': {
'cmd': cmd,
'row': row,
'col': col,
'res': res
}
}
Log.info('%s %s create collocation cfg success' %
(sat_pair, ymdhms))
CreateYamlCfg(dict3, yaml_file3)
cmd = '%s %s %s' % (python, job_exe, yaml_file3)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0911(job_exe, sat_pair, date_s, date_e, job_id):
"""
基于aqua(modis)和fy3b（mersi）的l3数据匹配结果
"""
# date_s, date_e = time.split("-")
# date_s = datetime.strptime(date_s, '%Y%m%d%H%M')
# date_e = datetime.strptime(date_e, '%Y%m%d%H%M')
Log.info(u'%s: %s 检验l3产品 开始...' % (job_id, job_exe))
reg_fy3b = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF'
reg_aqua_chl1 = 'L3m_%s__GLOB_4_GSM-MOD_%s_DAY_00.nc'
reg_aqua = 'L3m_%s__GLOB_4_AV-MOD_%s_DAY_00.nc'
fy3b_l3 = cfg_body['PATH']['OUT']['daily'] # fy3b_l3 apth
modis_l3 = cfg_body['PATH']['IN']['modis_l3'] # modis_l3
jobcfg_path = cfg_body['PATH']['IN']['jobCfg']
out_path = cfg_body['PATH']['MID']['match']
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
# 查找fy3b_mersi文件
match_fy3b_file = []
fy3b_l3_path_use = pb_io.path_replace_ymd(fy3b_l3, ymd)
file_fy3b = pb_io.find_file(fy3b_l3_path_use, reg_fy3b % (ymd))
match_fy3b_file.extend(file_fy3b)
# 检查是否已经生成产品
pruc_names = ['KD490', 'CHL1', 'POC', 'ZSD']
match_aqua = []
for pruc_name in pruc_names:
# outfile = out_path + pruc_name + os.sep + \
# ymd + '_' + pruc_name + "_5000.hdf"
# if os.path.isfile(outfile):
# print u'产品已经存在'
# pass
# else:
# 查找对应的aqua_modis产品文件
if pruc_name != 'CHL1':
file_aqua = pb_io.find_file(modis_l3,
reg_aqua % (ymd, pruc_name))
match_aqua.extend(file_aqua)
else:
file_aqua = pb_io.find_file(modis_l3,
reg_aqua % (ymd, pruc_name))
if len(file_aqua) > 0:
match_aqua.extend(file_aqua)
else:
file_aqua1 = pb_io.find_file(
modis_l3, reg_aqua_chl1 % (ymd, pruc_name))
match_aqua.extend(file_aqua1)
# 判断当前五分钟的数据是否都存在
if len(match_fy3b_file) > 0 and len(match_aqua) > 0:
com_dict = {
'PATH': {
'ipath1': match_fy3b_file,
'ipath2': match_aqua,
'out_path': out_path
},
'INFO': {
'time': ymd,
'pair': sat_pair
},
}
cfgFile = os.path.join(jobcfg_path, sat_pair, job_id,
'%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
# check_fy_aqual2 = CheckFyAqual2()
# check_fy_aqual2.main(
# match_fy3b_file, match_iop_file, match_oc_file, ymd_HM, out_path)
cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0910(job_exe, sat_pair, date_s, date_e, job_id):
"""
基于aqua(modis)和fy3b（mersi）的l2数据匹配结果
"""
# date_s, date_e = time.split("-")
# date_s = datetime.strptime(date_s, '%Y%m%d%H%M')
# date_e = datetime.strptime(date_e, '%Y%m%d%H%M')
Log.info(u'%s: %s 检验l2产品 开始...' % (job_id, job_exe))
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
reg_fy3b = 'FY3B_MERSI_ORBT_L2_OCC_MLT_NUL_.*._1000M.HDF'
reg_aqua_iop = 'A.*.L2_LAC_IOP.nc'
reg_aqua_oc = 'A.*.L2_LAC_OC.nc'
fy3b_l2 = cfg_body['PATH']['MID']['granule'] # fy3b_l2 apth
modis_l2 = cfg_body['PATH']['IN']['data'] # modis_l2
jobcfg_path = cfg_body['PATH']['IN']['jobCfg']
out_path = cfg_body['PATH']['MID']['match']
arg_list = []
while date_s <= date_e:
# ymd_HM = date_s.strftime('%Y%m%d_%H%M')
ymd = date_s.strftime('%Y%m%d')
# 解析mathcing: FY3A+MERSI_AQUA+MODIS FY3B+MERSI_AQUA+MODIS_check
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
fy3b_l2_path_use = pb_io.path_replace_ymd(fy3b_l2, ymd)
modis_l2_path_use = modis_l2 + '/AQUA/MODIS/L2/ORBIT' + \
os.sep + date_s.strftime('%Y%m')
file_list1 = pb_io.find_file(fy3b_l2_path_use, reg_fy3b)
file_list2 = pb_io.find_file(modis_l2_path_use, reg_aqua_iop)
# file_list3 = pb_io.find_file(modis_l2_path_use, reg_aqua_oc)
timeList = ReadCrossFile_LEO_LEO('FENGYUN-3B', 'AQUA', ymd)
# print timeList, CROSS_DIR, sat1, sat2, ymd
for crossTime in timeList:
ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S')
s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1))
e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1))
s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2))
e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2))
# 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据
match_fy3b_file = Find_data_fy_FromCrossTime(
file_list1, s_cross_time1, e_cross_time1)
match_iop_file = Find_data_aqua_FromCrossTime(
file_list2, s_cross_time2, e_cross_time2)
# match_oc_file = Find_data_aqua_FromCrossTime(
# file_list3, s_cross_time2, e_cross_time2)
# 判断当前五分钟的数据是否都存在
if len(match_fy3b_file) > 0 and len(match_iop_file) > 0:
for match_fy3b in match_fy3b_file:
for match_iop in match_iop_file:
com_dict = {
'PATH': {
'ipath1': [match_fy3b],
'ipath2': [match_iop],
'out_path': out_path
},
'INFO': {
'time': ymdhms,
'pair': sat_pair,
'lat': crossTime[0],
'lon': crossTime[1]
}
}
cfgFile = os.path.join(jobcfg_path, sat_pair, job_id,
'%s.yaml' % ymdhms)
CreateYamlCfg(com_dict, cfgFile)
# check_fy_aqual2 = CheckFyAqual2()
# check_fy_aqual2.main(
# match_fy3b_file, match_iop_file, match_oc_file, ymd_HM,
# out_path)
cmd = '%s %s %s %s' % (python, job_exe, sat_pair,
cfgFile)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0216(job_exe, sat_pair, date_s, date_e, job_id):
Log.info(u'%s: %s 水色L3产品检验处理开始...' % (job_id, job_exe))
# 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
in_path1 = cfg_body['PATH']['OUT']['daily']
in_path2 = cfg_body['PATH']['IN']['data']
out_path_cfg = cfg_body['PATH']['IN']['jobCfg']
out_path_match = cfg_body['PATH']['MID']['match']
# 存放分发列表
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
jjj = date_s.strftime('%j')
# 存放俩颗卫星的原始数据目录位置
full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd)
full_in_path2 = os.path.join(in_path2, '%s/%s/L3' % (sat2, sensor2))
print "inpath1", full_in_path1
print "inpath2", full_in_path2
sat11 = cfg_body['SAT_S2L'][sat1]
sat22 = cfg_body['SAT_S2L'][sat2]
# 读取交叉点上的俩颗卫星的交叉时间，1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间
reg1 = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % ymd
reg2 = 'L3m_%s__GLOB_4_AV-MOD_KD490_DAY_00.nc' % (ymd)
file_list1 = pb_io.find_file(full_in_path1, reg1)
file_list2 = pb_io.find_file(full_in_path2, reg2)
# 存放匹配信息的yaml配置文件存放位置
yaml_file3 = os.path.join(out_path_cfg, sat_pair, job_id, ymd,
'%s_%s_%s.yaml' % (ymd, sensor1, sensor2))
filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymd)
# 输出完整路径
full_filename3 = os.path.join(out_path_match, sat_pair, ymd[:6],
filename3)
# 投影参数
deg = 0.05
res = deg2meter(deg)
half_res = deg2meter(res) / 2.
cmd = '+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=-%f +y_0=%f +datum=WGS84' % (
half_res, half_res)
if len(file_list1) > 0 and len(file_list2) > 0:
print '111111'
dict3 = {
'INFO': {
'sat1': sat1,
'sensor1': sensor1,
'sat2': sat2,
'sensor2': sensor2,
'ymd': ymd,
'pair': sat_pair
},
'PATH': {
'opath': full_filename3,
'ipath1': file_list1,
'ipath2': file_list2
},
'PROJ': {
'cmd': cmd,
'res': deg
}
}
Log.info('%s %s create collocation cfg success' % (sat_pair, ymd))
CreateYamlCfg(dict3, yaml_file3)
cmd = '%s %s %s' % (python, job_exe, yaml_file3)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
