def job_0311(job_exe, sat_pair, date_s, date_e, job_id):
'''
:水色反演
'''
Log.info(u'%s: %s 水色反演处理开始...' % (job_id, job_exe))
# 去掉路径信息
in_path = cfg_body['PATH']['MID']['granule']
out_path = in_path
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_out_path = pb_io.path_replace_ymd(out_path, ymd)
if not os.path.isdir(use_out_path):
os.makedirs(use_out_path)
use_reg = '.*_%s_.*.HDF' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = 'idl -rt=%s -args %s %s/' % (job_exe, in_file,
use_out_path)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0710(job_exe, sat_pair, date_s, date_e, job_id):
'''
:月合成
'''
Log.info(u'%s: %s 月合成处理开始...' % (job_id, job_exe))
cfg_path = cfg_body['PATH']['MID']['incfg']
daily_path = cfg_body['PATH']['OUT']['daily']
month_path = cfg_body['PATH']['OUT']['monthly']
# 清理配置
if os.path.isdir(cfg_path):
shutil.rmtree(cfg_path)
arg_list = []
# 月首和月末 调整
ymd1 = date_s.strftime('%Y%m%d')
ymd2 = date_e.strftime('%Y%m%d')
lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1]
date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d')
date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d')
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
# 输出
com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAM_5000M.HDF' % (
cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd)
month_path_use = pb_io.path_replace_ymd(month_path, ymd)
com_out_file = os.path.join(month_path_use, com_filename)
# 输入
reg = '.*_%s.*.HDF' % ymd[0:6]
daily_path_use = pb_io.path_replace_ymd(daily_path, ymd)
data_list_use = pb_io.find_file(daily_path_use, reg)
if len(data_list_use) > 0:
com_dict = {
'PATH': {
'ipath': data_list_use,
'opath': com_out_file
}
}
cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
date_s = date_s + relativedelta(months=1)
reg = '.*.yaml'
FileLst = pb_io.find_file(cfg_path, reg)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0610(job_exe, sat_pair, date_s, date_e, job_id):
'''
:日合成
'''
Log.info(u'%s: %s 日合成处理开始...' % (job_id, job_exe))
granule_path = cfg_body['PATH']['MID']['granule']
proj_path = cfg_body['PATH']['MID']['projection']
cfg_path = cfg_body['PATH']['MID']['incfg']
daily_path = cfg_body['PATH']['OUT']['daily']
# 清理配置
if os.path.isdir(cfg_path):
shutil.rmtree(cfg_path)
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % (
cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd)
daily_path_use = pb_io.path_replace_ymd(daily_path, ymd)
com_out_file = os.path.join(daily_path_use, com_filename)
granule_path_use = pb_io.path_replace_ymd(granule_path, ymd)
reg = '.*_%s_.*.HDF' % ymd
granule_lst = pb_io.find_file(granule_path_use, reg)
# reg = '.*_%s_.*.HDF' % ymd
proj_path_use = pb_io.path_replace_ymd(proj_path, ymd)
granule_pro_lst = pb_io.find_file(proj_path_use, reg)
if len(granule_pro_lst) > 0:
com_dict = {
'PATH': {
'ipath': granule_lst,
'ppath': granule_pro_lst,
'opath': com_out_file
}
}
cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
date_s = date_s + relativedelta(days=1)
reg = '.*.yaml'
FileLst = pb_io.find_file(cfg_path, reg)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0711(job_exe, sat_pair, date_s, date_e, job_id):
'''
:月成后文件绘图
'''
Log.info(u'%s: %s 月合成出图处理开始...' % (job_id, job_exe))
ipath = cfg_body['PATH']['OUT']['monthly']
FileLst = []
arg_list = []
# 月首和月末 调整
ymd1 = date_s.strftime('%Y%m%d')
ymd2 = date_e.strftime('%Y%m%d')
lastday = calendar.monthrange(int(ymd2[:4]), int(ymd2[4:6]))[1]
date_s = datetime.strptime('%s01' % ymd1[0:6], '%Y%m%d')
date_e = datetime.strptime('%s%d' % (ymd2[0:6], lastday), '%Y%m%d')
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
path_use = pb_io.path_replace_ymd(ipath, ymd)
reg = '.*_%s_.*.HDF' % ymd
mlist = pb_io.find_file(path_use, reg)
FileLst.extend(mlist)
date_s = date_s + relativedelta(months=1)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0310(job_exe, sat_pair, date_s, date_e, job_id):
'''
:反演
'''
Log.info(u'%s: %s 反演预处理开始...' % (job_id, job_exe))
cfg = 'aerosol.cfg'
# 去掉路径信息
job_exe = os.path.basename(job_exe)
in_path = cfg_body['PATH']['MID']['calibrate']
reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = '.*_%s_.*.HDF' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = './%s %s %s ' % (job_exe, in_file, cfg)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0810(job_exe, sat_pair, date_s, date_e, job_id):
'''
:年合成
'''
Log.info(u'%s: %s 年合成处理开始...' % (job_id, job_exe))
cfg_path = cfg_body['PATH']['MID']['incfg']
month_path = cfg_body['PATH']['OUT']['monthly']
yearly_path = cfg_body['PATH']['OUT']['yearly']
# 清理配置
if os.path.isdir(cfg_path):
shutil.rmtree(cfg_path)
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
# 输出
com_filename = '%s_%s_GBAL_L3_OCC_MLT_GLL_%s_YEAR_5000M.HDF' % (
cfg_body['PATH']['sat'], cfg_body['PATH']['sensor'], ymd)
yearly_path_use = pb_io.path_replace_ymd(yearly_path, ymd)
com_out_file = os.path.join(yearly_path_use, com_filename)
# 输入
reg = '.*_%s.*.HDF' % ymd[0:4]
month_path_use = pb_io.path_replace_ymd(month_path, ymd)
data_list_use = pb_io.find_file(month_path_use, reg)
if len(data_list_use) > 0:
com_dict = {
'PATH': {
'ipath': data_list_use,
'opath': com_out_file
}
}
cfgFile = os.path.join(cfg_path, '%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
date_s = date_s + relativedelta(years=1)
reg = '.*.yaml'
FileLst = pb_io.find_file(cfg_path, reg)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def _get_obc_file(m1000_file, obc_path):
"""
通过 1KM 文件路径生成 OBC 文件的路径
:param m1000_file:
:param obc_path:
:return:
"""
ymd = _get_ymd(m1000_file)
hm = _get_hm(m1000_file)
time = ymd + hm
obc_path = path_replace_ymd(obc_path, ymd)
obc_files = get_files_by_ymd(obc_path,
time,
time,
ext='.HDF',
pattern_ymd=r".*_(\d{8})_(\d{4})_")
obc_file = obc_files[0]
return obc_file
def job_0110(job_exe, sat_pair, date_s, date_e, job_id):
"""
ncep处理的输入接口
"""
Log.info(u'%s: %s ncep处理开始...' % (job_id, job_exe))
in_path = cfg_body['PATH']['IN']['ncep']
reg = 'fnl_%s_.*'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = reg % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0210(job_exe, sat_pair, date_s, date_e, job_id):
"""
L1数据预处理的输入接口
"""
Log.info(u'%s: %s L1数据预处理开始...' % (job_id, job_exe))
in_path = cfg_body['PATH']['IN']['l1']
# reg = 'FY3[A-Z]_MERSI_GBAL_L1_%s_(\d{4})_1000M_MS.HDF'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = '.*_%s_.*.HDF$' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def _get_geo_file(m1000_file, geo_path):
"""
通过 1KM 文件路径生成 GEO 文件的路径
:param m1000_file:
:param geo_path:
:return:
"""
ymd = _get_ymd(m1000_file)
hm = _get_hm(m1000_file)
time = ymd + hm
geo_path = path_replace_ymd(geo_path, ymd)
print geo_path
print time
geo_files = get_files_by_ymd(geo_path,
time,
time,
ext='.HDF',
pattern_ymd=r".*_(\d{8})_(\d{4})_")
geo_file = geo_files[0]
return geo_file
def job_0611(job_exe, sat_pair, date_s, date_e, job_id):
'''
:合成后文件绘图
'''
Log.info(u'%s: %s 日合成出图处理开始...' % (job_id, job_exe))
ipath = cfg_body['PATH']['OUT']['daily']
FileLst = []
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
path_use = pb_io.path_replace_ymd(ipath, ymd)
reg = '.*_%s_.*.HDF' % ymd
dlist = pb_io.find_file(path_use, reg)
FileLst.extend(dlist)
date_s = date_s + relativedelta(days=1)
for in_file in FileLst:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
return arg_list
def job_0410(job_exe, sat_pair, date_s, date_e, job_id):
'''
:反演后的快视图
'''
Log.info(u'%s: %s 反演轨道产品快视图处理开始...' % (job_id, job_exe))
in_path = cfg_body['PATH']['MID']['granule']
# reg = 'FY3[A-Z]_MERSI_ORBT_L2_\w{3}_MLT_NUL_%s_(\d{4})_1000M.HDF'
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
use_in_path = pb_io.path_replace_ymd(in_path, ymd)
use_reg = '.*_%s_.*.HDF' % ymd
file_list = pb_io.find_file(use_in_path, use_reg)
for in_file in file_list:
cmd_list = '%s %s %s %s' % (python, job_exe, sat_pair, in_file)
arg_list.append(cmd_list)
date_s = date_s + relativedelta(days=1)
return arg_list
def _get_out_file(in_file, out_path, suffix):
"""
通过输入的文件名和输出路径获取输出文件的完整路径
:return:
"""
try:
if isinstance(in_file, str):
ymd = pb_time.get_ymd(in_file)
out_path = pb_io.path_replace_ymd(out_path, ymd)
_name = os.path.basename(in_file)
name = _name.replace("_c", "") + suffix
out_file = os.path.join(out_path, name)
if not os.path.isdir(os.path.dirname(out_file)):
try:
os.makedirs(os.path.dirname(out_file))
except OSError as why:
print why
return out_file
except Exception as why:
print why
return
def job_0211(job_exe, sat_pair, date_s, date_e, job_id, reload=None):
if reload is None:
Log.info(u'%s: %s 交叉匹配(use pre data)处理开始...' % (job_id, job_exe))
else:
print 'reload'
# 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
DATA_DIR = cfg_body['PATH']['IN']['data']
CALI_DIR = cfg_body['PATH']['MID']['calibrate']
jobCfg = cfg_body['PATH']['IN']['jobCfg']
match_path = cfg_body['PATH']['MID']['match']
L1_data_dir = cfg_body['PATH']['IN']['l1']
coeff_path = cfg_body['PATH']['IN']['coeff']
# 存放分发列表
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
jjj = date_s.strftime('%j')
coeff_file = os.path.join(coeff_path, '%s.txt' % ymd[:4])
print ymd
# 存放俩颗卫星的原始数据目录位置
if reload is None:
# inpath1 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' %
# (sat1, sensor1), ymd[:6])
FINAL_DIR = pb_io.path_replace_ymd(CALI_DIR, ymd)
inpath1 = os.path.join(FINAL_DIR)
else:
inpath1 = pb_io.path_replace_ymd(L1_data_dir, ymd)
inpath2 = os.path.join(DATA_DIR, '%s/%s/L1/ORBIT' % (
sat2,
sensor2,
), ymd[:6])
print "inpath1", inpath1
print "inpath2", inpath2
sat11 = cfg_body['SAT_S2L'][sat1]
sat22 = cfg_body['SAT_S2L'][sat2]
# 读取交叉点上的俩颗卫星的交叉时间，1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间
timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd)
print 'cross', len(timeList)
reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd
reg2 = 'MYD021KM.A%s%s.*.hdf' % (ymd[0:4], jjj)
file_list1 = pb_io.find_file(inpath1, reg1)
file_list2 = pb_io.find_file(inpath2, reg2)
# 根据交叉点时间，找到数据列表中需要的数据 select File
for crossTime in timeList:
Lat = crossTime[0]
Lon = crossTime[1]
ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S')
s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1))
e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1))
s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2))
e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2))
# 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据
list1 = Find_data_FromCrossTime(file_list1, s_cross_time1,
e_cross_time1)
list2 = Find_data_FromCrossTime(file_list2, s_cross_time2,
e_cross_time2)
print 'fy', len(list1)
print 'mo', len(list2)
# 存放匹配信息的yaml配置文件存放位置
yaml_file3 = os.path.join(
jobCfg, sat_pair, job_id, ymdhms[:8],
'%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2))
filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms)
# 输出完整路径
# FY3B+MERSI_AQUA+MODIS_L1 和 FY3B+MERSI_AQUA+MODIS 两种支持 wangpeng
# add 2018-09-14
full_filename3 = os.path.join(match_path, sat_pair, ymdhms[:6],
filename3)
# if reload is None:
# full_filename3 = os.path.join(
# match_path, sat_pair, ymdhms[:6], filename3)
# else:
# full_filename3 = os.path.join(
# match_path, sat_pair + '_L1', ymdhms[:6], filename3)
# 投影参数
cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % (
Lat, Lon)
if len(list1) > 0 and len(list2) > 0:
print '111111'
row = 128
col = 128
res = 8000
dict3 = {
'INFO': {
'sat1': sat1,
'sensor1': sensor1,
'sat2': sat2,
'sensor2': sensor2,
'ymd': ymdhms
},
'PATH': {
'opath': full_filename3,
'ipath1': list1,
'ipath2': list2,
'ipath_coeff': coeff_file
},
'PROJ': {
'cmd': cmd,
'row': row,
'col': col,
'res': res
}
}
Log.info('%s %s create collocation cfg success' %
(sat_pair, ymdhms))
CreateYamlCfg(dict3, yaml_file3)
if reload is None:
cmd = '%s %s %s 0' % (python, job_exe, yaml_file3)
else:
cmd = '%s %s %s 1' % (python, job_exe, yaml_file3)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def main(sat_sensor, in_file):
"""
对L2数据进行投影，记录投影后的位置信息和数据信息
:param sat_sensor: 卫星+传感器
:param in_file: HDF5 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 加载全局配置信息
out_path = gc.path_mid_projection
# 加载程序配置信息
# 加载卫星配置信息
res = sc.project_res
half_res = deg2meter(res) / 2.
cmd = sc.project_cmd % (half_res, half_res)
row = sc.project_row
col = sc.project_col
mesh_size = sc.project_mesh_size
# ######################## 开始处理 ###########################
print "-" * 100
print "Start projection."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
with time_block("One project time:", switch=TIME_TEST):
# 生成输出文件的文件名
ymd = _get_ymd(in_file)
hm = _get_hm(in_file)
sat, sensor = sat_sensor.split('+')
out_name = "{}_{}_ORBT_L2_OCC_MLT_NUL_{}_{}_{}.HDF".format(
sat, sensor, ymd, hm, mesh_size.upper())
out_path = pb_io.path_replace_ymd(out_path, ymd)
out_file = os.path.join(out_path, out_name)
# 如果输出文件已经存在，跳过
if os.path.isfile(out_file):
print "File is already exist, skip it: {}".format(out_file)
return
# 开始创建投影查找表
projection = Projection(cmd=cmd, row=row, col=col, res=res)
projection.project(in_file, out_file)
if not projection.error:
print ">>> {}".format(out_file)
else:
print "Error: Projection error: {}".format(in_file)
print "-" * 100
def job_0911(job_exe, sat_pair, date_s, date_e, job_id):
"""
基于aqua(modis)和fy3b（mersi）的l3数据匹配结果
"""
# date_s, date_e = time.split("-")
# date_s = datetime.strptime(date_s, '%Y%m%d%H%M')
# date_e = datetime.strptime(date_e, '%Y%m%d%H%M')
Log.info(u'%s: %s 检验l3产品 开始...' % (job_id, job_exe))
reg_fy3b = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF'
reg_aqua_chl1 = 'L3m_%s__GLOB_4_GSM-MOD_%s_DAY_00.nc'
reg_aqua = 'L3m_%s__GLOB_4_AV-MOD_%s_DAY_00.nc'
fy3b_l3 = cfg_body['PATH']['OUT']['daily'] # fy3b_l3 apth
modis_l3 = cfg_body['PATH']['IN']['modis_l3'] # modis_l3
jobcfg_path = cfg_body['PATH']['IN']['jobCfg']
out_path = cfg_body['PATH']['MID']['match']
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
# 查找fy3b_mersi文件
match_fy3b_file = []
fy3b_l3_path_use = pb_io.path_replace_ymd(fy3b_l3, ymd)
file_fy3b = pb_io.find_file(fy3b_l3_path_use, reg_fy3b % (ymd))
match_fy3b_file.extend(file_fy3b)
# 检查是否已经生成产品
pruc_names = ['KD490', 'CHL1', 'POC', 'ZSD']
match_aqua = []
for pruc_name in pruc_names:
# outfile = out_path + pruc_name + os.sep + \
# ymd + '_' + pruc_name + "_5000.hdf"
# if os.path.isfile(outfile):
# print u'产品已经存在'
# pass
# else:
# 查找对应的aqua_modis产品文件
if pruc_name != 'CHL1':
file_aqua = pb_io.find_file(modis_l3,
reg_aqua % (ymd, pruc_name))
match_aqua.extend(file_aqua)
else:
file_aqua = pb_io.find_file(modis_l3,
reg_aqua % (ymd, pruc_name))
if len(file_aqua) > 0:
match_aqua.extend(file_aqua)
else:
file_aqua1 = pb_io.find_file(
modis_l3, reg_aqua_chl1 % (ymd, pruc_name))
match_aqua.extend(file_aqua1)
# 判断当前五分钟的数据是否都存在
if len(match_fy3b_file) > 0 and len(match_aqua) > 0:
com_dict = {
'PATH': {
'ipath1': match_fy3b_file,
'ipath2': match_aqua,
'out_path': out_path
},
'INFO': {
'time': ymd,
'pair': sat_pair
},
}
cfgFile = os.path.join(jobcfg_path, sat_pair, job_id,
'%s.yaml' % ymd)
CreateYamlCfg(com_dict, cfgFile)
# check_fy_aqual2 = CheckFyAqual2()
# check_fy_aqual2.main(
# match_fy3b_file, match_iop_file, match_oc_file, ymd_HM, out_path)
cmd = '%s %s %s %s' % (python, job_exe, sat_pair, cfgFile)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0910(job_exe, sat_pair, date_s, date_e, job_id):
"""
基于aqua(modis)和fy3b（mersi）的l2数据匹配结果
"""
# date_s, date_e = time.split("-")
# date_s = datetime.strptime(date_s, '%Y%m%d%H%M')
# date_e = datetime.strptime(date_e, '%Y%m%d%H%M')
Log.info(u'%s: %s 检验l2产品 开始...' % (job_id, job_exe))
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
reg_fy3b = 'FY3B_MERSI_ORBT_L2_OCC_MLT_NUL_.*._1000M.HDF'
reg_aqua_iop = 'A.*.L2_LAC_IOP.nc'
reg_aqua_oc = 'A.*.L2_LAC_OC.nc'
fy3b_l2 = cfg_body['PATH']['MID']['granule'] # fy3b_l2 apth
modis_l2 = cfg_body['PATH']['IN']['data'] # modis_l2
jobcfg_path = cfg_body['PATH']['IN']['jobCfg']
out_path = cfg_body['PATH']['MID']['match']
arg_list = []
while date_s <= date_e:
# ymd_HM = date_s.strftime('%Y%m%d_%H%M')
ymd = date_s.strftime('%Y%m%d')
# 解析mathcing: FY3A+MERSI_AQUA+MODIS FY3B+MERSI_AQUA+MODIS_check
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
fy3b_l2_path_use = pb_io.path_replace_ymd(fy3b_l2, ymd)
modis_l2_path_use = modis_l2 + '/AQUA/MODIS/L2/ORBIT' + \
os.sep + date_s.strftime('%Y%m')
file_list1 = pb_io.find_file(fy3b_l2_path_use, reg_fy3b)
file_list2 = pb_io.find_file(modis_l2_path_use, reg_aqua_iop)
# file_list3 = pb_io.find_file(modis_l2_path_use, reg_aqua_oc)
timeList = ReadCrossFile_LEO_LEO('FENGYUN-3B', 'AQUA', ymd)
# print timeList, CROSS_DIR, sat1, sat2, ymd
for crossTime in timeList:
ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S')
s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1))
e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1))
s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2))
e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2))
# 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据
match_fy3b_file = Find_data_fy_FromCrossTime(
file_list1, s_cross_time1, e_cross_time1)
match_iop_file = Find_data_aqua_FromCrossTime(
file_list2, s_cross_time2, e_cross_time2)
# match_oc_file = Find_data_aqua_FromCrossTime(
# file_list3, s_cross_time2, e_cross_time2)
# 判断当前五分钟的数据是否都存在
if len(match_fy3b_file) > 0 and len(match_iop_file) > 0:
for match_fy3b in match_fy3b_file:
for match_iop in match_iop_file:
com_dict = {
'PATH': {
'ipath1': [match_fy3b],
'ipath2': [match_iop],
'out_path': out_path
},
'INFO': {
'time': ymdhms,
'pair': sat_pair,
'lat': crossTime[0],
'lon': crossTime[1]
}
}
cfgFile = os.path.join(jobcfg_path, sat_pair, job_id,
'%s.yaml' % ymdhms)
CreateYamlCfg(com_dict, cfgFile)
# check_fy_aqual2 = CheckFyAqual2()
# check_fy_aqual2.main(
# match_fy3b_file, match_iop_file, match_oc_file, ymd_HM,
# out_path)
cmd = '%s %s %s %s' % (python, job_exe, sat_pair,
cfgFile)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0215(job_exe, sat_pair, date_s, date_e, job_id):
Log.info(u'%s: %s 水色L2产品检验处理开始...' % (job_id, job_exe))
# 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
in_path1 = cfg_body['PATH']['MID']['granule']
in_path2 = cfg_body['PATH']['IN']['data']
out_path_cfg = cfg_body['PATH']['IN']['jobCfg']
out_path_match = cfg_body['PATH']['MID']['match']
# 存放分发列表
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
jjj = date_s.strftime('%j')
# 存放俩颗卫星的原始数据目录位置
full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd)
full_in_path2 = os.path.join(in_path2, '%s/%s/L2/ORBIT' % (
sat2,
sensor2,
), ymd[:6])
print "inpath1", full_in_path1
print "inpath2", full_in_path2
sat11 = cfg_body['SAT_S2L'][sat1]
sat22 = cfg_body['SAT_S2L'][sat2]
# 读取交叉点上的俩颗卫星的交叉时间，1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间
timeList = ReadCrossFile_LEO_LEO(sat11, sat22, ymd)
print 'cross', len(timeList)
reg1 = 'FY3B_MERSI.*_%s_.*.HDF' % ymd
reg2 = 'A%s%s.*.nc' % (ymd[0:4], jjj)
file_list1 = pb_io.find_file(full_in_path1, reg1)
file_list2 = pb_io.find_file(full_in_path2, reg2)
# 根据交叉点时间，找到数据列表中需要的数据 select File
for crossTime in timeList:
Lat = crossTime[0]
Lon = crossTime[1]
ymdhms = crossTime[2].strftime('%Y%m%d%H%M%S')
s_cross_time1 = crossTime[2] - relativedelta(seconds=int(sec1))
e_cross_time1 = crossTime[2] + relativedelta(seconds=int(sec1))
s_cross_time2 = crossTime[3] - relativedelta(seconds=int(sec2))
e_cross_time2 = crossTime[3] + relativedelta(seconds=int(sec2))
# 从数据列表中查找过此交叉点时间的数据块,两颗卫星的数据
list1 = Find_data_FromCrossTime(file_list1, s_cross_time1,
e_cross_time1)
list2 = Find_data_FromCrossTime(file_list2, s_cross_time2,
e_cross_time2)
print 'fy', len(list1)
print 'mo', len(list2)
# 存放匹配信息的yaml配置文件存放位置
yaml_file3 = os.path.join(
out_path_cfg, sat_pair, job_id, ymdhms[:8],
'%s_%s_%s.yaml' % (ymdhms, sensor1, sensor2))
filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymdhms)
# 输出完整路径
full_filename3 = os.path.join(out_path_match, sat_pair, ymdhms[:6],
filename3)
# 投影参数
cmd = '+proj=laea +lat_0=%f +lon_0=%f +x_0=0 +y_0=0 +ellps=WGS84' % (
Lat, Lon)
if len(list1) > 0 and len(list2) > 0:
print '111111'
row = 128
col = 128
res = 8000
dict3 = {
'INFO': {
'sat1': sat1,
'sensor1': sensor1,
'sat2': sat2,
'sensor2': sensor2,
'ymd': ymdhms,
'pair': sat_pair
},
'PATH': {
'opath': full_filename3,
'ipath1': list1,
'ipath2': list2
},
'PROJ': {
'cmd': cmd,
'row': row,
'col': col,
'res': res
}
}
Log.info('%s %s create collocation cfg success' %
(sat_pair, ymdhms))
CreateYamlCfg(dict3, yaml_file3)
cmd = '%s %s %s' % (python, job_exe, yaml_file3)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def job_0216(job_exe, sat_pair, date_s, date_e, job_id):
Log.info(u'%s: %s 水色L3产品检验处理开始...' % (job_id, job_exe))
# 解析mathcing: FY3A+MERSI_AQUA+MODIS ,根据下划线分割获取 卫星+传感器 ,再次分割获取俩颗卫星短名
sat1 = (sat_pair.split('_')[0]).split('+')[0]
sensor1 = (sat_pair.split('_')[0]).split('+')[1]
sat2 = (sat_pair.split('_')[1]).split('+')[0]
sensor2 = (sat_pair.split('_')[1]).split('+')[1]
# 解析global.cfg中的信息
sec1 = cfg_body['PAIRS'][sat_pair]['sec1']
sec2 = cfg_body['PAIRS'][sat_pair]['sec2']
in_path1 = cfg_body['PATH']['OUT']['daily']
in_path2 = cfg_body['PATH']['IN']['data']
out_path_cfg = cfg_body['PATH']['IN']['jobCfg']
out_path_match = cfg_body['PATH']['MID']['match']
# 存放分发列表
arg_list = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
jjj = date_s.strftime('%j')
# 存放俩颗卫星的原始数据目录位置
full_in_path1 = pb_io.path_replace_ymd(in_path1, ymd)
full_in_path2 = os.path.join(in_path2, '%s/%s/L3' % (sat2, sensor2))
print "inpath1", full_in_path1
print "inpath2", full_in_path2
sat11 = cfg_body['SAT_S2L'][sat1]
sat22 = cfg_body['SAT_S2L'][sat2]
# 读取交叉点上的俩颗卫星的交叉时间，1列=经度 2列=纬度 3列=卫星1时间 4列=卫星2时间
reg1 = 'FY3B_MERSI_GBAL_L3_OCC_MLT_GLL_%s_AOAD_5000M.HDF' % ymd
reg2 = 'L3m_%s__GLOB_4_AV-MOD_KD490_DAY_00.nc' % (ymd)
file_list1 = pb_io.find_file(full_in_path1, reg1)
file_list2 = pb_io.find_file(full_in_path2, reg2)
# 存放匹配信息的yaml配置文件存放位置
yaml_file3 = os.path.join(out_path_cfg, sat_pair, job_id, ymd,
'%s_%s_%s.yaml' % (ymd, sensor1, sensor2))
filename3 = '%s_MATCHEDPOINTS_%s.H5' % (sat_pair, ymd)
# 输出完整路径
full_filename3 = os.path.join(out_path_match, sat_pair, ymd[:6],
filename3)
# 投影参数
deg = 0.05
res = deg2meter(deg)
half_res = deg2meter(res) / 2.
cmd = '+proj=eqc +lat_ts=0 +lat_0=0 +lon_0=0 +x_0=-%f +y_0=%f +datum=WGS84' % (
half_res, half_res)
if len(file_list1) > 0 and len(file_list2) > 0:
print '111111'
dict3 = {
'INFO': {
'sat1': sat1,
'sensor1': sensor1,
'sat2': sat2,
'sensor2': sensor2,
'ymd': ymd,
'pair': sat_pair
},
'PATH': {
'opath': full_filename3,
'ipath1': file_list1,
'ipath2': file_list2
},
'PROJ': {
'cmd': cmd,
'res': deg
}
}
Log.info('%s %s create collocation cfg success' % (sat_pair, ymd))
CreateYamlCfg(dict3, yaml_file3)
cmd = '%s %s %s' % (python, job_exe, yaml_file3)
arg_list.append(cmd)
date_s = date_s + relativedelta(days=1)
return arg_list
def main(sat_sensor, in_file):
"""
使用矫正系数对 MERSI L1 的产品进行定标预处理
:param sat_sensor: (str) 卫星对
:param in_file: (str) 输入文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 全局配置接口
obc_path = gc.path_in_obc
geo_path = gc.path_in_geo
coeff_path = gc.path_in_coeff
out_path = gc.path_mid_calibrate
# 程序配置接口
# 卫星配置接口
launch_date = sc.launch_date
probe_count = sc.calibrate_probe_count
probe = sc.calibrate_probe
slide_step = sc.calibrate_slide_step
plot = sc.calibrate_plot
# ######################## 开始处理 ###########################
print '-' * 100
print 'Start calibration'
# 不同的卫星对使用不同的处理类和流程
if "fy3b" in sat_sensor.lower() and 'mersi' in sat_sensor.lower():
# 获取 M1000 文件和对应 OBC 文件
l1_1000m = in_file
obc_1000m = _get_obc_file(l1_1000m, obc_path)
if not os.path.isfile(l1_1000m):
log.error("File is not exist: {}".format(l1_1000m))
return
elif not os.path.isfile(obc_1000m):
log.error("File is not exist: {}".format(obc_1000m))
return
else:
print "<<< {}".format(l1_1000m)
print "<<< {}".format(obc_1000m)
ymd = _get_ymd(l1_1000m)
# 获取 coefficient 水色波段系统定标系数， 2013年以前和2013年以后不同
coeff_file = os.path.join(coeff_path, '{}.txt'.format(ymd[0:4]))
if not os.path.isfile(coeff_file):
log.error("File is not exist: {}".format(coeff_file))
return
else:
print "<<< {}".format(coeff_file)
# 获取输出文件
out_path = pb_io.path_replace_ymd(out_path, ymd)
_name = os.path.basename(l1_1000m)
out_file = os.path.join(out_path, _name)
# 如果输出文件已经存在，跳过预处理
if os.path.isfile(out_file):
print "File is already exist, skip it: {}".format(out_file)
return
# 初始化一个预处理实例
calibrate = CalibrateFY3B(l1_1000m=l1_1000m,
obc_1000m=obc_1000m,
coeff_file=coeff_file,
out_file=out_file,
launch_date=launch_date)
# 对 OBC 文件进行 SV 提取
calibrate.obc_sv_extract_fy3b(probe=probe,
probe_count=probe_count,
slide_step=slide_step)
# 重新定标 L1 数据
calibrate.calibrate()
# 将新数据写入 HDF5 文件
calibrate.write()
if not calibrate.error:
print ">>> {}".format(calibrate.out_file)
# 对原数据和处理后的数据各出一张真彩图
if plot == "on":
if not calibrate.error:
picture_suffix = "650_565_490"
file_name = os.path.splitext(out_file)[0]
out_pic_old = "{}_{}_old.{}".format(
file_name, picture_suffix, "png")
out_pic_new = "{}_{}_new.{}".format(
file_name, picture_suffix, "png")
# 如果输出文件已经存在，跳过
if os.path.isfile(out_pic_old):
print "File is already exist, skip it: {}".format(
out_pic_old)
return
else:
_plot_rgb_fy3b_old(in_file, out_pic_old)
if os.path.isfile(out_pic_new):
print "File is already exist, skip it: {}".format(
out_pic_new)
return
else:
_plot_rgb_fy3b_new(out_file, out_pic_new)
else:
print "Error: Calibrate error".format(in_file)
elif "fy3d" in sat_sensor.lower() and 'mersi' in sat_sensor.lower():
# 获取 M1000 文件和对应 GEO 文件
l1_1000m = in_file
geo_1000m = _get_geo_file(l1_1000m, geo_path)
if not os.path.isfile(l1_1000m):
log.error("File is not exist: {}".format(l1_1000m))
return
elif not os.path.isfile(geo_1000m):
log.error("File is not exist: {}".format(geo_1000m))
return
else:
print "<<< {}".format(l1_1000m)
print "<<< {}".format(geo_1000m)
ymd = _get_ymd(l1_1000m)
# 获取 coefficient 定标系数
coeff_file = os.path.join(coeff_path, '{}.txt'.format(ymd[0:4]))
if not os.path.isfile(coeff_file):
log.error("File is not exist: {}".format(coeff_file))
coeff_file = None
else:
print "<<< {}".format(coeff_file)
# 获取输出文件
out_path = pb_io.path_replace_ymd(out_path, ymd)
_name = os.path.basename(l1_1000m)
out_file = os.path.join(out_path, _name)
# 如果输出文件已经存在，跳过预处理
if os.path.isfile(out_file):
print "File is already exist, skip it: {}".format(out_file)
return
# 初始化一个预处理实例
calibrate = CalibrateFY3D()
# 重新定标 L1 数据
calibrate.calibrate(l1_1000m, geo_1000m, coeff_file)
# 将新数据写入 HDF5 文件
calibrate.write(out_file)
if not calibrate.error:
print ">>> {}".format(out_file)
if plot == "on":
if not calibrate.error:
picture_suffix = "670_555_490"
file_name = os.path.splitext(out_file)[0]
out_pic_old = "{}_{}_old.{}".format(
file_name, picture_suffix, "png")
out_pic_new = "{}_{}_new.{}".format(
file_name, picture_suffix, "png")
# 如果输出文件已经存在，跳过
if os.path.isfile(out_pic_old):
print "File is already exist, skip it: {}".format(
out_pic_old)
return
else:
pass
# _plot_rgb_fy3b_old(in_file, out_pic_old)
if os.path.isfile(out_pic_new):
print "File is already exist, skip it: {}".format(
out_pic_new)
return
else:
print out_file
_plot_rgb_fy3d_new(out_file, out_pic_new)
else:
print "***Error***: Calibrate error".format(in_file)
else:
print "***Error***: Don`t handle this sat and sensor: {}".format(
sat_sensor)
return
print '-' * 100
