def create_job_0210(job_name, job_id, ymd, job_cfg):
"""
气溶胶日合成代码
"""
# 输入信息
ipath_granule = job_cfg['PATH']['MID']['granule']
# 输出信息
opath_daily = job_cfg['PATH']['MID']['daily']
opath_yaml = job_cfg['PATH']['OUT']['interface']
# 实例
yy = ymd[0:4]
mm = ymd[4:6]
dd = ymd[6:8]
ipath_granule = str_format(ipath_granule, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_daily = str_format(opath_daily, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = str_format(opath_yaml, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = os.path.join(opath_yaml, job_id, ymd)
# 接口文件输出目录创建
if not os.path.isdir(opath_yaml):
os.makedirs(opath_yaml)
file_list = glob.glob('%s/*%s*.HDF5' % (ipath_granule, ymd))
file_list.sort()
if len(file_list) > 0:
# 输出接口文件
yaml_dict = {
'INFO': {
'job_name': job_name,
'ymd': ymd
},
'PATH': {
'ipath': file_list,
'opath': opath_daily
}
}
full_opath_yaml = os.path.join(opath_yaml, '%s.yaml' % (ymd))
write_yaml_file(yaml_dict, full_opath_yaml)
print('%s %s create yaml interface success' % (job_name, ymd))
def create_job_0110(job_name, job_id, ymd, job_cfg):
# 调度文件中的路径信息
ipath_01 = job_cfg['PAIRS'][job_name]['ipath_01']
ipath_02 = job_cfg['PAIRS'][job_name]['ipath_02']
ipath_03 = job_cfg['PAIRS'][job_name]['ipath_03']
opath_granule = job_cfg['PATH']['MID']['granule']
opath_yaml = job_cfg['PATH']['OUT']['interface']
# ipath_01 = str_format(ipath_01, {'JOBNAME': job_name, 'YYYYMMDD': ymd})
yy = ymd[0:4]
mm = ymd[4:6]
dd = ymd[6:8]
ipath_01 = str_format(ipath_01, {'YYYY': yy, 'MM': mm, 'DD': dd})
ipath_02 = str_format(ipath_02, {'YYYY': yy, 'MM': mm, 'DD': dd})
ipath_03 = str_format(ipath_03, {'YYYY': yy, 'MM': mm, 'DD': dd})
opath_granule = str_format(opath_granule, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = str_format(opath_yaml, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = os.path.join(opath_yaml, job_id, ymd)
# 接口文件输出目录创建
if not os.path.isdir(opath_yaml):
os.makedirs(opath_yaml)
print(ipath_01)
flist_l1b = glob.glob('%s/*%s*.HDF' % (ipath_01, ymd))
flist_geo = glob.glob('%s/*%s*.HDF' % (ipath_02, ymd))
flist_clm = glob.glob('%s/*%s*.HDF' % (ipath_03, ymd))
print('---', len(flist_l1b))
print('---', len(flist_geo))
print('---', len(flist_clm), ipath_03)
# 把L1数据放入字典
reg = '.*(\d{8})_(\d{4}).*.HDF'
flist_l1b_dict = dict()
for each in flist_l1b:
file_name = os.path.basename(each)
mreg = re.match(reg, file_name)
if mreg:
key_wd = '{}_{}'.format(mreg.group(1), mreg.group(2))
flist_l1b_dict[key_wd] = each
# 把GEO数据放入字典
reg = '.*(\d{8})_(\d{4}).*.HDF'
flist_geo_dict = dict()
for each in flist_geo:
file_name = os.path.basename(each)
mreg = re.match(reg, file_name)
if mreg:
key_wd = '{}_{}'.format(mreg.group(1), mreg.group(2))
flist_geo_dict[key_wd] = each
# 把云检测数据放入字典
reg = '.*(\d{8})_(\d{4}).*.HDF'
flist_clm_dict = dict()
for each in flist_clm:
file_name = os.path.basename(each)
mreg = re.match(reg, file_name)
if mreg:
key_wd = '{}_{}'.format(mreg.group(1), mreg.group(2))
flist_clm_dict[key_wd] = each
# 交集
if 'FY3D' in job_name:
key_wd_cross = flist_l1b_dict.keys() & flist_geo_dict.keys(
) & flist_clm_dict.keys()
else:
key_wd_cross = flist_l1b_dict.keys() & flist_clm_dict.keys()
for key in sorted(key_wd_cross):
ifile_l1b = flist_l1b_dict.get(key)
ifile_geo = flist_geo_dict.get(key)
ifile_clm = flist_clm_dict.get(key)
ymd = key.split('_')[0]
hms = key.split('_')[1] + '00'
# 输出接口文件
yaml_dict = {
'INFO': {
'job_name': job_name,
'ymd': ymd,
'hms': hms
},
'PATH': {
'ipath_l1b': ifile_l1b,
'ipath_geo': ifile_geo,
'ipath_clm': ifile_clm,
'opath': opath_granule
}
}
full_opath_yaml = os.path.join(opath_yaml, '%s_%s.yaml' % (ymd, hms))
write_yaml_file(yaml_dict, full_opath_yaml)
print('%s %s %s create yaml interface success' % (job_name, ymd, hms))
def create_job_0311(job_name, job_id, ymd, jjj, job_cfg, rewrite):
# 调度文件中的路径信息
yy = ymd[0:4]
mm = ymd[4:6]
dd = ymd[6:8]
# 输入信息
ipath_daily = job_cfg['PAIRS'][job_name]['ipath_daily']
# 输出信息
opath_daily = job_cfg['PATH']['MID']['daily']
opath_yaml = job_cfg['PATH']['OUT']['interface']
ipath_daily = str_format(ipath_daily, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_daily = str_format(opath_daily, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = str_format(opath_yaml, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = os.path.join(opath_yaml, job_id, ymd)
# 接口文件输出目录创建
if not os.path.isdir(opath_yaml):
os.makedirs(opath_yaml)
# 日志信息
path_log = job_cfg['PATH']['OUT']['log']
path_log = os.path.join(path_log, job_name, job_id, ymd[:4])
name_log = '%s.log' % ymd
# 清理日志
shutil.rmtree(path_log, ignore_errors=True)
vars_log = LogServer(path_log, name_log)
# 清理接口文件
if rewrite:
shutil.rmtree(opath_yaml, ignore_errors=True)
# 遍历目录
file_list = glob.glob('%s/*A%s%s*.hdf' % (ipath_daily, ymd[:4], jjj))
file_list.sort()
l1b_nums = len(file_list)
if l1b_nums <= 0:
lstr = log_str2str(job_name, job_id, ymd, '--', 'failed',
u'file nums: %s' % l1b_nums)
vars_log.error(lstr)
return
# 输出接口文件
yaml_dict = {
'INFO': {
'job_name': job_name,
'ymd': ymd,
'rewrite': rewrite
},
'PATH': {
'ipath': file_list,
'opath': opath_daily
}
}
hms = '000000'
full_opath_yaml = os.path.join(opath_yaml, '%s%s.yaml' % (ymd, hms))
try:
write_yaml_file(yaml_dict, full_opath_yaml)
lstr = log_str2str(job_name, job_id, ymd, hms, 'sucess', '--')
vars_log.info(lstr)
except Exception as e:
lstr = log_str2str(job_name, job_id, ymd, hms, 'failed', str(e))
vars_log.info(lstr)
def create_job_1010(job_name, job_id, ymd, jjj, job_cfg, rewrite):
# 调度文件中的路径信息
yy = ymd[0:4]
mm = ymd[4:6]
dd = ymd[6:8]
# 输入信息
ipath_h1 = job_cfg['PAIRS'][job_name]['ipath_orbit1']
ipath_h2 = job_cfg['PAIRS'][job_name]['ipath_orbit2']
ipath_h1 = str_format(ipath_h1, {'YYYY': yy, 'MM': mm, 'DD': dd})
ipath_h2 = str_format(ipath_h2, {'YYYY': yy, 'MM': mm, 'DD': dd})
print(ipath_h1)
print(ipath_h2)
# 输出信息
opath_check = job_cfg['PATH']['OUT']['check']
opath_check = str_format(opath_check, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
# 输出接口
opath_yaml = job_cfg['PATH']['OUT']['interface']
opath_yaml = str_format(opath_yaml, {'JOBNAME': job_name})
opath_yaml = os.path.join(opath_yaml, job_id, ymd)
# 接口文件输出目录创建
if not os.path.isdir(opath_yaml):
os.makedirs(opath_yaml)
# 日志信息
path_log = job_cfg['PATH']['OUT']['log']
path_log = os.path.join(path_log, job_name, job_id, ymd[:4])
name_log = '%s.log' % ymd
# 清理日志
shutil.rmtree(path_log, ignore_errors=True)
vars_log = LogServer(path_log, name_log)
# 清理接口文件
if rewrite:
shutil.rmtree(opath_yaml, ignore_errors=True)
# 遍历目录
file_list1 = glob.glob('%s/*.HDF5' % (ipath_h1))
file_list1.sort()
file_list2 = glob.glob('%s/*.HDF5' % (ipath_h2))
file_list2.sort()
namelist1 = [os.path.basename(each) for each in file_list1]
namelist2 = [os.path.basename(each) for each in file_list2]
corss_namelist = list(set(namelist1) & set(namelist2))
nums1 = len(corss_namelist)
if nums1 <= 0:
lstr = log_str2str(job_name, job_id, ymd, '--', 'failed',
u'file nums: %d' % (nums1))
vars_log.error(lstr)
return
# cross_file_list = list(set(file_list1).intersection(set(file_list2)))
# 输出接口文件
pattern = '.*(\d{8})_(\d{4})'
for each in corss_namelist:
print(each)
ifile1 = os.path.join(ipath_h1, each)
ifile2 = os.path.join(ipath_h2, each)
yaml_dict = {
'INFO': {
'job_name': job_name,
'ymd': ymd,
'rewrite': rewrite
},
'PATH': {
'ipath1': ifile1,
'ipath2': ifile2,
'opath': opath_check
}
}
m = re.match(pattern, each)
if m:
hms = m.group(2)
ofile_yaml = os.path.join(opath_yaml, '%s%s.yaml' % (ymd, hms))
try:
write_yaml_file(yaml_dict, ofile_yaml)
lstr = log_str2str(job_name, job_id, ymd, hms, 'sucess', '--')
vars_log.info(lstr)
except Exception as e:
lstr = log_str2str(job_name, job_id, ymd, hms, 'failed',
str(e))
vars_log.info(lstr)
def create_job_0110(job_name, job_id, ymd, job_cfg, rewrite, visfile, irfile):
# 调度文件中的路径信息
yy = ymd[0:4]
mm = ymd[4:6]
dd = ymd[6:8]
# 输入信息
ipath_geo = None
ipath_l1b = job_cfg['PAIRS'][job_name]['ipath_l1b']
ipath_clm = job_cfg['PAIRS'][job_name]['ipath_clm']
if 'ipath_geo' in job_cfg['PAIRS'][job_name]:
ipath_geo = job_cfg['PAIRS'][job_name]['ipath_geo']
ipath_l1b = str_format(ipath_l1b, {'YYYY': yy, 'MM': mm, 'DD': dd})
ipath_clm = str_format(ipath_clm, {'YYYY': yy, 'MM': mm, 'DD': dd})
ipath_geo = str_format(ipath_geo, {'YYYY': yy, 'MM': mm, 'DD': dd})
# 输出信息
opath_granule = job_cfg['PATH']['MID']['granule']
opath_yaml = job_cfg['PATH']['OUT']['interface']
opath_granule = str_format(opath_granule, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = str_format(opath_yaml, {
'JOBNAME': job_name,
'YYYY': yy,
'MM': mm,
'DD': dd
})
opath_yaml = os.path.join(opath_yaml, job_id, ymd)
# 接口文件输出目录创建
if not os.path.isdir(opath_yaml):
os.makedirs(opath_yaml)
# 日志信息
path_log = job_cfg['PATH']['OUT']['log']
path_log = os.path.join(path_log, job_name, job_id, ymd)
name_log = '%s.log' % ymd
# 清理日志
shutil.rmtree(path_log, ignore_errors=True)
vars_log = LogServer(path_log, name_log)
# 清理接口文件
if rewrite:
shutil.rmtree(opath_yaml, ignore_errors=True)
# 遍历文件
flist_l1b = glob.glob('%s/*%s*.HDF' % (ipath_l1b, ymd))
flist_clm = glob.glob('%s/*%s*.HDF' % (ipath_clm, ymd))
flist_geo = glob.glob('%s/*%s*.HDF' % (ipath_geo, ymd))
l1b_nums = len(flist_l1b)
clm_nums = len(flist_clm)
geo_nums = len(flist_geo)
if l1b_nums <= 0:
lstr = log_str2str(job_name, job_id, ymd, '--', '--',
u'l1b file nums: %s' % l1b_nums)
vars_log.error(lstr)
return
if clm_nums <= 0:
lstr = log_str2str(job_name, job_id, ymd, '--', '--',
u'clm file nums: %s' % clm_nums)
vars_log.error(lstr)
return
if geo_nums <= 0:
lstr = log_str2str(job_name, job_id, ymd, '--', '--',
u'geo file nums: %s' % geo_nums)
vars_log.error(lstr)
# 把L1数据放入字典
reg = '.*(\d{8})_(\d{4}).*.HDF'
flist_l1b_dict = dict()
for each in flist_l1b:
file_name = os.path.basename(each)
mreg = re.match(reg, file_name)
if mreg:
key_wd = '{}_{}'.format(mreg.group(1), mreg.group(2))
flist_l1b_dict[key_wd] = each
# 把云检测数据放入字典
reg = '.*(\d{8})_(\d{4}).*.HDF'
flist_clm_dict = dict()
for each in flist_clm:
file_name = os.path.basename(each)
mreg = re.match(reg, file_name)
if mreg:
key_wd = '{}_{}'.format(mreg.group(1), mreg.group(2))
flist_clm_dict[key_wd] = each
# 把GEO数据放入字典
reg = '.*(\d{8})_(\d{4}).*.HDF'
flist_geo_dict = dict()
for each in flist_geo:
file_name = os.path.basename(each)
mreg = re.match(reg, file_name)
if mreg:
key_wd = '{}_{}'.format(mreg.group(1), mreg.group(2))
flist_geo_dict[key_wd] = each
# 交集
sat = job_name.split('_')[0]
if sat in ['FY3D', 'FY3C']:
key_wd_cross = flist_l1b_dict.keys() & flist_geo_dict.keys(
) & flist_clm_dict.keys()
else:
key_wd_cross = flist_l1b_dict.keys() & flist_clm_dict.keys()
# 交集是0的记录日志
key_wd_nums = len(key_wd_cross)
if key_wd_nums <= 0:
lstr = log_str2str(job_name, job_id, ymd, '--', '--',
u'cross time nums: %s' % key_wd_nums)
vars_log.error(lstr)
return
for key in sorted(key_wd_cross):
ymd = key.split('_')[0]
hms = key.split('_')[1] + '00'
full_opath_yaml = os.path.join(opath_yaml, '%s%s.yaml' % (ymd, hms))
if os.path.isfile(opath_yaml):
lstr = log_str2str(job_name, job_id, ymd, hms, 'skip', '--')
vars_log.info(lstr)
continue
ifile_l1b = flist_l1b_dict.get(key)
ifile_clm = flist_clm_dict.get(key)
ifile_geo = flist_geo_dict.get(key)
# 输出接口文件
yaml_dict = {
'INFO': {
'job_name': job_name,
'ymd': ymd,
'hms': hms,
'rewrite': rewrite
},
'PATH': {
'ipath_l1b': ifile_l1b,
'ipath_geo': ifile_geo,
'ipath_clm': ifile_clm,
'opath': opath_granule
},
'CALFILE': {
'visfile': visfile,
'irfile': irfile
}
}
try:
write_yaml_file(yaml_dict, full_opath_yaml)
lstr = log_str2str(job_name, job_id, ymd, hms, 'sucess', '--')
vars_log.info(lstr)
except Exception as e:
lstr = log_str2str(job_name, job_id, ymd, hms, 'failed', str(e))
vars_log.info(lstr)