def main(sat_sensor, in_file):
"""
对L3数据进行合成
:param sat_sensor: 卫星+传感器
:param in_file: yaml 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 加载全局配置信息
# 加载程序配置信息
# 加载卫星配置信息
# ######################## 开始处理 ###########################
print "-" * 100
print "Start plot combine map."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
combine = CombineL3() # 初始化一个合成实例
combine.load_yaml(in_file) # 加载 yaml 文件
with time_block("One combine time", switch=TIME_TEST):
combine.combine()
with time_block("One write time", switch=TIME_TEST):
combine.write()
if not combine.error:
print ">>> {}".format(combine.ofile)
else:
print "Error: Combine days error: {}".format(in_file)
print '-' * 100
def main(sat_sensor, in_file):
"""
:param sat_sensor: (str) 卫星对
:param in_file: (str) 输入文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 全局配置接口
out_path = gc.path_mid_ncep
# 程序配置接口
# 卫星配置接口
suffix = sc.ncep2byte_filename_suffix
ncep_table = sc.ncep2byte_ncep_table
######################### 开始处理 ###########################
print "-" * 100
print "Start ncep to byte."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
ymdhm = _get_ymdhm(in_file)
out_file = _get_out_file(in_file, out_path, suffix)
# 判断是否使用新命令进行处理
if int(ymdhm) >= 201501221200:
new = True
else:
new = False
if pb_io.is_none(in_file, out_file, ncep_table):
log.error("Error: {}".format(in_file))
return
ncep2byte = Ncep2Byte(in_file, out_file, new=new, ncep_table=ncep_table)
ncep2byte.ncep2byte()
if not ncep2byte.error:
print ">>> {}".format(ncep2byte.out_file)
else:
log.error("Error: {}".format(in_file))
print "-" * 100
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
cfgFile = os.path.join(ProjPath, 'cfg', 'global.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print (u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
ORBIT_DIR = inCfg['PATH']['IN']['ORBIT']
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
DMS_DIR = inCfg['PATH']['OUT']['DMS']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 获取开机线程的个数，开启线程池。
threadNum = inCfg['CROND']['threads']
pool = Pool(processes=int(threadNum))
if len(args) == 2:
Log.info(u'手动蝴蝶图绘制程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
# 定义参数List，传参给线程池
args_List = []
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
sys.exit(-1)
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
cfgFile = os.path.join(MainPath, 'cfg', 'global.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print (u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
PARAM_DIR = inCfg['PATH']['PARAM']
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
if len(args) == 2:
Log.info(u'手动日合成程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
run(satPair, ymd)
date_s = date_s + relativedelta(days=1)
else:
print 'args: FY3A+MERSI_AQUA+MODIS yyyymmdd-yyyymmdd '
def main(sat_sensor, in_file):
"""
绘制 HDF5.dataset 的快视图。支持真彩图和灰度图
:param sat_sensor: 卫星+传感器
:param in_file: HDF5 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
log = LogServer(gc.path_out_log)
# 加载全局配置信息
# 加载程序配置信息
# 加载卫星配置信息
dataset = sc.plt_quick_view_rgb_dataset
rgb_suffix = sc.plt_quick_view_rgb_suffix
colorbar_range = sc.plt_quick_view_img_colorbar_range
log10_set = sc.plt_quick_view_img_log10_set
log10_ticks = sc.plt_quick_view_img_log10_ticks
log10_tick_labels = sc.plt_quick_view_img_log10_tick_labels
# ######################## 开始处理 ###########################
print '-' * 100
print "Start plot quick view picture."
if not os.path.isfile(in_file):
log.error("File not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
in_file_name = os.path.splitext(in_file)[0]
# 绘制真彩图
out_picture = "{}_{}.{}".format(in_file_name, rgb_suffix, "png")
# 如果文件已经存在，跳过
if not os.path.isfile(out_picture):
r_set, g_set, b_set = dataset
rgb = RGB(in_file, r_set, g_set, b_set, out_picture)
rgb.plot()
if not rgb.error:
print ">>> {}".format(out_picture)
print '-' * 100
else:
print "Error: Plot RGB error: {}".format(in_file)
else:
print "File is already exist, skip it: {}".format(out_picture)
# 绘制热度图
for legend in colorbar_range:
dataset_name = legend[0] # 数据集名称
vmax = float(legend[1]) # color bar 范围 最大值
vmin = float(legend[2]) # color bar 范围 最小值
out_picture = "{}_{}.{}".format(in_file_name, dataset_name.replace("Aod", "AOD"), "png")
# 如果文件已经存在，跳过
if os.path.isfile(out_picture):
print "File is already exist, skip it: {}".format(out_picture)
continue
heat_map = {
"vmin": vmin,
"vmax": vmax,
"cmap": "jet",
"fill_value": -32767,
}
if dataset_name in log10_set:
if dataset_name == "Ocean_TSM":
log10_ticks.append(2.00)
log10_tick_labels.append("100")
if dataset_name == "Ocean_YS443":
log10_ticks = log10_ticks[:-3]
log10_tick_labels = log10_tick_labels[:-3]
heat_map["colorbar_ticks"] = log10_ticks
heat_map["colorbar_tick_label"] = log10_tick_labels
lats, lons = get_lats_lons(in_file)
lat_lon_text = _get_lat_lon_text(lats, lons)
if is_none(lats, lons):
lat_lon_line = None
else:
lat_lon_line = {
"lats": lats, # 经度数据集名称
"lons": lons, # 维度数据集名称
"step": 5.0, # 线密度
"line_width": 0.01,
"text": lat_lon_text,
}
quick_view = QuickView(in_file, dataset_name, out_picture, main_view=heat_map,
lat_lon_line=lat_lon_line)
quick_view.plot()
if not quick_view.error:
print ">>> {}".format(quick_view.out_picture)
else:
print "Error: Plot heat view error: {}".format(in_file)
print '-' * 100
sys.exit(-1)
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
# cfgName = MainFile.split('.')[0]
cfgFile = os.path.join(MainPath, 'dm_sno.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print (u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
LogPath = inCfg['PATH']['LOG'] # log存放地址
Log = LogServer(LogPath)
# 手动执行跟2个参数，卫星明和时间段
if len(args) == 2:
satName = args[0] # 卫星全名
str_time = args[1] # 程序执行时间范围
date_s, date_e = pb_time.arg_str2date(str_time)
Log.info(u'手动轨迹计算程序开始运行-----------------------------')
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
run(satName, ymd)
date_s = date_s + relativedelta(days=1)
# 自动执行不跟参数，卫星明和时间段从配置中获取
elif len(args) == 0:
else:
Log.error(
"%s : the Length of sat_time and sat_list are not the same!"
% s1.sat)
'''
input: 配置文件的路径，时间段
'''
args = sys.argv[1:]
if len(args) == 2:
str_conf = args[0]
str_time = args[1]
if not os.path.isfile(str_conf):
print(u'配置文件不存在 %s' % str_conf)
sys.exit(-1)
OdmCfg = ConfigObj(str_conf)
day_counts = int(OdmCfg['ORBIT']['ORBIT_DAYS']) # 轨迹算几天，预报算几天
date_s, date_e = pb_time.arg_str2date(str_time)
date_e = date_e + relativedelta(days=(day_counts - 1))
else: # 跟参数，则处理输入的时段数据
print 'args: cfgFile yyyymmdd-yyyymmdd'
sys.exit(-1)
LogPath = OdmCfg['PATH']['LOG']
Log = LogServer(LogPath)
Log.info(u'交叉点预报程序开始运行-----------------------------')
run(date_s, date_e)
print HELP_INFO
sys.exit(-1)
# 获取程序所在位置，拼接配置文件
MAIN_PATH, MAIN_FILE = os.path.split(os.path.realpath(__file__))
CONFIG_FILE = os.path.join(MAIN_PATH, "cfg", "global.cfg")
PYTHON_PATH = os.environ.get("PYTHONPATH")
DV_PATH = os.path.join(PYTHON_PATH, "DV")
# 配置不存在预警
if not os.path.isfile(CONFIG_FILE):
print (u"配置文件不存在 %s" % CONFIG_FILE)
sys.exit(-1)
# 载入配置文件
GLOBAL_CONFIG = ConfigObj(CONFIG_FILE)
MATCH_DIR = GLOBAL_CONFIG['PATH']['MID']['MATCH_DATA']
MBA_DIR = GLOBAL_CONFIG['PATH']['OUT']['MBA']
LOG_PATH = GLOBAL_CONFIG["PATH"]["OUT"]["LOG"]
LOG = LogServer(LOG_PATH)
if len(ARGS) == 2:
PAIR = ARGS[0]
YMD = ARGS[1]
run(PAIR, YMD)
else:
print HELP_INFO
sys.exit(-1)
omPath = os.path.dirname(ProjPath)
dvPath = os.path.join(os.path.dirname(omPath), 'DV')
cfgFile = os.path.join(ProjPath, 'cfg', 'global.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
ABR_DIR = inCfg['PATH']['OUT']['ABR']
OMB_DIR = inCfg['PATH']['OUT']['OMB']
ABC_DIR = inCfg['PATH']['OUT']['ABC']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 开启进程池
threadNum = inCfg['CROND']['threads']
pool = Pool(processes=int(threadNum))
if len(args) == 2:
Log.info(u'手动长时间序列绘图程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_start, date_end = pb_time.arg_str2date(str_time)
run(satPair, date_start, date_end)
Log.info(u'SUCCESS，手动长时间序列绘图程序-----------------------------')
elif len(args) == 1:
Log.info(u'手动长时间序列绘图程序开始运行 -----------------------------')
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
cfgFile = os.path.join(ProjPath, 'cfg', 'global.cfg')
nrtcCfg = ConfigObj(os.path.join(MainPath, 'FY2X_NRTC.attr'))
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
PARAM_DIR = inCfg['PATH']['PARAM']
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 开启进程池
threadNum = int(inCfg['CROND']['threads'])
pool = Pool(processes=threadNum)
if len(args) == 2:
Log.info(u'手动NRTC产品生成程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
if threadNum == 1:
run(satPair, ymd)
def main(sat_sensor, in_file):
"""
绘制 L3 产品的全球投影图。
:param sat_sensor: 卫星+传感器
:param in_file: HDF5 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 加载全局配置信息
# 加载程序配置信息
# 加载卫星配置信息
colorbar_range = sc.plt_combine_colorbar_range
area_range = sc.plt_combine_area_range
plot_global = sc.plt_combine_plot_global
plot_china = sc.plt_combine_plot_china
log10_ticks = sc.plt_combine_log10_ticks
log10_tick_labels = sc.plt_combine_log10_tick_label
log10_set = sc.plt_combine_log10_set
# ######################## 开始处理 ###########################
print "-" * 100
print "Start plot combine map."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
sat, sensor = sat_sensor.split("+")
for legend in colorbar_range:
print "*" * 100
dataset_name, name, vmax, vmin, colorbar_label = legend
vmax = float(vmax) # color bar 范围 最大值
vmin = float(vmin) # color bar 范围 最小值
dir_path = os.path.dirname(in_file)
ymd = _get_ymd(in_file)
kind = _get_kind(in_file)
ymd_date = datetime.strptime(ymd, "%Y%m%d")
if dataset_name in log10_set:
_ticks = log10_ticks
_tick_labels = log10_tick_labels
if dataset_name == "Ocean_TSM":
_ticks.append(2.00)
_tick_labels.append("100")
if dataset_name == "Ocean_YS443":
_ticks = _ticks[:-3]
_tick_labels = _tick_labels[:-3]
else:
_ticks = None
_tick_labels = None
png = "{}_{}_{}_{}.png".format(sat_sensor,
dataset_name.replace("Aod", "AOD"), ymd,
kind)
title = _get_title(sat, sensor, name, kind, ymd_date)
plot_map = {
"title": title,
"legend": {
"vmax": vmax,
"vmin": vmin,
"label": colorbar_label,
"ticks": _ticks,
"tick_labels": _tick_labels
},
"area_range": area_range,
"lat_lon_line": {
"delat": 30,
"delon": 30,
},
}
# 画全球范围
if plot_global.lower() == "on":
pic_name = os.path.join(dir_path, "picture_global", png)
# 如果输出文件已经存在，跳过
if os.path.isfile(pic_name):
print "File is already exist, skip it: {}".format(pic_name)
else:
if dataset_name in log10_set:
plot_map["log10"] = True
with time_block("Draw combine time:", switch=TIME_TEST):
plot_map_global = PlotMapL3(in_file,
dataset_name,
pic_name,
map_=plot_map)
plot_map_global.draw_combine()
if not plot_map_global.error:
print ">>> {}".format(plot_map_global.out_file)
else:
print "Error: Plot global picture error: {}".format(
in_file)
# 单画中国区域
if plot_china.lower() == "on":
pic_name = os.path.join(dir_path, "picture_china", png)
# 如果输出文件已经存在，跳过
if os.path.isfile(pic_name):
print "File is already exist, skip it: {}".format(pic_name)
else:
area_range_china = {
"lat_s": "56",
"lat_n": "2",
"lon_w": "65",
"lon_e": "150",
}
lat_lon_line = {
"delat": 10,
"delon": 10,
}
plot_map["area_range"] = area_range_china
plot_map["lat_lon_line"] = lat_lon_line
with time_block("Draw combine time:", switch=TIME_TEST):
plot_map_china = PlotMapL3(in_file,
dataset_name,
pic_name,
map_=plot_map)
plot_map_china.draw_combine()
if not plot_map_china.error:
print ">>> {}".format(plot_map_china.out_file)
else:
print "Error: Plot china picture error: {}".format(in_file)
print '-' * 100
# 程序全局入口
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
# cfgName = MainFile.split('.')[0]
cfgFile = os.path.join(MainPath, 'dm_sno.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
LogPath = inCfg['PATH']['LOG'] # log存放地址
Log = LogServer(LogPath)
# 获取命令行参数
args = sys.argv[1:]
help_info = \
u'''
【参数1】：卫星全名
【参数2】：yyyymmdd-yyyymmdd
'''
if '-h' in args:
print help_info
sys.exit(-1)
# 手动执行跟2个参数，卫星明和时间段
if len(args) == 2:
Log.info(u'手动交叉预报程序开始运行-----------------------------')
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
# cfgName = MainFile.split('.')[0]
cfgFile = os.path.join(MainPath, 'dm_odm.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
# 初始化日志
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 读取需要的信息
CROSS_DIR = inCfg['PATH']['IN']['CROSS']
SNOX_DIR = inCfg['PATH']['IN']['SNOX']
ORDER_DIR = inCfg['PATH']['OUT']['ORDER']
MVREC_DIR = inCfg['PATH']['OUT']['MVREC']
ID_DIR = inCfg['PATH']['OUT']['ID']
MAIL_HOST = inCfg['MAIL']['MAIL_HOST']
MAIL_USER = inCfg['MAIL']['MAIL_USER']
MAIL_PAWD = inCfg['MAIL']['MAIL_PAWD']
use_proxy = inCfg['PROXY']['use_proxy']
PROXY_HOST = inCfg['PROXY']['host']
PROXY_PORT = int(inCfg['PROXY']['port'])
def main(sat_sensor, in_file):
"""
使用矫正系数对 MERSI L1 的产品进行定标预处理
:param sat_sensor: (str) 卫星对
:param in_file: (str) 输入文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 全局配置接口
obc_path = gc.path_in_obc
geo_path = gc.path_in_geo
coeff_path = gc.path_in_coeff
out_path = gc.path_mid_calibrate
# 程序配置接口
# 卫星配置接口
launch_date = sc.launch_date
probe_count = sc.calibrate_probe_count
probe = sc.calibrate_probe
slide_step = sc.calibrate_slide_step
plot = sc.calibrate_plot
# ######################## 开始处理 ###########################
print '-' * 100
print 'Start calibration'
# 不同的卫星对使用不同的处理类和流程
if "fy3b" in sat_sensor.lower() and 'mersi' in sat_sensor.lower():
# 获取 M1000 文件和对应 OBC 文件
l1_1000m = in_file
obc_1000m = _get_obc_file(l1_1000m, obc_path)
if not os.path.isfile(l1_1000m):
log.error("File is not exist: {}".format(l1_1000m))
return
elif not os.path.isfile(obc_1000m):
log.error("File is not exist: {}".format(obc_1000m))
return
else:
print "<<< {}".format(l1_1000m)
print "<<< {}".format(obc_1000m)
ymd = _get_ymd(l1_1000m)
# 获取 coefficient 水色波段系统定标系数， 2013年以前和2013年以后不同
coeff_file = os.path.join(coeff_path, '{}.txt'.format(ymd[0:4]))
if not os.path.isfile(coeff_file):
log.error("File is not exist: {}".format(coeff_file))
return
else:
print "<<< {}".format(coeff_file)
# 获取输出文件
out_path = pb_io.path_replace_ymd(out_path, ymd)
_name = os.path.basename(l1_1000m)
out_file = os.path.join(out_path, _name)
# 如果输出文件已经存在，跳过预处理
if os.path.isfile(out_file):
print "File is already exist, skip it: {}".format(out_file)
return
# 初始化一个预处理实例
calibrate = CalibrateFY3B(l1_1000m=l1_1000m,
obc_1000m=obc_1000m,
coeff_file=coeff_file,
out_file=out_file,
launch_date=launch_date)
# 对 OBC 文件进行 SV 提取
calibrate.obc_sv_extract_fy3b(probe=probe,
probe_count=probe_count,
slide_step=slide_step)
# 重新定标 L1 数据
calibrate.calibrate()
# 将新数据写入 HDF5 文件
calibrate.write()
if not calibrate.error:
print ">>> {}".format(calibrate.out_file)
# 对原数据和处理后的数据各出一张真彩图
if plot == "on":
if not calibrate.error:
picture_suffix = "650_565_490"
file_name = os.path.splitext(out_file)[0]
out_pic_old = "{}_{}_old.{}".format(
file_name, picture_suffix, "png")
out_pic_new = "{}_{}_new.{}".format(
file_name, picture_suffix, "png")
# 如果输出文件已经存在，跳过
if os.path.isfile(out_pic_old):
print "File is already exist, skip it: {}".format(
out_pic_old)
return
else:
_plot_rgb_fy3b_old(in_file, out_pic_old)
if os.path.isfile(out_pic_new):
print "File is already exist, skip it: {}".format(
out_pic_new)
return
else:
_plot_rgb_fy3b_new(out_file, out_pic_new)
else:
print "Error: Calibrate error".format(in_file)
elif "fy3d" in sat_sensor.lower() and 'mersi' in sat_sensor.lower():
# 获取 M1000 文件和对应 GEO 文件
l1_1000m = in_file
geo_1000m = _get_geo_file(l1_1000m, geo_path)
if not os.path.isfile(l1_1000m):
log.error("File is not exist: {}".format(l1_1000m))
return
elif not os.path.isfile(geo_1000m):
log.error("File is not exist: {}".format(geo_1000m))
return
else:
print "<<< {}".format(l1_1000m)
print "<<< {}".format(geo_1000m)
ymd = _get_ymd(l1_1000m)
# 获取 coefficient 定标系数
coeff_file = os.path.join(coeff_path, '{}.txt'.format(ymd[0:4]))
if not os.path.isfile(coeff_file):
log.error("File is not exist: {}".format(coeff_file))
coeff_file = None
else:
print "<<< {}".format(coeff_file)
# 获取输出文件
out_path = pb_io.path_replace_ymd(out_path, ymd)
_name = os.path.basename(l1_1000m)
out_file = os.path.join(out_path, _name)
# 如果输出文件已经存在，跳过预处理
if os.path.isfile(out_file):
print "File is already exist, skip it: {}".format(out_file)
return
# 初始化一个预处理实例
calibrate = CalibrateFY3D()
# 重新定标 L1 数据
calibrate.calibrate(l1_1000m, geo_1000m, coeff_file)
# 将新数据写入 HDF5 文件
calibrate.write(out_file)
if not calibrate.error:
print ">>> {}".format(out_file)
if plot == "on":
if not calibrate.error:
picture_suffix = "670_555_490"
file_name = os.path.splitext(out_file)[0]
out_pic_old = "{}_{}_old.{}".format(
file_name, picture_suffix, "png")
out_pic_new = "{}_{}_new.{}".format(
file_name, picture_suffix, "png")
# 如果输出文件已经存在，跳过
if os.path.isfile(out_pic_old):
print "File is already exist, skip it: {}".format(
out_pic_old)
return
else:
pass
# _plot_rgb_fy3b_old(in_file, out_pic_old)
if os.path.isfile(out_pic_new):
print "File is already exist, skip it: {}".format(
out_pic_new)
return
else:
print out_file
_plot_rgb_fy3d_new(out_file, out_pic_new)
else:
print "***Error***: Calibrate error".format(in_file)
else:
print "***Error***: Don`t handle this sat and sensor: {}".format(
sat_sensor)
return
print '-' * 100
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
cfgFile = os.path.join(ProjPath, 'cfg', 'global.cfg')
omdFile = os.path.join(ProjPath, 'cfg', 'dm_odm.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
MBA_DIR = inCfg['PATH']['OUT']['MBA']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 获取开机线程的个数，开启线程池。
threadNum = inCfg['CROND']['threads']
pool = Pool(processes=int(threadNum))
if len(args) == 2:
Log.info(u'手动月统计绘图程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
pool.apply_async(run, (satPair, ymd))
date_s = date_s + relativedelta(months=1)
def main(sat_sensor, in_file):
"""
对L2数据进行合成
:param sat_sensor: 卫星+传感器
:param in_file: yaml 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 加载全局配置信息
# 加载程序配置信息
# 加载卫星配置信息
res = sc.project_res
half_res = deg2meter(res) / 2.
cmd = sc.project_cmd % (half_res, half_res)
row = sc.project_row
col = sc.project_col
combine_quick = sc.combine_quick
# ######################## 开始处理 ###########################
print "-" * 100
print "Start combine."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
if not combine_quick:
combine = CombineL2() # 初始化一个投影实例
combine.load_cmd_info(cmd=cmd, res=res, row=row, col=col)
combine.load_yaml(in_file) # 加载 yaml 文件
with time_block("One combine time:", switch=TIME_TEST):
combine.combine()
else:
combine = CombineL2Quick() # 初始化一个投影实例
combine.load_cmd_info(cmd=cmd, res=res, row=row, col=col)
combine.load_yaml(in_file) # 加载 yaml 文件
with time_block("One combine time:", switch=TIME_TEST):
combine.combine()
with time_block("One write time:", switch=TIME_TEST):
combine.write()
if not combine.error:
print ">>> {}".format(combine.ofile)
else:
print "Error: Combine day error: {}".format(in_file)
print "-" * 100
def main(sat_sensor, in_file):
"""
对L3数据进行合成
:param sat_sensor: 卫星+传感器
:param in_file: yaml 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
yc = Config(in_file)
log = LogServer(gc.path_out_log)
# 加载全局配置信息
sat_sensor1 = sat_sensor.split('_')[1]
sat_sensor2 = sat_sensor.split('_')[0]
sat1, sensor1 = sat_sensor1.split('+')
sat2, sensor2 = sat_sensor2.split('+')
# 加载卫星配置信息
s_channel1 = sc.name
s_channel2 = sc.name
timseries_channels_config = sc.timeseries_l2_channels
# 加载业务配置信息
# ######################## 开始处理 ###########################
print "-" * 100
print "Start plot verify result."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
all_files = yc.path_ipath
# 加载数据
data_absolute = dict()
data_relative = dict()
date = dict()
ref_s1_all = dict()
ref_s2_all = dict()
amount_all = dict()
date_start = ymd2date(yc.info_ymd_s)
date_end = ymd2date(yc.info_ymd_e)
point_count_min = 2
result = dict()
while date_start <= date_end:
ymd_now = date_start.strftime('%Y%m%d')
in_files = get_one_day_files(all_files=all_files, ymd=ymd_now, ext='.h5',
pattern_ymd=r'.*_(\d{8})')
cross_data = ReadCrossDataL2()
cross_data.read_cross_data(in_files=in_files)
# 循环通道数据
for channel in cross_data.data:
if channel not in data_absolute:
data_absolute[channel] = list()
if channel not in data_relative:
data_relative[channel] = list()
if channel not in date:
date[channel] = list()
if channel not in ref_s1_all:
ref_s1_all[channel] = list()
if channel not in ref_s2_all:
ref_s2_all[channel] = list()
if channel not in amount_all:
amount_all[channel] = list()
if channel not in result:
result[channel] = dict()
if not isinstance(cross_data.data[channel], dict):
continue
ref_s2 = cross_data.data[channel]['MERSI_FovMean']
fine_count = len(ref_s2)
print '---INFO--- {} Points: {}'.format(channel, fine_count)
if fine_count < point_count_min:
print '***WARNING***Dont have enough point to plot: < {}'.format(point_count_min)
continue
ref_s1 = cross_data.data[channel]['MODIS_FovMean']
# 过滤 3 倍std之外的点
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
min_ref_s2 = mean_ref_s2 - 3 * std_ref_s2
max_ref_s2 = mean_ref_s2 + 3 * std_ref_s2
idx = np.logical_and(ref_s2 >= min_ref_s2, ref_s2 <= max_ref_s2)
ref_s2 = ref_s2[idx]
ref_s1 = ref_s1[idx]
# 计算相对偏差和绝对偏差
bias = Bias()
absolute_bias = bias.absolute_deviation(ref_s1, ref_s2)
relative_bias = bias.relative_deviation(ref_s1, ref_s2)
data_absolute[channel].append(np.mean(absolute_bias))
data_relative[channel].append(np.mean(relative_bias))
date[channel].append(date_start)
mean_absolute = np.nanmean(absolute_bias)
std_absolute = np.nanstd(absolute_bias)
amount_absolute = len(absolute_bias)
median_absolute = np.nanmedian(absolute_bias)
rms_absolute = rms(absolute_bias)
mean_relative = np.nanmean(relative_bias)
std_relative = np.nanstd(relative_bias)
amount_relative = len(relative_bias)
median_relative = np.nanmedian(relative_bias)
rms_relative = rms(relative_bias)
mean_ref_s1 = np.nanmean(ref_s1)
std_ref_s1 = np.nanstd(ref_s1)
amount_ref_s1 = len(ref_s1)
median_ref_s1 = np.nanmedian(ref_s1)
rms_ref_s1 = rms(ref_s1)
ref_s1_all[channel].append(mean_ref_s1)
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
amount_ref_s2 = len(ref_s2)
median_ref_s2 = np.nanmedian(ref_s2)
rms_ref_s2 = rms(ref_s2)
ref_s2_all[channel].append(mean_ref_s2)
amount_all[channel].append(amount_ref_s1)
fix_point = sc.plot_scatter_fix_ref
f025_absolute, f025_relative = get_dif_pdif(ref_s1, ref_s2, fix_point)
result_names = ['Dif_mean', 'Dif_std', 'Dif_median', 'Dif_count',
'Dif_rms', 'Dif_025',
'PDif_mean', 'PDif_std', 'PDif_median', 'PDif_count',
'PDif_rms', 'PDif_025',
'{}_MODIS_mean'.format(channel), '{}_MODIS_std'.format(channel), '{}_MODIS_median'.format(channel), '{}_MODIS_count'.format(channel),
'{}_MODIS_rms'.format(channel),
'{}_MERSI_mean'.format(channel), '{}_MERSI_std'.format(channel), '{}_MERSI_median'.format(channel), '{}_MERSI_count'.format(channel),
'{}_MERSI_rms'.format(channel),
'Date']
datas = [mean_absolute, std_absolute, median_absolute, amount_absolute,
rms_absolute, f025_absolute,
mean_relative, std_relative, median_relative, amount_relative,
rms_relative, f025_relative,
mean_ref_s1, std_ref_s1, median_ref_s1, amount_ref_s1,
rms_ref_s1,
mean_ref_s2, std_ref_s2, median_ref_s2, amount_ref_s2,
rms_ref_s2,
ymd_now]
for result_name, data in zip(result_names, datas):
if result_name not in result[channel]:
result[channel][result_name] = list()
else:
result[channel][result_name].append(data)
date_start = date_start + relativedelta(days=1)
for channel in data_absolute:
plot_config = timseries_channels_config[channel]
dif_y_range = plot_config.get('dif_y_range')
pdif_y_range = plot_config.get('pdif_y_range')
ref_s1_y_range = plot_config.get('ref_s1_y_range')
ref_s2_y_range = plot_config.get('ref_s2_y_range')
count_y_range = plot_config.get('count_y_range')
absolute_bias = data_absolute[channel]
if len(absolute_bias) == 0:
print 'Dont have enough point to plot, is 0: {}'.format(channel)
continue
relative_bias = data_relative[channel]
date_channel = date[channel]
ref_s1_channel = ref_s1_all[channel]
ref_s2_channel = ref_s2_all[channel]
amount_channel = amount_all[channel]
# 绘制时间序列图
channel1 = channel
index_channel1 = s_channel1.index(channel1)
channel2 = s_channel2[index_channel1]
title_series = '{}_{} {}_{} Time Series'.format(sat_sensor1, channel1, sat_sensor2,
channel2)
title_ref_s1 = '{}_{} {} Time Series'.format(channel1, sat_sensor1, channel1)
title_ref_s2 = '{}_{} {} Time Series'.format(channel2, sat_sensor2, channel2)
title_amount = '{}_{} {}_{} Matched Points Count Time Series'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
y_label_series_absolute = 'Dif {}-{}'.format(sensor1, sensor2)
y_label_series_relative = 'PDif ({}/{})-1'.format(sensor1, sensor2)
y_label_ref_s1 = '{}'.format(channel1)
y_label_ref_s2 = '{}'.format(channel2)
y_label_amount = 'Count'
picture_path = yc.path_opath
# 孙凌添加,出两张图,限制Y轴坐标的图和不限制Y轴坐标的图,这里是限制Y轴坐标的图
picture_name_absolute = 'Time_Series_Dif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_relative = 'Time_Series_PDif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_ref_s1 = 'Time_Series_{}_{}.png'.format(sat_sensor1, channel1)
picture_name_ref_s2 = 'Time_Series_{}_{}.png'.format(sat_sensor2, channel2)
picture_name_amount = 'Time_Series_Count_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_file_absolute = os.path.join(picture_path, picture_name_absolute)
picture_file_relative = os.path.join(picture_path, picture_name_relative)
picture_file_ref_s1 = os.path.join(picture_path, picture_name_ref_s1)
picture_file_ref_s2 = os.path.join(picture_path, picture_name_ref_s2)
picture_file_amount = os.path.join(picture_path, picture_name_amount)
plot_time_series(day_data_x=date_channel, day_data_y=absolute_bias,
y_range=dif_y_range,
out_file=picture_file_absolute,
title=title_series, y_label=y_label_series_absolute,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=relative_bias,
y_range=pdif_y_range,
out_file=picture_file_relative,
title=title_series, y_label=y_label_series_relative,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=ref_s1_channel,
y_range=ref_s1_y_range,
out_file=picture_file_ref_s1,
title=title_ref_s1, y_label=y_label_ref_s1,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
zero_line=False)
plot_time_series(day_data_x=date_channel, day_data_y=ref_s2_channel,
y_range=ref_s2_y_range,
out_file=picture_file_ref_s2,
title=title_ref_s2, y_label=y_label_ref_s2,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
zero_line=False)
plot_time_series(day_data_x=date_channel, day_data_y=amount_channel,
y_range=count_y_range,
out_file=picture_file_amount,
title=title_amount, y_label=y_label_amount,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
plot_background=False)
# 孙凌添加,出两张图,限制Y轴坐标的图和不限制Y轴坐标的图,这里是不限制Y轴坐标的图
picture_name_absolute = 'Time_Series_Dif_{}_{}_{}_{}_NL.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_relative = 'Time_Series_PDif_{}_{}_{}_{}_NL.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_ref_s1 = 'Time_Series_{}_{}_NL.png'.format(sat_sensor1, channel1)
picture_name_ref_s2 = 'Time_Series_{}_{}_NL.png'.format(sat_sensor2, channel2)
picture_name_amount = 'Time_Series_Count_{}_{}_{}_{}_NL.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_file_absolute = os.path.join(picture_path, picture_name_absolute)
picture_file_relative = os.path.join(picture_path, picture_name_relative)
picture_file_ref_s1 = os.path.join(picture_path, picture_name_ref_s1)
picture_file_ref_s2 = os.path.join(picture_path, picture_name_ref_s2)
picture_file_amount = os.path.join(picture_path, picture_name_amount)
plot_time_series(day_data_x=date_channel, day_data_y=absolute_bias,
out_file=picture_file_absolute,
title=title_series, y_label=y_label_series_absolute,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=relative_bias,
out_file=picture_file_relative,
title=title_series, y_label=y_label_series_relative,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=ref_s1_channel,
out_file=picture_file_ref_s1,
title=title_ref_s1, y_label=y_label_ref_s1,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=ref_s2_channel,
out_file=picture_file_ref_s2,
title=title_ref_s2, y_label=y_label_ref_s2,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e, )
plot_time_series(day_data_x=date_channel, day_data_y=amount_channel,
out_file=picture_file_amount,
title=title_amount, y_label=y_label_amount,
ymd_start=yc.info_ymd_s, ymd_end=yc.info_ymd_e,
plot_background=False)
# 输出HDF5
hdf5_name = '{}_{}_Dif_PDif_Daily.HDF'.format(sat_sensor1, sat_sensor2)
out_path = yc.path_opath
out_file_hdf5 = os.path.join(out_path, hdf5_name)
make_sure_path_exists(out_path)
write_hdf5(out_file_hdf5, result)
keys = amount_all.keys()
keys.sort()
for channel in keys:
print 'CHANNEL: {} POINT: {}'.format(channel, np.sum(amount_all[channel]))
print '-' * 100
# 输出月的HDF5
# 加载数据
data_absolute = dict()
data_relative = dict()
date = dict()
ref_s1_all = dict()
ref_s2_all = dict()
amount_all = dict()
date_start = ymd2date(yc.info_ymd_s)
date_end = ymd2date(yc.info_ymd_e)
result = dict()
while date_start <= date_end:
ymd_now = date_start.strftime('%Y%m')
in_files = get_one_day_files(all_files=all_files, ymd=ymd_now, ext='.h5',
pattern_ymd=r'.*_(\d{6})')
cross_data = ReadCrossDataL2()
cross_data.read_cross_data(in_files=in_files)
# 循环通道数据
for channel in cross_data.data:
if channel not in data_absolute:
data_absolute[channel] = list()
if channel not in data_relative:
data_relative[channel] = list()
if channel not in date:
date[channel] = list()
if channel not in ref_s1_all:
ref_s1_all[channel] = list()
if channel not in ref_s2_all:
ref_s2_all[channel] = list()
if channel not in amount_all:
amount_all[channel] = list()
if channel not in result:
result[channel] = dict()
ref_s1 = cross_data.data[channel]['MERSI_FovMean']
if len(ref_s1) == 0:
print '{} {} : Dont have enough point, is 0'.format(ymd_now, channel)
continue
ref_s2 = cross_data.data[channel]['MODIS_FovMean']
# 过滤 3 倍std之外的点
mean_ref_s1 = np.nanmean(ref_s1)
std_ref_s1 = np.nanstd(ref_s1)
min_ref_s1 = mean_ref_s1 - 3 * std_ref_s1
max_ref_s1 = mean_ref_s1 + 3 * std_ref_s1
idx = np.logical_and(ref_s1 >= min_ref_s1, ref_s1 <= max_ref_s1)
ref_s1 = ref_s1[idx]
ref_s2 = ref_s2[idx]
# 计算相对偏差和绝对偏差
bias = Bias()
absolute_bias = bias.absolute_deviation(ref_s1, ref_s2)
relative_bias = bias.relative_deviation(ref_s1, ref_s2)
data_absolute[channel].append(np.mean(absolute_bias))
data_relative[channel].append(np.mean(relative_bias))
date[channel].append(date_start)
mean_absolute = np.nanmean(absolute_bias)
std_absolute = np.nanstd(absolute_bias)
amount_absolute = len(absolute_bias)
median_absolute = np.nanmedian(absolute_bias)
rms_absolute = rms(absolute_bias)
mean_relative = np.nanmean(relative_bias)
std_relative = np.nanstd(relative_bias)
amount_relative = len(relative_bias)
median_relative = np.nanmedian(relative_bias)
rms_relative = rms(relative_bias)
mean_ref_s1 = np.nanmean(ref_s1)
std_ref_s1 = np.nanstd(ref_s1)
amount_ref_s1 = len(ref_s1)
median_ref_s1 = np.nanmedian(ref_s1)
rms_ref_s1 = rms(ref_s1)
ref_s1_all[channel].append(mean_ref_s1)
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
amount_ref_s2 = len(ref_s2)
median_ref_s2 = np.nanmedian(ref_s2)
rms_ref_s2 = rms(ref_s2)
ref_s2_all[channel].append(mean_ref_s2)
amount_all[channel].append(amount_ref_s1)
fix_point = sc.plot_scatter_fix_ref
f025_absolute, f025_relative = get_dif_pdif(ref_s1, ref_s2, fix_point)
result_names = ['Dif_mean', 'Dif_std', 'Dif_median', 'Dif_count',
'Dif_rms', 'Dif_025',
'PDif_mean', 'PDif_std', 'PDif_median', 'PDif_count',
'PDif_rms', 'PDif_025',
'{}_MODIS_mean'.format(channel), '{}_MODIS_std'.format(channel), '{}_MODIS_median'.format(channel), '{}_MODIS_count'.format(channel),
'{}_MODIS_rms'.format(channel),
'{}_MERSI_mean'.format(channel), '{}_MERSI_std'.format(channel), '{}_MERSI_median'.format(channel), '{}_MERSI_count'.format(channel),
'{}_MERSI_rms'.format(channel),
'Date']
datas = [mean_absolute, std_absolute, median_absolute, amount_absolute,
rms_absolute, f025_absolute,
mean_relative, std_relative, median_relative, amount_relative,
rms_relative, f025_relative,
mean_ref_s1, std_ref_s1, median_ref_s1, amount_ref_s1,
rms_ref_s1,
mean_ref_s2, std_ref_s2, median_ref_s2, amount_ref_s2,
rms_ref_s2,
ymd_now]
for result_name, data in zip(result_names, datas):
if result_name not in result[channel]:
result[channel][result_name] = list()
else:
result[channel][result_name].append(data)
date_start = date_start + relativedelta(months=1)
# 输出HDF5
hdf5_name = '{}_{}_Dif_PDif_Monthly.HDF'.format(sat_sensor1, sat_sensor2)
out_path = yc.path_opath
out_file_hdf5 = os.path.join(out_path, hdf5_name)
make_sure_path_exists(out_path)
write_hdf5(out_file_hdf5, result)
print '-' * 100
dvPath = os.path.join(os.path.dirname(omPath), "DV")
cfgFile = os.path.join(ProjPath, "cfg", "global.cfg")
# 配置不存在预警
if not os.path.isfile(cfgFile):
print (u"配置文件不存在 %s" % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
MATCH_DIR = inCfg["PATH"]["MID"]["MATCH_DATA"]
DRA_DIR = inCfg["PATH"]["OUT"]["DRA"]
MRA_DIR = inCfg["PATH"]["OUT"]["MRA"]
ABR_DIR = inCfg["PATH"]["OUT"]["ABR"]
LogPath = inCfg["PATH"]["OUT"]["LOG"]
Log = LogServer(LogPath)
# 开启进程池
# threadNum = inCfg["CROND"]["threads"]
threadNum = "10"
pool = Pool(processes=int(threadNum))
if len(args) == 2:
Log.info(u"手动日月回归分析程序开始运行-----------------------------")
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
isMonthly = False
if len(str_time) == 13:
isMonthly = True
timeStep = relativedelta(months=1)
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
# cfgName = MainFile.split('.')[0]
cfgFile = os.path.join(MainPath, 'dm_odm.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
# 初始化日志
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 启动socket服务,防止多实例运行
port = 9803
sserver = SocketServer()
if sserver.createSocket(port) == False:
Log.info(u'----已经有一个实例在实行')
sys.exit(-1)
# 读取目录信息
T_DATA = inCfg['PATH']['OUT']['T_DATA']
MVREC = inCfg['PATH']['OUT']['MVREC']
S_DATA = inCfg['PATH']['OUT']['S_DATA']
Log.info(u'运行数据迁移程序 -----------------------------')
# 获取程序所在位置，拼接配置文件
MAIN_PATH, MAIN_FILE = os.path.split(os.path.realpath(__file__))
CONFIG_FILE = os.path.join(MAIN_PATH, "cfg", "global.cfg")
PYTHON_PATH = os.environ.get("PYTHONPATH")
DV_PATH = os.path.join(PYTHON_PATH, "DV")
# 配置不存在预警
if not os.path.isfile(CONFIG_FILE):
print (u"配置文件不存在 %s" % CONFIG_FILE)
sys.exit(-1)
# 载入配置文件
GLOBAL_CONFIG = ConfigObj(CONFIG_FILE)
PARAM_DIR = GLOBAL_CONFIG['PATH']['PARAM']
MATCH_DIR = GLOBAL_CONFIG['PATH']['MID']['MATCH_DATA']
LogPath = GLOBAL_CONFIG['PATH']['OUT']['LOG']
Log = LogServer(LogPath) # 初始化日志
if len(ARGS) == 2:
satPair = ARGS[0]
str_time = ARGS[1]
for NC_TYPE in ["NRTC", "RAC"]:
Log.info(u'开始运行国际标准%s的生成程序-----------------------------' % NC_TYPE)
run(satPair, str_time, NC_TYPE)
else:
print HELP_INFO
sys.exit(-1)
from PB import pb_space, pb_io, pb_time
from PB.CSC.pb_csc_console import LogServer
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
cfgFile = os.path.join(MainPath, 'cfg', 'global.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 经纬度方向list封装
codes = [
Path.MOVETO,
Path.LINETO,
Path.CLOSEPOLY,
]
TmpLst = [Path.LINETO] * 35
codes[1:-1] = TmpLst
BandLst = ['CH_21', 'CH_22', 'CH_24', 'CH_25']
def run(satPair, ymd):
# 解析配置文件
# 获取程序所在位置，拼接配置文件
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
# cfgName = MainFile.split('.')[0]
cfgFile = os.path.join(MainPath, 'dm_odm.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
# 初始化日志
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 读取配置文件信息
ORDER = inCfg['PATH']['OUT']['ORDER'] # 读取目录信息
MVREC = inCfg['PATH']['OUT']['MVREC']
T_DATA = inCfg['PATH']['OUT']['T_DATA']
# 判断系统的类型
if 'nt' in os.name:
OSTYPE = 'windows'
elif 'posix' in os.name:
OSTYPE = 'linux'
else:
Log.info(u'不识别的系统类型')
sys.exit(-1)
dvPath = os.path.join(os.path.dirname(omPath), 'DV')
cfgFile = os.path.join(ProjPath, 'cfg', 'global.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print (u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
DRA_DIR = inCfg['PATH']['OUT']['DRA']
MRA_DIR = inCfg['PATH']['OUT']['MRA']
ABR_DIR = inCfg['PATH']['OUT']['ABR']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 开启进程池
threadNum = inCfg['CROND']['threads']
pool = Pool(processes=int(threadNum))
if len(args) == 2:
Log.info(u'手动日月回归分析程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
isMonthly = False
if len(str_time) == 13:
isMonthly = True
timeStep = relativedelta(months=1)
elif len(str_time) == 17:
MainPath, MainFile = os.path.split(os.path.realpath(__file__))
ProjPath = os.path.dirname(MainPath)
cfgFile = os.path.join(ProjPath, 'cfg', 'global.cfg')
# 配置不存在预警
if not os.path.isfile(cfgFile):
print(u'配置文件不存在 %s' % cfgFile)
sys.exit(-1)
# 载入配置文件
inCfg = ConfigObj(cfgFile)
MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
ABR_DIR = inCfg['PATH']['OUT']['ABR']
OMB_DIR = inCfg['PATH']['OUT']['OMB']
ABC_DIR = inCfg['PATH']['OUT']['ABC']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 开启进程池
threadNum = inCfg['CROND']['threads']
pool = Pool(processes=int(threadNum))
if len(args) == 2:
Log.info(u'手动长时间序列绘图程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
run(satPair, date_s, date_e)
elif len(args) == 1:
Log.info(u'手动长时间序列绘图程序开始运行 -----------------------------')
satPair = args[0]
# 获取程序所在位置，拼接配置文件
MAIN_PATH, MAIN_FILE = os.path.split(os.path.realpath(__file__))
PROJECT_PATH = os.path.dirname(MAIN_PATH)
CONFIG_FILE = os.path.join(PROJECT_PATH, "cfg", "global.cfg")
PYTHON_PATH = os.environ.get("PYTHONPATH")
DV_PATH = os.path.join(PYTHON_PATH, "DV")
# 配置不存在预警
if not os.path.isfile(CONFIG_FILE):
print (u"配置文件不存在 %s" % CONFIG_FILE)
sys.exit(-1)
GLOBAL_CONFIG = ConfigObj(CONFIG_FILE)
StdNC_DIR = GLOBAL_CONFIG['PATH']['OUT']['ISN']
DBB_DIR = GLOBAL_CONFIG['PATH']['OUT']['DBB']
LogPath = GLOBAL_CONFIG['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
if len(ARGS) == 2:
satPair = ARGS[0]
str_time = ARGS[1]
DATE_START, DATE_END = pb_time.arg_str2date(str_time)
PAIR_1 = GLOBAL_CONFIG['DOUBLE_BIAS'][satPair]['pair1']
PAIR_2 = GLOBAL_CONFIG['DOUBLE_BIAS'][satPair]['pair2']
run(PAIR_1, PAIR_2, DATE_START, DATE_END)
else:
print HELP_INFO
sys.exit(-1)
def main(sat_sensor, in_file):
"""
对L3数据进行合成
:param sat_sensor: 卫星+传感器
:param in_file: yaml 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
yc = Config(in_file)
log = LogServer(gc.path_out_log)
# 加载全局配置信息
sat_sensor1 = sat_sensor.split('_')[1]
sat_sensor2 = sat_sensor.split('_')[0]
sat1, sensor1 = sat_sensor1.split('+')
sat2, sensor2 = sat_sensor2.split('+')
# 加载卫星配置信息
s_channel1 = sc.name
s_channel2 = sc.name
# 加载业务配置信息
# ######################## 开始处理 ###########################
print "-" * 100
print "Start plot verify result."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
in_files = yc.path_ipath
# 加载数据
cross_data = ReadCrossDataL2()
cross_data.read_cross_data(in_files=in_files)
# 循环通道数据
info = {}
for channel in cross_data.data:
point_count_min = 10
if not isinstance(cross_data.data[channel], dict):
continue
ref_s2 = cross_data.data[channel]['MERSI_FovMean']
fine_count = len(ref_s2)
print '---INFO--- {} Points: {}'.format(channel, fine_count)
if fine_count < point_count_min:
print '***WARNING***Dont have enough point to plot: < {}'.format(
point_count_min)
continue
ref_s1 = cross_data.data[channel]['MODIS_FovMean']
lat = cross_data.data[channel]['MERSI_Lats']
lon = cross_data.data[channel]['MERSI_Lons']
# 过滤 3 倍std之外的点
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
min_ref_s2 = mean_ref_s2 - 3 * std_ref_s2
max_ref_s2 = mean_ref_s2 + 3 * std_ref_s2
idx = np.logical_and(ref_s2 >= min_ref_s2, ref_s2 <= max_ref_s2)
ref_s2 = ref_s2[idx]
ref_s1 = ref_s1[idx]
lat = lat[idx]
lon = lon[idx]
# 计算相对偏差和绝对偏差
bias = Bias()
absolute_bias = bias.absolute_deviation(ref_s1, ref_s2)
relative_bias = bias.relative_deviation(ref_s1, ref_s2)
mean_absolute = np.nanmean(absolute_bias)
std_absolute = np.nanstd(absolute_bias)
amount_absolute = len(absolute_bias)
median_absolute = np.nanmedian(absolute_bias)
rms_absolute = rms(absolute_bias)
mean_relative = np.nanmean(relative_bias)
std_relative = np.nanstd(relative_bias)
amount_relative = len(relative_bias)
median_relative = np.nanmedian(relative_bias)
rms_relative = rms(relative_bias)
mean_ref_s1 = np.nanmean(ref_s1)
std_ref_s1 = np.nanstd(ref_s1)
amount_ref_s1 = len(ref_s1)
median_ref_s1 = np.nanmedian(ref_s1)
rms_ref_s1 = rms(ref_s1)
mean_ref_s2 = np.nanmean(ref_s2)
std_ref_s2 = np.nanstd(ref_s2)
amount_ref_s2 = len(ref_s2)
median_ref_s2 = np.nanmedian(ref_s2)
rms_ref_s2 = rms(ref_s2)
################################### #######################################
# 绘制直方图
channel1 = channel
index_channel1 = s_channel1.index(channel1)
channel2 = s_channel2[index_channel1]
title_hist_sat1 = '{}_{} Histogram'.format(sat_sensor1, channel1)
title_hist_sat2 = '{}_{} Histogram'.format(sat_sensor2, channel2)
x_label_hist_sat1 = '{} {}'.format(channel2, sat_sensor1)
x_label_hist_sat2 = '{} {}'.format(channel2, sat_sensor2)
y_label_hist = 'Count'
bins_count = 200
picture_path = yc.path_opath
picture_name_sat1 = 'Histogram_{}_{}.png'.format(sat_sensor1, channel1)
picture_name_sat2 = 'Histogram_{}_{}.png'.format(sat_sensor2, channel2)
annotate_hist_sat1 = {
'left_top': [
'{}@Mean={:.4f}'.format(channel1, mean_ref_s1),
'{}@Std={:.4f}'.format(channel1, std_ref_s1),
'{}@Median={:.4f}'.format(channel1, median_ref_s1),
'{}@RMS={:.4f}'.format(channel1, rms_ref_s1),
'{}@Count={:4d}'.format(channel1, amount_ref_s1),
]
}
annotate_hist_sat2 = {
'left_top': [
'{}@Mean={:.4f}'.format(channel2, mean_ref_s2),
'{}@Std={:.4f}'.format(channel2, std_ref_s2),
'{}@Median={:.4f}'.format(channel2, median_ref_s2),
'{}@RMS={:.4f}'.format(channel2, rms_ref_s2),
'{}@Count={:4d}'.format(channel2, amount_ref_s2),
]
}
picture_file_sat1 = os.path.join(picture_path, picture_name_sat1)
picture_file_sat2 = os.path.join(picture_path, picture_name_sat2)
plot_histogram(
data=ref_s1,
title=title_hist_sat1,
x_label=x_label_hist_sat1,
y_label=y_label_hist,
bins_count=bins_count,
out_file=picture_file_sat1,
ymd_start=yc.info_ymd_s,
ymd_end=yc.info_ymd_e,
annotate=annotate_hist_sat1,
)
plot_histogram(
data=ref_s2,
title=title_hist_sat2,
x_label=x_label_hist_sat2,
y_label=y_label_hist,
bins_count=bins_count,
out_file=picture_file_sat2,
ymd_start=yc.info_ymd_s,
ymd_end=yc.info_ymd_e,
annotate=annotate_hist_sat2,
)
# 绘制回归图
title_regression = '{}_{} {}_{} Diagonal Regression'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
x_label_regression = '{} {}'.format(channel1, sat_sensor1)
y_label_regression = '{} {}'.format(channel2, sat_sensor2)
annotate_regression = {
'left_top': [
'MERSI@Mean={:.4f}'.format(mean_ref_s2),
'MERSI@Std={:.4f}'.format(std_ref_s2),
'MERSI@Median={:.4f}'.format(median_ref_s2),
'MERSI@RMS={:.4f}'.format(rms_ref_s2),
'MERSI@Count={:4d}'.format(amount_ref_s2),
]
}
picture_path = yc.path_opath
picture_name_regression = 'Diagonal_Regression_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_file_regression = os.path.join(picture_path,
picture_name_regression)
plot_regression(data_x=ref_s1,
data_y=ref_s2,
out_file=picture_file_regression,
title=title_regression,
x_label=x_label_regression,
y_label=y_label_regression,
ymd_start=yc.info_ymd_s,
ymd_end=yc.info_ymd_e,
annotate=annotate_regression,
plot_zero=False)
################################### 偏差 #######################################
fix_point = sc.plot_scatter_fix_ref
fix_dif, fix_pdif = get_dif_pdif(ref_s1, ref_s2, fix_point)
annotate_absolute = {
'left_top': [
'Dif@{:.2f}={:.4f}'.format(fix_point, fix_dif),
'Dif@Mean={:.4f}'.format(mean_absolute),
'Dif@Std={:.4f}'.format(std_absolute),
'Dif@Median={:.4f}'.format(median_absolute),
'Dif@RMS={:.4f}'.format(rms_absolute),
'Dif@Count={:4d}'.format(amount_absolute),
]
}
annotate_relative = {
'left_top': [
'PDif@{:.2f}={:.4f}'.format(fix_point, fix_pdif),
'PDif@Mean={:.4f}'.format(mean_relative),
'PDif@Std={:.4f}'.format(std_relative),
'PDif@Median={:.4f}'.format(median_relative),
'PDif@RMS={:.4f}'.format(rms_relative),
'PDif@Count={:4d}'.format(amount_relative),
]
}
# 绘制偏差直方图
channel1 = channel
index_channel1 = s_channel1.index(channel1)
channel2 = s_channel2[index_channel1]
title_hist = '{}_{} {}_{} Histogram'.format(sat_sensor1, channel1,
sat_sensor2, channel2)
x_label_hist_absolute = 'Dif {}-{}'.format(sensor1, sensor2)
x_label_hist_relative = 'PDif ({}/{})-1'.format(sensor1, sensor2)
y_label_hist = 'Count'
bins_count = 200
picture_path = yc.path_opath
picture_name_absolute = 'Histogram_Dif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_relative = 'Histogram_PDif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
annotate_hist_absolute = annotate_absolute
annotate_hist_relative = annotate_relative
picture_file_absolute = os.path.join(picture_path,
picture_name_absolute)
picture_file_relative = os.path.join(picture_path,
picture_name_relative)
plot_histogram(
data=absolute_bias,
title=title_hist,
x_label=x_label_hist_absolute,
y_label=y_label_hist,
bins_count=bins_count,
out_file=picture_file_absolute,
ymd_start=yc.info_ymd_s,
ymd_end=yc.info_ymd_e,
annotate=annotate_hist_absolute,
)
plot_histogram(
data=relative_bias,
title=title_hist,
x_label=x_label_hist_relative,
y_label=y_label_hist,
bins_count=bins_count,
out_file=picture_file_relative,
ymd_start=yc.info_ymd_s,
ymd_end=yc.info_ymd_e,
annotate=annotate_hist_relative,
)
# 绘制偏差散点图
title_scatter = '{}_{} {}_{} Scattergram'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
y_label_scatter_absolute = 'Dif {}-{}'.format(sensor1, sensor2)
y_label_scatter_relative = 'PDif ({}/{})-1'.format(sensor1, sensor2)
x_label_scatter = '{} {}'.format(channel2, sat_sensor2)
annotate_scatter_absolute = annotate_absolute
annotate_scatter_relative = annotate_relative
picture_path = yc.path_opath
picture_name_absolute = 'Scattergram_Dif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_relative = 'Scattergram_PDif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_file_absolute = os.path.join(picture_path,
picture_name_absolute)
picture_file_relative = os.path.join(picture_path,
picture_name_relative)
plot_regression(data_x=ref_s2,
data_y=absolute_bias,
out_file=picture_file_absolute,
title=title_scatter,
x_label=x_label_scatter,
y_label=y_label_scatter_absolute,
ymd_start=yc.info_ymd_s,
ymd_end=yc.info_ymd_e,
annotate=annotate_scatter_absolute,
plot_slope=False,
plot_zero=False)
plot_regression(data_x=ref_s2,
data_y=relative_bias,
out_file=picture_file_relative,
title=title_scatter,
x_label=x_label_scatter,
y_label=y_label_scatter_relative,
ymd_start=yc.info_ymd_s,
ymd_end=yc.info_ymd_e,
annotate=annotate_scatter_relative,
plot_slope=False,
plot_zero=False)
# 绘制偏差全球分布图
title_map_absolute = '{}_{} {}_{} Global Distribution Dif {}-{}'.format(
sat_sensor1, channel1, sat_sensor2, channel2, sensor1, sensor2)
title_map_relative = '{}_{} {}_{} Global Distribution PDif ({}/{})-1'.format(
sat_sensor1, channel1, sat_sensor2, channel2, sensor1, sensor2)
picture_name_absolute = 'Map_Dif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_name_relative = 'Map_PDif_{}_{}_{}_{}.png'.format(
sat_sensor1, channel1, sat_sensor2, channel2)
picture_path = yc.path_opath
picture_file_map_absolute = os.path.join(picture_path,
picture_name_absolute)
picture_file_map_relative = os.path.join(picture_path,
picture_name_relative)
plot_bias_map(lat=lat,
lon=lon,
data=absolute_bias,
out_file=picture_file_map_absolute,
title=title_map_absolute)
plot_bias_map(lat=lat,
lon=lon,
data=relative_bias,
out_file=picture_file_map_relative,
title=title_map_relative,
vmin=-0.2,
vmax=0.2)
keys = info.keys()
keys.sort()
for channel in keys:
print 'CHANNEL: {} POINT: {}'.format(channel, info[channel])
print '-' * 100
def main(sat_sensor, in_file):
"""
对L2数据进行投影，记录投影后的位置信息和数据信息
:param sat_sensor: 卫星+传感器
:param in_file: HDF5 文件
:return:
"""
# ######################## 初始化 ###########################
# 获取程序所在位置，拼接配置文件
app = InitApp(sat_sensor)
if app.error:
print "Load config file error."
return
gc = app.global_config
sc = app.sat_config
log = LogServer(gc.path_out_log)
# 加载全局配置信息
out_path = gc.path_mid_projection
# 加载程序配置信息
# 加载卫星配置信息
res = sc.project_res
half_res = deg2meter(res) / 2.
cmd = sc.project_cmd % (half_res, half_res)
row = sc.project_row
col = sc.project_col
mesh_size = sc.project_mesh_size
# ######################## 开始处理 ###########################
print "-" * 100
print "Start projection."
if not os.path.isfile(in_file):
log.error("File is not exist: {}".format(in_file))
return
print "<<< {}".format(in_file)
with time_block("One project time:", switch=TIME_TEST):
# 生成输出文件的文件名
ymd = _get_ymd(in_file)
hm = _get_hm(in_file)
sat, sensor = sat_sensor.split('+')
out_name = "{}_{}_ORBT_L2_OCC_MLT_NUL_{}_{}_{}.HDF".format(
sat, sensor, ymd, hm, mesh_size.upper())
out_path = pb_io.path_replace_ymd(out_path, ymd)
out_file = os.path.join(out_path, out_name)
# 如果输出文件已经存在，跳过
if os.path.isfile(out_file):
print "File is already exist, skip it: {}".format(out_file)
return
# 开始创建投影查找表
projection = Projection(cmd=cmd, row=row, col=col, res=res)
projection.project(in_file, out_file)
if not projection.error:
print ">>> {}".format(out_file)
else:
print "Error: Projection error: {}".format(in_file)
print "-" * 100