MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA']
ABR_DIR = inCfg['PATH']['OUT']['ABR']
OMB_DIR = inCfg['PATH']['OUT']['OMB']
ABC_DIR = inCfg['PATH']['OUT']['ABC']
LogPath = inCfg['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
# 开启进程池
threadNum = inCfg['CROND']['threads']
pool = Pool(processes=int(threadNum))
if len(args) == 2:
Log.info(u'手动长时间序列绘图程序开始运行-----------------------------')
satPair = args[0]
str_time = args[1]
date_s, date_e = pb_time.arg_str2date(str_time)
run(satPair, date_s, date_e)
elif len(args) == 1:
Log.info(u'手动长时间序列绘图程序开始运行 -----------------------------')
satPair = args[0]
run(satPair, None, None)
elif len(args) == 0:
Log.info(u'自动长时间序列绘图程序开始运行 -----------------------------')
rolldays = inCfg['CROND']['rolldays']
pairLst = inCfg['PAIRS'].keys()
# 定义参数List，传参给线程池
args_List = []
for satPair in pairLst:
ProjMode1 = len(inCfg['PAIRS'][satPair]['colloc_exe'])
def run(rollday):
rollday = rollday
ipath = inCfg['ext'][satFlag]['ipath']
mpath = inCfg['ext'][satFlag]['mpath']
ifile = inCfg['ext'][satFlag]['regular']
percent = int(inCfg['ext'][satFlag]['percent'])
share = int(inCfg['ext'][satFlag]['share'])
window = int(inCfg['ext'][satFlag]['window'])
# lanch_date = inCfg['ext'][satFlag]['lanch_date']
# 按天处理
date_s, date_e = pb_time.arg_str2date(str_time)
while date_s <= date_e:
ymd = date_s.strftime('%Y%m%d')
######### 一、查找当前天滚动后的所有文件 ##############
FileLst = dcc_find_file(ipath, ifile, date_s, rollday)
######### 二、读取所rolldays条件内所有文件并对数据累加 #############
dcc = DccDataRead()
dcc.FileLst = FileLst
dcc.load()
if len(FileLst) != 0:
######### 三、计算中值，均值，概率密度 #############################
Dn_mean, Dn_median, Dn_mode = dcc_data_process(
dcc.dn[:, :, window], share)
Ref_mean, Ref_median, Ref_mode = dcc_data_process(
dcc.ref[:, :, window], share)
print 'rollday: %s, date: %s' % (rollday, ymd)
######### 四、写入数据，按照通道进行输出，规范dcc输出格式 ###########
######### 写入DN值
for i in range(Dn_mean.shape[0]):
band = i + 1
if i >= 4 and 'FY3C+MERSI' in satFlag:
band = band + 1
##### 1、拼接文件名
dnName = 'DCC_%s_DN_CH_%02d_Rolldays_%s_ALL_Daily.txt' % (
satFlag, band, rollday)
##### 2、拼接完整输出文件
dnOutFile = os.path.join(mpath, rollday, dnName)
dccFiles = len(FileLst)
DnPoints = len(dcc.dn[i, :, window])
##### 3、拼接文件头和数据体信息
Title = ('%-15s' * 8 + '\n') % ('date', 'Avg', 'Med', 'Mod',
'dccFiles', 'dccPoint',
'dccPrecent', 'dccDim')
Data = ('%-15s' + '%-15.6f' * 3 + '%-15d' * 4 +
'\n') % (ymd, Dn_mean[i], Dn_median[i], Dn_mode[i],
dccFiles, DnPoints, percent, window)
##### 4、写入文件
dcc_data_write(Title, Data, dnOutFile)
######### 写入Ref值
for i in range(Ref_mean.shape[0]):
band = i + 1
if i >= 4 and 'FY3C+MERSI' in satFlag:
band = band + 1
##### 1、拼接文件名
refName = 'DCC_%s_REF_CH_%02d_Rolldays_%s_ALL_Daily.txt' % (
satFlag, band, rollday)
##### 2、拼接完整输出文件
refOutFile = os.path.join(mpath, rollday, refName)
dccFiles = len(FileLst)
RefPoints = len(dcc.ref[i, :, window])
##### 3、拼接文件头信息
Title = ('%-15s' * 8 + '\n') % ('date', 'Avg', 'Med', 'Mod',
'dccFiles', 'dccPoint',
'dccPrecent', 'dccDim')
Data = ('%-15s' + '%-15.6f' * 3 + '%-15d' * 4 +
'\n') % (ymd, Ref_mean[i], Ref_median[i], Ref_mode[i],
dccFiles, RefPoints, percent, window)
##### 4、写入文件
dcc_data_write(Title, Data, refOutFile)
date_s = date_s + relativedelta(days=1)
print 'success daily: %s' % rollday
# 计算月平均
in_file = os.path.join(mpath, rollday)
file_list = get_file_list(in_file, r'.*Daily')
for day_file in file_list:
out_file = day_file.replace('Daily', 'Monthly')
title = ('%-15s' * 8 + '\n') % ('date', 'Avg', 'Med', 'Mod',
'dccFiles', 'dccPoint', 'dccPrecent',
'dccDim')
day_datas = load_day_ext(day_file)
month_data = month_average(day_datas)
with open(out_file, 'w') as f:
f.write(title)
f.writelines(month_data)
print 'success Monthly: %s' % rollday
# 获取程序所在位置，拼接配置文件
MAIN_PATH, MAIN_FILE = os.path.split(os.path.realpath(__file__))
PROJECT_PATH = os.path.dirname(MAIN_PATH)
CONFIG_FILE = os.path.join(PROJECT_PATH, "cfg", "global.cfg")
PYTHON_PATH = os.environ.get("PYTHONPATH")
DV_PATH = os.path.join(PYTHON_PATH, "DV")
# 配置不存在预警
if not os.path.isfile(CONFIG_FILE):
print (u"配置文件不存在 %s" % CONFIG_FILE)
sys.exit(-1)
GLOBAL_CONFIG = ConfigObj(CONFIG_FILE)
StdNC_DIR = GLOBAL_CONFIG['PATH']['OUT']['ISN']
DBB_DIR = GLOBAL_CONFIG['PATH']['OUT']['DBB']
LogPath = GLOBAL_CONFIG['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
if len(ARGS) == 2:
satPair = ARGS[0]
str_time = ARGS[1]
DATE_START, DATE_END = pb_time.arg_str2date(str_time)
PAIR_1 = GLOBAL_CONFIG['DOUBLE_BIAS'][satPair]['pair1']
PAIR_2 = GLOBAL_CONFIG['DOUBLE_BIAS'][satPair]['pair2']
run(PAIR_1, PAIR_2, DATE_START, DATE_END)
else:
print HELP_INFO
sys.exit(-1)
def main():
try:
opts, _ = getopt.getopt(sys.argv[1:], "hv:j:s:t:",
["version", "help", "job=", "sat="
"time="])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(1)
for key, val in opts:
if key in ('-v', '--version'):
verbose = '1.0.1'
print 'Version: %s' % verbose
sys.exit()
elif key in ("-h", "--help"):
usage()
sys.exit()
elif key in ("-j", "--job"):
job_id = val
elif key in ("-s", "--sat"):
sat_pair = val
elif key in ("-t", "--time"):
str_time = val
else:
assert False, "unhandled option"
rolldays = cfg_body['CROND']['rolldays']
# 自动或手动的时间处理
if 'AUTO' in str_time:
datelist = []
for rdays in rolldays:
date_s = (datetime.utcnow() - relativedelta(days=int(rdays)))
datelist.append(date_s)
date_list = zip(datelist, datelist)
else:
date_s, date_e = pb_time.arg_str2date(str_time)
date_list = zip([date_s], [date_e])
# 自动或是手动卫星的处理,默认书写顺序, 有依赖关系时按顺序书写
if 'ALL' in sat_pair:
sat_pair_list = cfg_body['PAIRS'].keys()
else:
sat_pair_list = [sat_pair]
# 卫星对
for sat_pair in sat_pair_list:
# 获取该卫星对作业流名称,没写则不做
job_flow_name = cfg_body['PAIRS'][sat_pair]['job_flow']
if len(job_flow_name) == 0:
continue
# 根据自动或是手动的作业流的处理
if 'ALL' in job_id:
# 根据作业流获取作业流ID
job_flow_id_list = cfg_body['JOB_FLOW_DEF'][job_flow_name]
else:
job_flow_id_list = ['job_%s' % job_id]
# 分解作业步
for job_id_name in job_flow_id_list:
# 根据命令行输入的作业id获取函数模块名字
job_mode = cfg_body['BAND_JOB_MODE'][job_id_name]
job_mode = os.path.join(main_path, job_mode)
# 多个时间段 依次处理
for date_s, date_e in date_list:
# 获取作业需要的参数列表
arg_list = eval(job_id_name)(job_mode, sat_pair, date_s,
date_e, job_id_name)
# 执行
run_command_parallel(arg_list)
# 获取程序所在位置，拼接配置文件
MAIN_PATH, MAIN_FILE = os.path.split(os.path.realpath(__file__))
CONFIG_FILE = os.path.join(MAIN_PATH, "cfg", "global.cfg")
PYTHON_PATH = os.environ.get("PYTHONPATH")
DV_PATH = os.path.join(PYTHON_PATH, "DV")
# 配置不存在预警
if not os.path.isfile(CONFIG_FILE):
print (u"配置文件不存在 %s" % CONFIG_FILE)
sys.exit(-1)
# 载入配置文件
GLOBAL_CONFIG = ConfigObj(CONFIG_FILE)
MATCH_DIR = GLOBAL_CONFIG['PATH']['MID']['MATCH_DATA']
ABR_DIR = GLOBAL_CONFIG['PATH']['OUT']['ABR']
OMB_DIR = GLOBAL_CONFIG['PATH']['OUT']['OMB']
ABC_DIR = GLOBAL_CONFIG['PATH']['OUT']['ABC']
LogPath = GLOBAL_CONFIG['PATH']['OUT']['LOG']
Log = LogServer(LogPath)
if len(ARGS) == 2:
satPair = ARGS[0]
str_time = ARGS[1]
date_start, date_end = pb_time.arg_str2date(str_time)
run(satPair, date_start, date_end)
else:
print HELP_INFO
sys.exit(-1)
def run():
# 按天处理
date_s, date_e = pb_time.arg_str2date(str_time)
date_c = date_s
# 初始化 hist 类
hist = HistComm(cfgFile)
while date_c <= date_e:
ymd = date_c.strftime('%Y%m%d')
ym = ymd[0:6]
# 加载数据
file_name = hist.regular.replace('%YYYY%MM%DD', ymd)
slt_file = os.path.join(hist.ifile, ym, file_name)
# 如果文件存在，加载数据
if os.path.isfile(slt_file):
hist.load_data(slt_file)
else:
date_c = date_c + relativedelta(days=1)
continue
for var in hist.var:
for k, ch in enumerate(hist.chan):
pic_name = '%s_%s_%s_Histogram.png' % (hist.sat_sensor, var,
ch)
out_pic = os.path.join(hist.ofile, ym, ymd, pic_name)
# 如果目录不存在，创建目录
if not os.path.isdir(os.path.dirname(out_pic)):
os.makedirs(os.path.dirname(out_pic))
print k
print hist.data.get(var).shape
data = hist.data.get(var)[k]
# 过滤数据
data = filter_fy3d_data(data, var)
if len(data) != 0:
print(len(data))
print 'max min', data.max(), data.min()
# 计算 Mode
hists, bin_edges = np.histogram(data, bins=200)
idx = np.argmax(hists)
mode = (bin_edges[idx] + bin_edges[idx + 1]) / 2.0
# 配置图片文字
if var == 'DN':
name = 'Degradation'
else:
name = var
title = '%s %s %s Histogram' % (hist.sat_sensor, ch, name)
xlabel = '%s' % name
ylabel = 'Numbers'
titledict = {
'title': title,
'xlabel': xlabel,
'ylabel': ylabel
}
tl_list = [[
'{}: {:.4f}'.format('Mode', mode),
]] # 左边注释
draw_histogram(out_pic,
data,
titledict=titledict,
tl_list=tl_list)
print out_pic
date_c = date_c + relativedelta(days=1)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "hv:j:s:t:",
["version", "help", "job=", "sat="
"time="])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(1)
for key, val in opts:
if key in ('-v', '--version'):
verbose = '1.0.1'
print 'Version: %s' % verbose
sys.exit()
elif key in ("-h", "--help"):
usage()
sys.exit()
elif key in ("-j", "--job"):
job_id = val
elif key in ("-s", "--sat"):
sat_pair = val
elif key in ("-t", "--time"):
str_time = val
else:
assert False, "unhandled option"
date_s, date_e = pb_time.arg_str2date(str_time)
date_start = date_s
job_exe = inCfg['PAIRS'][sat_pair]['job_%s' % job_id]
if len(job_exe) > 0:
print 'start %s process ......' % job_id
if job_id in ['01', '02', '03', '04']:
# 创建job01 ~ job04 作业需要的输入文件
craete_incfg_job_01_04(sat_pair, date_s, date_e, job_id)
# 拼接job01 ~ job04 作业的参数命令行
arg_list = get_arglist_job01_04(job_exe, sat_pair, date_start,
date_e, job_id)
elif job_id in ['07']:
arg_list = get_arglist_job07(job_exe, sat_pair, date_s, date_e,
job_id)
elif job_id in ['08']:
arg_list = get_arglist_job08(job_exe, sat_pair, date_s, date_e,
job_id)
elif job_id in ['09', '10']:
arg_list = get_arglist_job09_10(job_exe, sat_pair, date_s, date_e,
job_id)
elif job_id in ['11']:
arg_list = get_arglist_job11(job_exe, sat_pair, date_s, date_e,
job_id)
# 运行所有参数
run_command(arg_list)
else:
print 'job_%s not process ......' % job_id
def run(rollday):
try:
sat1, sat2 = satFlag.split('_')
except ValueError:
sat1 = satFlag
sat2 = None
rollday = rollday
ipath = inCfg['plt'][satFlag]['ipath']
opath = inCfg['plt'][satFlag]['opath']
lanch_date = inCfg['plt'][satFlag]['lanch_date']
var = inCfg['plt'][satFlag]['var']
band = inCfg['plt'][satFlag]['band']
ref_range = inCfg['plt'][satFlag]['REF_range']
dn_range = inCfg['plt'][satFlag]['DN_range']
date_s, date_e = pb_time.arg_str2date(str_time)
# 拼接需要读取的文件
if not isinstance(var, list):
var = [var]
print(var)
for each in var:
for k, ch in enumerate(band):
print each, ch
# 日数据
filename_day = 'DCC_%s_%s_%s_Rolldays_%s_ALL_Daily.txt' % (
satFlag, each, ch, rollday)
dcc_file_day = os.path.join(ipath, rollday, filename_day)
# 月数据
filename_month = 'DCC_%s_%s_%s_Rolldays_%s_ALL_Monthly.txt' % (
satFlag, each, ch, rollday)
dcc_file_month = os.path.join(ipath, rollday, filename_month)
######### 一、读取dcc提取后的标准文件 ##############
ary_day = dcc_data_read(dcc_file_day)
ary_month = dcc_data_read(dcc_file_month)
######### 二、计算衰减(日数据) ##############
date_day = ary_day['date']
idx = np.where(
ary_day['date'] == datetime.strptime(lanch_date, '%Y%m%d'))
first_avg = ary_day['avg'][idx]
first_med = ary_day['med'][idx]
first_mod = ary_day['mod'][idx]
# 如果是 DN，需要计算和发星第一天的相对百分比
if 'DN' in each:
avg_day = ary_day['avg'] / first_avg
med_day = ary_day['med'] / first_med
mod_day = ary_day['mod'] / first_mod
elif 'REF' in each:
# 如果是绘制相对偏差，已经是计算好的相对百分比，不需要除100
# sat2 存在，代表是绘制两颗卫星的相对偏差
if sat2:
avg_day = ary_day['avg']
med_day = ary_day['med']
mod_day = ary_day['mod']
# 如果是绘制自身 REF 变化，需要除 100，还原 REF 真实值
else:
avg_day = ary_day['avg'] / 100.
med_day = ary_day['med'] / 100.
mod_day = ary_day['mod'] / 100.
else:
print 'error: %s is not supported.' % each
return
print 'max: avg med mod', \
avg_day.max(), med_day.max(), mod_day.max()
print 'min: avg med mod', \
avg_day.min(), med_day.min(), mod_day.min()
datas_day = {
'avg_day': avg_day,
'med_day': med_day,
'mod_day': mod_day,
}
######### 三、格式化数据(月数据) ##############
date_month = ary_month['date'] + relativedelta(days=14)
# 如果是 DN，需要计算和发星第一天的相对百分比
if 'DN' in each:
avg_month = ary_month['avg'] / first_avg
med_month = ary_month['med'] / first_med
mod_month = ary_month['mod'] / first_mod
elif 'REF' in each:
# 如果是绘制相对偏差，已经是计算好的相对百分比，不需要除100
# sat2 存在，代表是绘制两颗卫星的相对偏差
if sat2:
avg_month = ary_month['avg']
med_month = ary_month['med']
mod_month = ary_month['mod']
# 如果是绘制自身 REF 变化，需要除 100，还原 REF 真实值
else:
avg_month = ary_month['avg'] / 100.
med_month = ary_month['med'] / 100.
mod_month = ary_month['mod'] / 100.
else:
print 'error: %s is not supported.' % each
return
print 'max: avg med mod', avg_month.max(), \
med_month.max(), mod_month.max()
print 'min: avg med mod', avg_month.min(), \
med_month.min(), mod_month.min()
datas_month = {
'avg_month': avg_month,
'med_month': med_month,
'mod_month': mod_month,
}
######## 四、绘图 ###################
# 绘图 y 轴的数据范围
if each == 'REF':
y_range = ref_range
else:
y_range = dn_range
min_yaxis = float(y_range[k].split('_')[0])
max_yaxis = float(y_range[k].split('_')[1])
data_types = ['avg', 'med', 'mod']
for data_type in data_types:
ymd_e = date_e.strftime('%Y%m%d')
outPng = os.path.join(
opath, rollday, ymd_e,
'DCC_%s_%s_%s_Rolldays_%s_Timeseries_%s.png' %
(satFlag, each, ch, rollday, data_type))
date_D = date_day
date_M = date_month
if sat2:
title = '%s Minus %s %s TimeSeries' % (sat1, sat2, ch)
else:
title = '%s %s TimeSeries' % (sat1, ch)
data_D_name = '%s_day' % data_type
data_D = datas_day.get(data_D_name)
data_M_name = '%s_month' % data_type
data_M = datas_month.get(data_M_name)
name = ''
if each == 'DN':
name = 'Degradation' # DN 的图像上面 ylabel 名字
elif each == 'REF':
if sat2:
name = 'Bias' # Bias 的图像上面 ylabel 名字
else:
name = 'REF' # REF 的图像上面 ylabel 名字
plot_bias(date_D, data_D, date_M, data_M, outPng, title,
date_s, date_e, name, data_type, min_yaxis,
max_yaxis)
print(outPng)
