MATCH_DIR = inCfg['PATH']['MID']['MATCH_DATA'] ABR_DIR = inCfg['PATH']['OUT']['ABR'] OMB_DIR = inCfg['PATH']['OUT']['OMB'] ABC_DIR = inCfg['PATH']['OUT']['ABC'] LogPath = inCfg['PATH']['OUT']['LOG'] Log = LogServer(LogPath) # 开启进程池 threadNum = inCfg['CROND']['threads'] pool = Pool(processes=int(threadNum)) if len(args) == 2: Log.info(u'手动长时间序列绘图程序开始运行-----------------------------') satPair = args[0] str_time = args[1] date_s, date_e = pb_time.arg_str2date(str_time) run(satPair, date_s, date_e) elif len(args) == 1: Log.info(u'手动长时间序列绘图程序开始运行 -----------------------------') satPair = args[0] run(satPair, None, None) elif len(args) == 0: Log.info(u'自动长时间序列绘图程序开始运行 -----------------------------') rolldays = inCfg['CROND']['rolldays'] pairLst = inCfg['PAIRS'].keys() # 定义参数List,传参给线程池 args_List = [] for satPair in pairLst: ProjMode1 = len(inCfg['PAIRS'][satPair]['colloc_exe'])
def run(rollday): rollday = rollday ipath = inCfg['ext'][satFlag]['ipath'] mpath = inCfg['ext'][satFlag]['mpath'] ifile = inCfg['ext'][satFlag]['regular'] percent = int(inCfg['ext'][satFlag]['percent']) share = int(inCfg['ext'][satFlag]['share']) window = int(inCfg['ext'][satFlag]['window']) # lanch_date = inCfg['ext'][satFlag]['lanch_date'] # 按天处理 date_s, date_e = pb_time.arg_str2date(str_time) while date_s <= date_e: ymd = date_s.strftime('%Y%m%d') ######### 一、查找当前天滚动后的所有文件 ############## FileLst = dcc_find_file(ipath, ifile, date_s, rollday) ######### 二、读取所rolldays条件内所有文件并对数据累加 ############# dcc = DccDataRead() dcc.FileLst = FileLst dcc.load() if len(FileLst) != 0: ######### 三、计算中值,均值,概率密度 ############################# Dn_mean, Dn_median, Dn_mode = dcc_data_process( dcc.dn[:, :, window], share) Ref_mean, Ref_median, Ref_mode = dcc_data_process( dcc.ref[:, :, window], share) print 'rollday: %s, date: %s' % (rollday, ymd) ######### 四、写入数据,按照通道进行输出,规范dcc输出格式 ########### ######### 写入DN值 for i in range(Dn_mean.shape[0]): band = i + 1 if i >= 4 and 'FY3C+MERSI' in satFlag: band = band + 1 ##### 1、拼接文件名 dnName = 'DCC_%s_DN_CH_%02d_Rolldays_%s_ALL_Daily.txt' % ( satFlag, band, rollday) ##### 2、拼接完整输出文件 dnOutFile = os.path.join(mpath, rollday, dnName) dccFiles = len(FileLst) DnPoints = len(dcc.dn[i, :, window]) ##### 3、拼接文件头和数据体信息 Title = ('%-15s' * 8 + '\n') % ('date', 'Avg', 'Med', 'Mod', 'dccFiles', 'dccPoint', 'dccPrecent', 'dccDim') Data = ('%-15s' + '%-15.6f' * 3 + '%-15d' * 4 + '\n') % (ymd, Dn_mean[i], Dn_median[i], Dn_mode[i], dccFiles, DnPoints, percent, window) ##### 4、写入文件 dcc_data_write(Title, Data, dnOutFile) ######### 写入Ref值 for i in range(Ref_mean.shape[0]): band = i + 1 if i >= 4 and 'FY3C+MERSI' in satFlag: band = band + 1 ##### 1、拼接文件名 refName = 'DCC_%s_REF_CH_%02d_Rolldays_%s_ALL_Daily.txt' % ( satFlag, band, rollday) ##### 2、拼接完整输出文件 refOutFile = os.path.join(mpath, rollday, refName) dccFiles = len(FileLst) RefPoints = len(dcc.ref[i, :, window]) ##### 3、拼接文件头信息 Title = ('%-15s' * 8 + '\n') % ('date', 'Avg', 'Med', 'Mod', 'dccFiles', 'dccPoint', 'dccPrecent', 'dccDim') Data = ('%-15s' + '%-15.6f' * 3 + '%-15d' * 4 + '\n') % (ymd, Ref_mean[i], Ref_median[i], Ref_mode[i], dccFiles, RefPoints, percent, window) ##### 4、写入文件 dcc_data_write(Title, Data, refOutFile) date_s = date_s + relativedelta(days=1) print 'success daily: %s' % rollday # 计算月平均 in_file = os.path.join(mpath, rollday) file_list = get_file_list(in_file, r'.*Daily') for day_file in file_list: out_file = day_file.replace('Daily', 'Monthly') title = ('%-15s' * 8 + '\n') % ('date', 'Avg', 'Med', 'Mod', 'dccFiles', 'dccPoint', 'dccPrecent', 'dccDim') day_datas = load_day_ext(day_file) month_data = month_average(day_datas) with open(out_file, 'w') as f: f.write(title) f.writelines(month_data) print 'success Monthly: %s' % rollday
# 获取程序所在位置,拼接配置文件 MAIN_PATH, MAIN_FILE = os.path.split(os.path.realpath(__file__)) PROJECT_PATH = os.path.dirname(MAIN_PATH) CONFIG_FILE = os.path.join(PROJECT_PATH, "cfg", "global.cfg") PYTHON_PATH = os.environ.get("PYTHONPATH") DV_PATH = os.path.join(PYTHON_PATH, "DV") # 配置不存在预警 if not os.path.isfile(CONFIG_FILE): print (u"配置文件不存在 %s" % CONFIG_FILE) sys.exit(-1) GLOBAL_CONFIG = ConfigObj(CONFIG_FILE) StdNC_DIR = GLOBAL_CONFIG['PATH']['OUT']['ISN'] DBB_DIR = GLOBAL_CONFIG['PATH']['OUT']['DBB'] LogPath = GLOBAL_CONFIG['PATH']['OUT']['LOG'] Log = LogServer(LogPath) if len(ARGS) == 2: satPair = ARGS[0] str_time = ARGS[1] DATE_START, DATE_END = pb_time.arg_str2date(str_time) PAIR_1 = GLOBAL_CONFIG['DOUBLE_BIAS'][satPair]['pair1'] PAIR_2 = GLOBAL_CONFIG['DOUBLE_BIAS'][satPair]['pair2'] run(PAIR_1, PAIR_2, DATE_START, DATE_END) else: print HELP_INFO sys.exit(-1)
def main(): try: opts, _ = getopt.getopt(sys.argv[1:], "hv:j:s:t:", ["version", "help", "job=", "sat=" "time="]) except getopt.GetoptError as err: # print help information and exit: print str(err) # will print something like "option -a not recognized" usage() sys.exit(1) for key, val in opts: if key in ('-v', '--version'): verbose = '1.0.1' print 'Version: %s' % verbose sys.exit() elif key in ("-h", "--help"): usage() sys.exit() elif key in ("-j", "--job"): job_id = val elif key in ("-s", "--sat"): sat_pair = val elif key in ("-t", "--time"): str_time = val else: assert False, "unhandled option" rolldays = cfg_body['CROND']['rolldays'] # 自动或手动的时间处理 if 'AUTO' in str_time: datelist = [] for rdays in rolldays: date_s = (datetime.utcnow() - relativedelta(days=int(rdays))) datelist.append(date_s) date_list = zip(datelist, datelist) else: date_s, date_e = pb_time.arg_str2date(str_time) date_list = zip([date_s], [date_e]) # 自动或是手动卫星的处理,默认书写顺序, 有依赖关系时按顺序书写 if 'ALL' in sat_pair: sat_pair_list = cfg_body['PAIRS'].keys() else: sat_pair_list = [sat_pair] # 卫星对 for sat_pair in sat_pair_list: # 获取该卫星对作业流名称,没写则不做 job_flow_name = cfg_body['PAIRS'][sat_pair]['job_flow'] if len(job_flow_name) == 0: continue # 根据自动或是手动的作业流的处理 if 'ALL' in job_id: # 根据作业流获取作业流ID job_flow_id_list = cfg_body['JOB_FLOW_DEF'][job_flow_name] else: job_flow_id_list = ['job_%s' % job_id] # 分解作业步 for job_id_name in job_flow_id_list: # 根据命令行输入的作业id获取函数模块名字 job_mode = cfg_body['BAND_JOB_MODE'][job_id_name] job_mode = os.path.join(main_path, job_mode) # 多个时间段 依次处理 for date_s, date_e in date_list: # 获取作业需要的参数列表 arg_list = eval(job_id_name)(job_mode, sat_pair, date_s, date_e, job_id_name) # 执行 run_command_parallel(arg_list)
# 获取程序所在位置,拼接配置文件 MAIN_PATH, MAIN_FILE = os.path.split(os.path.realpath(__file__)) CONFIG_FILE = os.path.join(MAIN_PATH, "cfg", "global.cfg") PYTHON_PATH = os.environ.get("PYTHONPATH") DV_PATH = os.path.join(PYTHON_PATH, "DV") # 配置不存在预警 if not os.path.isfile(CONFIG_FILE): print (u"配置文件不存在 %s" % CONFIG_FILE) sys.exit(-1) # 载入配置文件 GLOBAL_CONFIG = ConfigObj(CONFIG_FILE) MATCH_DIR = GLOBAL_CONFIG['PATH']['MID']['MATCH_DATA'] ABR_DIR = GLOBAL_CONFIG['PATH']['OUT']['ABR'] OMB_DIR = GLOBAL_CONFIG['PATH']['OUT']['OMB'] ABC_DIR = GLOBAL_CONFIG['PATH']['OUT']['ABC'] LogPath = GLOBAL_CONFIG['PATH']['OUT']['LOG'] Log = LogServer(LogPath) if len(ARGS) == 2: satPair = ARGS[0] str_time = ARGS[1] date_start, date_end = pb_time.arg_str2date(str_time) run(satPair, date_start, date_end) else: print HELP_INFO sys.exit(-1)
def run(): # 按天处理 date_s, date_e = pb_time.arg_str2date(str_time) date_c = date_s # 初始化 hist 类 hist = HistComm(cfgFile) while date_c <= date_e: ymd = date_c.strftime('%Y%m%d') ym = ymd[0:6] # 加载数据 file_name = hist.regular.replace('%YYYY%MM%DD', ymd) slt_file = os.path.join(hist.ifile, ym, file_name) # 如果文件存在,加载数据 if os.path.isfile(slt_file): hist.load_data(slt_file) else: date_c = date_c + relativedelta(days=1) continue for var in hist.var: for k, ch in enumerate(hist.chan): pic_name = '%s_%s_%s_Histogram.png' % (hist.sat_sensor, var, ch) out_pic = os.path.join(hist.ofile, ym, ymd, pic_name) # 如果目录不存在,创建目录 if not os.path.isdir(os.path.dirname(out_pic)): os.makedirs(os.path.dirname(out_pic)) print k print hist.data.get(var).shape data = hist.data.get(var)[k] # 过滤数据 data = filter_fy3d_data(data, var) if len(data) != 0: print(len(data)) print 'max min', data.max(), data.min() # 计算 Mode hists, bin_edges = np.histogram(data, bins=200) idx = np.argmax(hists) mode = (bin_edges[idx] + bin_edges[idx + 1]) / 2.0 # 配置图片文字 if var == 'DN': name = 'Degradation' else: name = var title = '%s %s %s Histogram' % (hist.sat_sensor, ch, name) xlabel = '%s' % name ylabel = 'Numbers' titledict = { 'title': title, 'xlabel': xlabel, 'ylabel': ylabel } tl_list = [[ '{}: {:.4f}'.format('Mode', mode), ]] # 左边注释 draw_histogram(out_pic, data, titledict=titledict, tl_list=tl_list) print out_pic date_c = date_c + relativedelta(days=1)
def main(): try: opts, args = getopt.getopt(sys.argv[1:], "hv:j:s:t:", ["version", "help", "job=", "sat=" "time="]) except getopt.GetoptError as err: # print help information and exit: print str(err) # will print something like "option -a not recognized" usage() sys.exit(1) for key, val in opts: if key in ('-v', '--version'): verbose = '1.0.1' print 'Version: %s' % verbose sys.exit() elif key in ("-h", "--help"): usage() sys.exit() elif key in ("-j", "--job"): job_id = val elif key in ("-s", "--sat"): sat_pair = val elif key in ("-t", "--time"): str_time = val else: assert False, "unhandled option" date_s, date_e = pb_time.arg_str2date(str_time) date_start = date_s job_exe = inCfg['PAIRS'][sat_pair]['job_%s' % job_id] if len(job_exe) > 0: print 'start %s process ......' % job_id if job_id in ['01', '02', '03', '04']: # 创建job01 ~ job04 作业需要的输入文件 craete_incfg_job_01_04(sat_pair, date_s, date_e, job_id) # 拼接job01 ~ job04 作业的参数命令行 arg_list = get_arglist_job01_04(job_exe, sat_pair, date_start, date_e, job_id) elif job_id in ['07']: arg_list = get_arglist_job07(job_exe, sat_pair, date_s, date_e, job_id) elif job_id in ['08']: arg_list = get_arglist_job08(job_exe, sat_pair, date_s, date_e, job_id) elif job_id in ['09', '10']: arg_list = get_arglist_job09_10(job_exe, sat_pair, date_s, date_e, job_id) elif job_id in ['11']: arg_list = get_arglist_job11(job_exe, sat_pair, date_s, date_e, job_id) # 运行所有参数 run_command(arg_list) else: print 'job_%s not process ......' % job_id
def run(rollday): try: sat1, sat2 = satFlag.split('_') except ValueError: sat1 = satFlag sat2 = None rollday = rollday ipath = inCfg['plt'][satFlag]['ipath'] opath = inCfg['plt'][satFlag]['opath'] lanch_date = inCfg['plt'][satFlag]['lanch_date'] var = inCfg['plt'][satFlag]['var'] band = inCfg['plt'][satFlag]['band'] ref_range = inCfg['plt'][satFlag]['REF_range'] dn_range = inCfg['plt'][satFlag]['DN_range'] date_s, date_e = pb_time.arg_str2date(str_time) # 拼接需要读取的文件 if not isinstance(var, list): var = [var] print(var) for each in var: for k, ch in enumerate(band): print each, ch # 日数据 filename_day = 'DCC_%s_%s_%s_Rolldays_%s_ALL_Daily.txt' % ( satFlag, each, ch, rollday) dcc_file_day = os.path.join(ipath, rollday, filename_day) # 月数据 filename_month = 'DCC_%s_%s_%s_Rolldays_%s_ALL_Monthly.txt' % ( satFlag, each, ch, rollday) dcc_file_month = os.path.join(ipath, rollday, filename_month) ######### 一、读取dcc提取后的标准文件 ############## ary_day = dcc_data_read(dcc_file_day) ary_month = dcc_data_read(dcc_file_month) ######### 二、计算衰减(日数据) ############## date_day = ary_day['date'] idx = np.where( ary_day['date'] == datetime.strptime(lanch_date, '%Y%m%d')) first_avg = ary_day['avg'][idx] first_med = ary_day['med'][idx] first_mod = ary_day['mod'][idx] # 如果是 DN,需要计算和发星第一天的相对百分比 if 'DN' in each: avg_day = ary_day['avg'] / first_avg med_day = ary_day['med'] / first_med mod_day = ary_day['mod'] / first_mod elif 'REF' in each: # 如果是绘制相对偏差,已经是计算好的相对百分比,不需要除100 # sat2 存在,代表是绘制两颗卫星的相对偏差 if sat2: avg_day = ary_day['avg'] med_day = ary_day['med'] mod_day = ary_day['mod'] # 如果是绘制自身 REF 变化,需要除 100,还原 REF 真实值 else: avg_day = ary_day['avg'] / 100. med_day = ary_day['med'] / 100. mod_day = ary_day['mod'] / 100. else: print 'error: %s is not supported.' % each return print 'max: avg med mod', \ avg_day.max(), med_day.max(), mod_day.max() print 'min: avg med mod', \ avg_day.min(), med_day.min(), mod_day.min() datas_day = { 'avg_day': avg_day, 'med_day': med_day, 'mod_day': mod_day, } ######### 三、格式化数据(月数据) ############## date_month = ary_month['date'] + relativedelta(days=14) # 如果是 DN,需要计算和发星第一天的相对百分比 if 'DN' in each: avg_month = ary_month['avg'] / first_avg med_month = ary_month['med'] / first_med mod_month = ary_month['mod'] / first_mod elif 'REF' in each: # 如果是绘制相对偏差,已经是计算好的相对百分比,不需要除100 # sat2 存在,代表是绘制两颗卫星的相对偏差 if sat2: avg_month = ary_month['avg'] med_month = ary_month['med'] mod_month = ary_month['mod'] # 如果是绘制自身 REF 变化,需要除 100,还原 REF 真实值 else: avg_month = ary_month['avg'] / 100. med_month = ary_month['med'] / 100. mod_month = ary_month['mod'] / 100. else: print 'error: %s is not supported.' % each return print 'max: avg med mod', avg_month.max(), \ med_month.max(), mod_month.max() print 'min: avg med mod', avg_month.min(), \ med_month.min(), mod_month.min() datas_month = { 'avg_month': avg_month, 'med_month': med_month, 'mod_month': mod_month, } ######## 四、绘图 ################### # 绘图 y 轴的数据范围 if each == 'REF': y_range = ref_range else: y_range = dn_range min_yaxis = float(y_range[k].split('_')[0]) max_yaxis = float(y_range[k].split('_')[1]) data_types = ['avg', 'med', 'mod'] for data_type in data_types: ymd_e = date_e.strftime('%Y%m%d') outPng = os.path.join( opath, rollday, ymd_e, 'DCC_%s_%s_%s_Rolldays_%s_Timeseries_%s.png' % (satFlag, each, ch, rollday, data_type)) date_D = date_day date_M = date_month if sat2: title = '%s Minus %s %s TimeSeries' % (sat1, sat2, ch) else: title = '%s %s TimeSeries' % (sat1, ch) data_D_name = '%s_day' % data_type data_D = datas_day.get(data_D_name) data_M_name = '%s_month' % data_type data_M = datas_month.get(data_M_name) name = '' if each == 'DN': name = 'Degradation' # DN 的图像上面 ylabel 名字 elif each == 'REF': if sat2: name = 'Bias' # Bias 的图像上面 ylabel 名字 else: name = 'REF' # REF 的图像上面 ylabel 名字 plot_bias(date_D, data_D, date_M, data_M, outPng, title, date_s, date_e, name, data_type, min_yaxis, max_yaxis) print(outPng)