def find_model(self, model_type, lead, init_time): model_dir = self.p.getstr('config', model_type + '_INPUT_DIR') # max_forecast = self.p.getint('config', model_type+'_MAX_FORECAST') forecasts = model_type + '_FORECASTS' max_forecast = util.getlistint(self.p.getstr('config', forecasts))[-1] init_interval = self.p.getint('config', model_type + '_INIT_INTERVAL') lead_check = lead time_check = init_time time_offset = 0 found = False while lead_check <= max_forecast: native_template = self.p.getraw('filename_templates', model_type + '_NATIVE_TEMPLATE') model_ss = sts.StringSub(self.logger, native_template, init=time_check, lead=str(lead_check).zfill(2)) model_file = model_ss.doStringSub() print("model file: " + model_file) model_path = os.path.join(model_dir, model_file) if os.path.exists(model_path): found = True break time_check = util.shift_time(time_check, -init_interval) lead_check = lead_check + init_interval if found: return model_path else: return ''
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST') if self.p.has_option('config', 'FCST_MIN_FORECAST'): min_forecast = self.p.getint('config', 'FCST_MIN_FORECAST') else: min_forecast = 0 #lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) if init_time == -1: #Create a list of files to loop over gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ')) init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL') valid_hr = int(valid_time[8:10]) #Find lead times lead_seq = [] for gs in gen_seq: if valid_hr >= gs: current_lead = valid_hr - gs elif valid_hr < gs: current_lead = valid_hr + gs while current_lead <= max_forecast: lead_seq.append(current_lead) current_lead = current_lead + 24 for lead in lead_seq: if lead < min_forecast: continue task_info.lead = lead for var_info in var_list: self.run_at_time_once(task_info, var_info)
def run_at_time(self, init_time): task_info = TaskInfo() task_info.init_time = init_time fcst_vars = util.getlist(self.p.getstr('config', 'FCST_VARS')) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) for lead in lead_seq: task_info.lead = lead for fcst_var in fcst_vars: task_info.fcst_var = fcst_var # loop over models to compare accums = util.getlist( self.p.getstr('config', fcst_var + "_ACCUM")) ob_types = util.getlist( self.p.getstr('config', fcst_var + "_OBTYPE")) for accum in accums: task_info.level = accum for ob_type in ob_types: task_info.ob_type = ob_type if lead < int(accum): continue # self.run_at_time_fcst(task_info) self.run_at_time_once(task_info.getValidTime(), task_info.level, task_info.ob_type, task_info.fcst_var)
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) for lead in lead_seq: task_info.lead = lead for var_info in var_list: self.run_at_time_once(task_info, var_info)
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) for lead in lead_seq: task_info.lead = lead task_info.valid_time = -1 for var_info in var_list: level = var_info.obs_level if level[0].isalpha(): level = var_info.obs_level[1:] self.run_at_time_once(task_info.getValidTime(), level, var_info.obs_name)
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) # want to combine fcst data files to get total accum matching obs? # obs_level = self.p.getstr('config', 'OBS_LEVEL') fcst_level = self.p.getstr('config', 'FCST_LEVEL') # TODO: should use getpath or something? in_dir = self.p.getdir('FCST_PCP_COMBINE_INPUT_DIR') in_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_INPUT_TEMPLATE') out_dir = self.p.getdir('FCST_PCP_COMBINE_OUTPUT_DIR') out_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_OUTPUT_TEMPLATE') for lead in lead_seq: task_info.lead = lead for var_info in var_list: out_level = var_info.obs_level if out_level[0].isalpha(): out_level = out_level[1:] if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \ self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD": self.run_add_method(task_info.getValidTime(), task_info.getInitTime(), out_level, var_info.obs_name, "FCST", True) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM": self.run_sum_method(task_info.getValidTime(), task_info.getInitTime(), fcst_level, out_level, in_dir, out_dir, out_template) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUBTRACT": self.run_subtract_method(task_info, var_info, int(out_level), in_dir, out_dir, in_template, out_template) else: self.logger.error("Invalid PCP_COMBINE_METHOD specified") exit(1)
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) model_templates = util.getlist( self.p.getraw('filename_templates', 'FCST_GRID_STAT_INPUT_TEMPLATE')) model_types = util.getlist(self.p.getraw('config', 'MODEL_TYPE')) max_forecasts = util.getlist( self.p.getraw('config', 'FCST_MAX_FORECAST')) self.check_model(model_templates, model_types, max_forecasts) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) for md in range(len(model_templates)): cur_model = [ model_templates[md], model_types[md], max_forecasts[md] ] for lead in lead_seq: task_info.lead = lead for var_info in var_list: self.run_at_time_once(task_info, var_info, cur_model)
def create_c_dict(self): """! Create a data structure (dictionary) that contains all the values set in the configuration files Args: Returns: c_dict - a dictionary containing the settings in the configuration files (that aren't in the metplus_data, metplus_system, and metplus_runtime config files. """ c_dict = super(PB2NCWrapper, self).create_c_dict() c_dict['SKIP_IF_OUTPUT_EXISTS'] = self.config.getbool('config', 'PB2NC_SKIP_IF_OUTPUT_EXISTS', False) c_dict['OFFSETS'] = util.getlistint(self.config.getstr('config', 'PB2NC_OFFSETS', '0')) # Directories c_dict['OBS_INPUT_DIR'] = self.config.getdir('PB2NC_INPUT_DIR') c_dict['OUTPUT_DIR'] = self.config.getdir('PB2NC_OUTPUT_DIR') c_dict['OBS_INPUT_TEMPLATE'] = self.config.getraw('filename_templates', 'PB2NC_INPUT_TEMPLATE') c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('filename_templates', 'PB2NC_OUTPUT_TEMPLATE') c_dict['OBS_INPUT_DATATYPE'] = self.config.getstr('config', 'PB2NC_INPUT_DATATYPE', '') # Configuration c_dict['CONFIG_FILE'] = self.config.getstr('config', 'PB2NC_CONFIG_FILE') c_dict['MESSAGE_TYPE'] = util.getlist( self.config.getstr('config', 'PB2NC_MESSAGE_TYPE', '[]')) tmp_message_type = str(c_dict['MESSAGE_TYPE']).replace("\'", "\"") c_dict['MESSAGE_TYPE'] = ''.join(tmp_message_type) c_dict['STATION_ID'] = util.getlist( self.config.getstr('config', 'PB2NC_STATION_ID', '[]')) tmp_message_type = str(c_dict['STATION_ID']).replace("\'", "\"") c_dict['STATION_ID'] = ''.join(tmp_message_type.split()) grid_id = self.config.getstr('config', 'PB2NC_GRID') if grid_id.startswith('G'): # Reformat grid ids that begin with 'G' ( G10, G1, etc.) to format # Gnnn c_dict['GRID'] = self.reformat_grid_id(grid_id) else: c_dict['GRID'] = grid_id c_dict['POLY'] = self.config.getstr('config', 'PB2NC_POLY') c_dict['BUFR_VAR_LIST'] = util.getlist( self.config.getstr('config', 'PB2NC_OBS_BUFR_VAR_LIST', '[]')) c_dict['TIME_SUMMARY_FLAG'] = self.config.getbool('config', 'PB2NC_TIME_SUMMARY_FLAG') c_dict['TIME_SUMMARY_BEG'] = self.config.getstr('config', 'PB2NC_TIME_SUMMARY_BEG') c_dict['TIME_SUMMARY_END'] = self.config.getstr('config', 'PB2NC_TIME_SUMMARY_END') c_dict['TIME_SUMMARY_VAR_NAMES'] = util.getlist( self.config.getstr('config', 'PB2NC_TIME_SUMMARY_VAR_NAMES')) c_dict['TIME_SUMMARY_TYPES'] = util.getlist( self.config.getstr('config', 'PB2NC_TIME_SUMMARY_TYPES')) c_dict['OBS_WINDOW_BEGIN'] = \ self.config.getseconds('config', 'PB2NC_WINDOW_BEGIN', self.config.getseconds('config', 'OBS_WINDOW_BEGIN', 0)) c_dict['OBS_WINDOW_END'] = \ self.config.getseconds('config', 'PB2NC_WINDOW_END', self.config.getseconds('config', 'OBS_WINDOW_END', 0)) c_dict['OBS_FILE_WINDOW_BEGIN'] = \ self.config.getseconds('config', 'PB2NC_FILE_WINDOW_BEGIN', self.config.getseconds('config', 'OBS_FILE_WINDOW_BEGIN', 0)) c_dict['OBS_FILE_WINDOW_END'] = \ self.config.getseconds('config', 'PB2NC_FILE_WINDOW_END', self.config.getseconds('config', 'OBS_FILE_WINDOW_END', 0)) c_dict['ALLOW_MULTIPLE_FILES'] = True return c_dict
def grid2grid_sfc_plot_format(self): self.logger.info("Formatting for plotting for grid2grid-sfc") #read config use_init = self.p.getbool('config', 'LOOP_BY_INIT', True) if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_INIT_BEG", start_t) self.add_env_var("FCST_INIT_END", end_t) else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t) self.add_env_var("FCST_VALID_END", end_t) self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR') stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR') var_list = util.parse_var_list(self.p) region_list = util.getlist(self.p.getstr('config', 'REGION_LIST')) lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST')) model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST')) self.add_env_var('INTERP', 'NEAREST') if use_init: loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR') loop_end_hour = self.p.getint('config', 'INIT_END_HOUR') loop_inc = self.p.getint('config', 'INIT_INC') else: loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR') loop_end_hour = self.p.getint('config', 'VALID_END_HOUR') loop_inc = self.p.getint('config', 'VALID_INC') loop_hour = loop_beg_hour while loop_hour <= loop_end_hour: loop_hour_str = str(loop_hour).zfill(2) #filtering times based on if files made based on init_time or valid_time if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_VALID_HOUR", "") self.add_env_var("FCST_INIT_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"') else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"') self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") self.add_env_var("FCST_INIT_HOUR", "") for model in model_list: self.add_env_var('MODEL', model) #build -lookin directory self.set_lookin_dir( os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z', model)) for var_info in var_list: fcst_var_name = var_info.fcst_name fcst_var_level = var_info.fcst_level #fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";") obs_var_name = var_info.obs_name obs_var_level = var_info.obs_level #obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";") self.add_env_var('FCST_VAR_NAME', fcst_var_name) self.add_env_var('FCST_VAR_LEVEL', fcst_var_level) self.add_env_var('OBS_VAR_NAME', obs_var_name) self.add_env_var('OBS_VAR_LEVEL', obs_var_level) for region in region_list: self.add_env_var('REGION', region) for lead in lead_list: if lead < 10: lead_string = '0' + str(lead) else: lead_string = str(lead) self.add_env_var('LEAD', lead_string) if not os.path.exists( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)): os.makedirs( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)) ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat") dump_row_file = os.path.join( stat_analysis_out_dir, loop_hour_str + "Z", model, region, model + "_f" + lead_string + "_fcst" + fcst_var_name + fcst_var_level + "_obs" + obs_var_name + obs_var_level + ".stat") ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat") job = "-job filter -dump_row " + dump_row_file self.add_env_var("JOB", job) #get stat_analysis config file self.set_param_file( self.p.getstr('config', 'STAT_ANALYSIS_CONFIG')) #environment self.logger.debug("") self.logger.debug("ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") self.logger.debug( "COPYABLE ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") #build command cmd = self.get_command() if cmd is None: print( "ERROR: stat_analysis could not generate command" ) return self.logger.info("") self.build() self.clear() loop_hour += loop_inc
def grid2grid_anom_plot_format(self): self.logger.info("Formatting for plotting for grid2grid-anom") #read config use_init = self.p.getbool('config', 'LOOP_BY_INIT', True) if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_INIT_BEG", start_t) self.add_env_var("FCST_INIT_END", end_t) else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t) self.add_env_var("FCST_VALID_END", end_t) self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR') stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR') var_list = util.parse_var_list(self.p) region_list = util.getlist(self.p.getstr('config', 'REGION_LIST')) lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST')) model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST')) if use_init: loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR') loop_end_hour = self.p.getint('config', 'INIT_END_HOUR') loop_inc = self.p.getint('config', 'INIT_INC') else: loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR') loop_end_hour = self.p.getint('config', 'VALID_END_HOUR') loop_inc = self.p.getint('config', 'VALID_INC') loop_hour = loop_beg_hour while loop_hour <= loop_end_hour: loop_hour_str = str(loop_hour).zfill(2) #filtering times based on if files made based on init_time or valid_time if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_VALID_HOUR", "") self.add_env_var("FCST_INIT_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"') else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"') self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") self.add_env_var("FCST_INIT_HOUR", "") for model in model_list: self.add_env_var('MODEL', model) #build -lookin directory self.set_lookin_dir( os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z', model)) for var_info in var_list: fcst_var_name = var_info.fcst_name fcst_var_level = var_info.fcst_level #fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";") obs_var_name = var_info.obs_name obs_var_level = var_info.obs_level #obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";") self.add_env_var('FCST_VAR_NAME', fcst_var_name) self.add_env_var('FCST_VAR_LEVEL', fcst_var_level) self.add_env_var('OBS_VAR_NAME', obs_var_name) self.add_env_var('OBS_VAR_LEVEL', obs_var_level) interp_mthd = [] if var_info.fcst_name == 'HGT' or var_info.obs_name == 'HGT': fourier_decomp_height = self.p.getbool( 'config', 'FOURIER_HEIGHT_DECOMP') if fourier_decomp_height: wave_num_beg_list = util.getlist( self.p.getstr('config', 'WAVE_NUM_BEG_LIST')) wave_num_end_list = util.getlist( self.p.getstr('config', 'WAVE_NUM_END_LIST')) if len(wave_num_beg_list) != len( wave_num_end_list): print( "ERROR: WAVE_NUM_BEG_LIST and WAVE_NUM_END_LIST do not have the same number of elements" ) exit(1) else: interp_mthd.append("NEAREST") for wn in range(len(wave_num_beg_list)): wb = wave_num_beg_list[wn] we = wave_num_end_list[wn] wave_num_pairing = "WV1_" + wb + "-" + we interp_mthd.append(wave_num_pairing) else: interp_mthd.append("NEAREST") else: interp_mthd.append("NEAREST") for region in region_list: self.add_env_var('REGION', region) for lead in lead_list: if lead < 10: lead_string = '0' + str(lead) else: lead_string = str(lead) self.add_env_var('LEAD', lead_string) if not os.path.exists( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)): os.makedirs( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)) for im in interp_mthd: self.add_env_var('INTERP', im) if im == "NEAREST": ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat") dump_row_file = os.path.join( stat_analysis_out_dir, loop_hour_str + "Z", model, region, model + "_f" + lead_string + "_fcst" + fcst_var_name + fcst_var_level + "_obs" + obs_var_name + obs_var_level + ".stat") ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat") else: ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat") dump_row_file = os.path.join( stat_analysis_out_dir, loop_hour_str + "Z", model, region, model + "_f" + lead_string + "_fcst" + fcst_var_name + fcst_var_level + "_obs" + obs_var_name + obs_var_level + "_" + im + ".stat") ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+"_"+im+".stat") job = "-job filter -dump_row " + dump_row_file self.add_env_var("JOB", job) #get stat_analysis config file self.set_param_file( self.p.getstr('config', 'STAT_ANALYSIS_CONFIG')) #environment self.logger.debug("") self.logger.debug( "ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") self.logger.debug( "COPYABLE ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") #build command cmd = self.get_command() if cmd is None: print( "ERROR: stat_analysis could not generate command" ) return self.logger.info("") self.build() self.clear() loop_hour += loop_inc
def grid2grid_pres_plots(self): logging_filename = self.logger.handlers[0].baseFilename self.add_env_var("LOGGING_FILENAME", logging_filename) plotting_scripts_dir = self.p.getdir('PLOTTING_SCRIPTS_DIR') #read config use_init = self.p.getbool('config', 'LOOP_BY_INIT', True) if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR') loop_end_hour = self.p.getint('config', 'INIT_END_HOUR') loop_inc = self.p.getint('config', 'INIT_INC') date_filter_method = "Initialization" self.add_env_var("START_T", start_t) self.add_env_var("END_T", end_t) self.add_env_var("DATE_FILTER_METHOD", date_filter_method) else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR') loop_end_hour = self.p.getint('config', 'VALID_END_HOUR') loop_inc = self.p.getint('config', 'VALID_INC') date_filter_method = "Valid" self.add_env_var("START_T", start_t) self.add_env_var("END_T", end_t) self.add_env_var("DATE_FILTER_METHOD", date_filter_method) stat_files_input_dir = self.p.getdir('STAT_FILES_INPUT_DIR') plotting_out_dir = self.p.getdir('PLOTTING_OUT_DIR') if os.path.exists(plotting_out_dir): self.logger.info(plotting_out_dir + " exist, removing") util.rmtree(plotting_out_dir) region_list = util.getlist(self.p.getstr('config', 'REGION_LIST')) lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST')) model_list = self.p.getstr('config', 'MODEL_LIST') plot_stats_list = self.p.getstr('config', 'PLOT_STATS_LIST') self.add_env_var("STAT_FILES_INPUT_DIR", stat_files_input_dir) self.add_env_var("PLOTTING_OUT_DIR", plotting_out_dir) self.add_env_var("MODEL_LIST", model_list) self.add_env_var("PLOT_STATS_LIST", plot_stats_list) #need to grab var info in special way that differs from util.parse_var_list #need variables with cooresponding list of levels; logic derived from util.parse_var_list var_info_list = [] # find all FCST_VARn_NAME keys in the conf files all_conf = self.p.keys('config') fcst_indices = [] regex = re.compile("FCST_VAR(\d+)_NAME") for conf in all_conf: result = regex.match(conf) if result is not None: fcst_indices.append(result.group(1)) # loop over all possible variables and add them to list for n in fcst_indices: # get fcst var info if available if self.p.has_option('config', "FCST_VAR" + n + "_NAME"): fcst_name = self.p.getstr('config', "FCST_VAR" + n + "_NAME") fcst_extra = "" if self.p.has_option('config', "FCST_VAR" + n + "_OPTIONS"): fcst_extra = self.p.getraw('config', "FCST_VAR" + n + "_OPTIONS") fcst_levels = util.getlist( self.p.getstr('config', "FCST_VAR" + n + "_LEVELS")) # if OBS_VARn_X does not exist, use FCST_VARn_X if self.p.has_option('config', "OBS_VAR" + n + "_NAME"): obs_name = self.p.getstr('config', "OBS_VAR" + n + "_NAME") else: obs_name = fcst_name obs_extra = "" if self.p.has_option('config', "OBS_VAR" + n + "_OPTIONS"): obs_extra = self.p.getraw('config', "OBS_VAR" + n + "_OPTIONS") ##else: ## obs_extra = fcst_extra ##fcst_levels = util.getlist(self.p.getstr('config', "FCST_VAR"+n+"_LEVELS")) if self.p.has_option('config', "OBS_VAR" + n + "_LEVELS"): obs_levels = util.getlist( self.p.getstr('config', "FCST_VAR" + n + "_LEVELS")) else: obs_levels = fcst_levels if len(fcst_levels) != len(obs_levels): print("ERROR: FCST_VAR"+n+"_LEVELS and OBS_VAR"+n+\ "_LEVELS do not have the same number of elements") exit(1) fo = util.FieldObj() fo.fcst_name = fcst_name fo.obs_name = obs_name fo.fcst_extra = fcst_extra fo.obs_extra = obs_extra fo.fcst_level = fcst_levels fo.obs_level = obs_levels var_info_list.append(fo) loop_hour = loop_beg_hour while loop_hour <= loop_end_hour: loop_hour_str = str(loop_hour).zfill(2) self.add_env_var('CYCLE', loop_hour_str) for v in var_info_list: fcst_var_levels_list = v.fcst_level self.add_env_var('FCST_VAR_NAME', v.fcst_name) #self.add_env_var('FCST_VAR_EXTRA', v.fcst_extra) self.add_env_var( 'FCST_VAR_LEVELS_LIST', ''.join(fcst_var_levels_list).replace( "P", " P").lstrip().replace(" P", ", P")) obs_var_levels_list = v.obs_level self.add_env_var('OBS_VAR_NAME', v.obs_name) #self.add_env_var('OBS_VAR_EXTRA', v.obs_extra) self.add_env_var( 'OBS_VAR_LEVELS_LIST', ''.join(obs_var_levels_list).replace( "P", " P").lstrip().replace(" P", ", P")) for region in region_list: self.add_env_var('REGION', region) for lead in lead_list: if lead < 10: lead_string = '0' + str(lead) else: lead_string = str(lead) self.add_env_var('LEAD', lead_string) for vl in range(len(fcst_var_levels_list)): self.add_env_var('FCST_VAR_LEVEL', fcst_var_levels_list[vl]) self.add_env_var('OBS_VAR_LEVEL', obs_var_levels_list[vl]) py_cmd = os.path.join( "python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_pres_ts.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") ####py_cmd = os.path.join("python3")+" "+os.path.join(plotting_scripts_dir, "plot_grid2grid_pres_tp.py") #add python3 at top of script py_cmd = os.path.join("python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_pres_tp.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") self.add_env_var("LEAD_LIST", self.p.getstr('config', 'LEAD_LIST')) py_cmd = os.path.join("python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_pres_tsmean.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") ####py_cmd = os.path.join("python3")+" "+os.path.join(plotting_scripts_dir, "plot_grid2grid_pres_tpmean.py") #add python3 at top of script py_cmd = os.path.join("python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_pres_tpmean.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") loop_hour += loop_inc
def grid2grid_sfc_plots(self): logging_filename = self.logger.handlers[0].baseFilename self.add_env_var("LOGGING_FILENAME", logging_filename) plotting_scripts_dir = self.p.getdir('PLOTTING_SCRIPTS_DIR') #read config use_init = self.p.getbool('config', 'LOOP_BY_INIT', True) if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR') loop_end_hour = self.p.getint('config', 'INIT_END_HOUR') loop_inc = self.p.getint('config', 'INIT_INC') date_filter_method = "Initialization" self.add_env_var("START_T", start_t) self.add_env_var("END_T", end_t) self.add_env_var("DATE_FILTER_METHOD", date_filter_method) else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR') loop_end_hour = self.p.getint('config', 'VALID_END_HOUR') loop_inc = self.p.getint('config', 'VALID_INC') date_filter_method = "Valid" self.add_env_var("START_T", start_t) self.add_env_var("END_T", end_t) self.add_env_var("DATE_FILTER_METHOD", date_filter_method) stat_files_input_dir = self.p.getdir('STAT_FILES_INPUT_DIR') plotting_out_dir = self.p.getdir('PLOTTING_OUT_DIR') if os.path.exists(plotting_out_dir): self.logger.info(plotting_out_dir + " exist, removing") util.rmtree(plotting_out_dir) region_list = util.getlist(self.p.getstr('config', 'REGION_LIST')) lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST')) model_list = self.p.getstr('config', 'MODEL_LIST') plot_stats_list = self.p.getstr('config', 'PLOT_STATS_LIST') self.add_env_var("STAT_FILES_INPUT_DIR", stat_files_input_dir) self.add_env_var("PLOTTING_OUT_DIR", plotting_out_dir) self.add_env_var("MODEL_LIST", model_list) self.add_env_var("PLOT_STATS_LIST", plot_stats_list) var_list = util.parse_var_list(self.p) loop_hour = loop_beg_hour while loop_hour <= loop_end_hour: loop_hour_str = str(loop_hour).zfill(2) self.add_env_var('CYCLE', loop_hour_str) for var_info in var_list: fcst_var_name = var_info.fcst_name fcst_var_level = var_info.fcst_level #fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";") obs_var_name = var_info.obs_name obs_var_level = var_info.obs_level #obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";") self.add_env_var('FCST_VAR_NAME', fcst_var_name) self.add_env_var('FCST_VAR_LEVEL', fcst_var_level) self.add_env_var('OBS_VAR_NAME', obs_var_name) self.add_env_var('OBS_VAR_LEVEL', obs_var_level) for region in region_list: self.add_env_var('REGION', region) for lead in lead_list: if lead < 10: lead_string = '0' + str(lead) else: lead_string = str(lead) self.add_env_var('LEAD', lead_string) py_cmd = os.path.join("python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_sfc_ts.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") self.add_env_var("LEAD_LIST", self.p.getstr('config', 'LEAD_LIST')) py_cmd = os.path.join("python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_sfc_tsmean.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") loop_hour += loop_inc
def test_getlist_int(): l = '6, 7, 42' test_list = util.getlistint(l) assert (test_list == [6, 7, 42])
def main(): logger = logging.getLogger('run_example') init_time = 0 start_time = 0 end_time = 0 time_interval = 1 short_opts = "c:r:h" long_opts = ["config=", "help", "runtime="] # All command line input, get options and arguments try: opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts) except getopt.GetoptError as err: print(str(err)) usage('SCRIPT IS EXITING DUE TO UNRECOGNIZED COMMAND LINE OPTION') for k, v in opts: if k in ('-c', '--config'): # adds the conf file to the list of arguments. args.append(config_launcher.set_conf_file_path(v)) elif k in ('-h', '--help'): usage() exit() elif k in ('-r', '--runtime'): start_time = v end_time = v else: assert False, "UNHANDLED OPTION" if not args: args = None (parm, infiles, moreopt) = config_launcher.parse_launch_args(args, usage, None, logger) p = config_launcher.launch(infiles, moreopt) logger = util.get_logger(p) logger.setLevel(logging.DEBUG) if start_time == 0: start_time = p.getstr('config', 'START_TIME') end_time = p.getstr('config', 'END_TIME') time_interval = p.getstr('config', 'TIME_INTERVAL') # Get the list of processes to call process_list = util.getlist(p.getstr('config', 'PROCESS_LIST')) model_type = p.getstr('config', 'MODEL_TYPE') fcst_vars = util.getlist(p.getstr('config', 'FCST_VARS')) lead_seq = util.getlistint(p.getstr('config', 'LEAD_SEQ')) init_time = start_time while init_time <= end_time: print("") print("****************************************") print("* RUNNING MET+") print("* EVALUATING " + model_type + " at init time: " + init_time) print("****************************************") logger.info("****************************************") logger.info("* RUNNING MET+") logger.info("* EVALUATING " + model_type + " at init time: " + init_time) logger.info("****************************************") for lead in lead_seq: for fcst_var in fcst_vars: # loop over models to compare accums = util.getlist(p.getstr('config', fcst_var + "_ACCUM")) ob_types = util.getlist( p.getstr('config', fcst_var + "_OBTYPE")) for accum in accums: for ob_type in ob_types: if lead < int(accum): continue obs_var = p.getstr('config', ob_type + "_VAR") logger.info("") logger.info("") logger.info("For " + init_time + " F" + str(lead) + ", processing " + model_type + "_" + fcst_var + "_" + accum + " vs " + ob_type + " " + obs_var + "_" + accum) valid_time = util.shift_time(init_time, lead) data_interval = p.getint('config', ob_type + '_DATA_INTERVAL') if int(valid_time[8:10]) % data_interval != 0: logger.warning("No observation for valid time: " + valid_time + ". Skipping...") continue for process in process_list: if process == "pcp_combine": run_pcp = CG_pcp_combine(p, logger) run_pcp.run_at_time(valid_time, accum, ob_type, fcst_var) elif process == "regrid_data_plane": run_regrid = CG_regrid_data_plane(p, logger) run_regrid.run_at_time(valid_time, accum, ob_type) elif process == "grid_stat": run_grid_stat = CG_grid_stat(p, logger) run_grid_stat.run_at_time( init_time, lead, accum, ob_type, fcst_var) else: print("ERROR: Invalid process in process list") exit(1) init_time = util.shift_time(init_time, int(time_interval)) (logger).info("END OF EXECUTION")
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST') if init_time == -1: #Create a list of files to loop over gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ')) init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL') valid_hr = int(valid_time[8:10]) #Find lead times lead_seq = [] for gs in gen_seq: if valid_hr >= gs: current_lead = valid_hr - gs elif valid_hr < gs: current_lead = valid_hr + gs while current_lead <= max_forecast: lead_seq.append(current_lead) current_lead = current_lead + 24 lead_seq = sorted(lead_seq) if valid_time == -1: lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) # want to combine fcst data files to get total accum matching obs? # obs_level = self.p.getstr('config', 'OBS_LEVEL') fcst_level = self.p.getstr('config', 'FCST_LEVEL') # TODO: should use getpath or something? in_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_INPUT_DIR') in_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_INPUT_TEMPLATE') out_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_OUTPUT_DIR') out_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_OUTPUT_TEMPLATE') for lead in lead_seq: task_info.lead = lead for var_info in var_list: out_level = var_info.obs_level if out_level[0].isalpha(): out_level = out_level[1:] if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \ self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD": self.run_add_method(task_info.getValidTime(), task_info.getInitTime(), out_level, var_info.obs_name, "FCST", True) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM": self.run_sum_method(task_info.getValidTime(), task_info.getInitTime(), fcst_level, out_level, in_dir, out_dir, out_template) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUBTRACT": self.run_subtract_method(task_info, var_info, int(out_level), in_dir, out_dir, in_template, out_template) else: self.logger.error("Invalid PCP_COMBINE_METHOD specified") exit(1)
def create_c_dict(self): """! Create a dictionary that holds all the values set in the METplus config file for the point-stat wrapper. Returns: c_dict - A dictionary containing the key-value pairs set in the METplus configuration file. """ c_dict = super(PointStatWrapper, self).create_c_dict() c_dict['ALLOW_MULTIPLE_FILES'] = True c_dict['OFFSETS'] = util.getlistint( self.config.getstr('config', 'POINT_STAT_OFFSETS', '0')) c_dict['FCST_INPUT_TEMPLATE'] = \ self.config.getraw('filename_templates', 'FCST_POINT_STAT_INPUT_TEMPLATE') c_dict['OBS_INPUT_TEMPLATE'] = \ self.config.getraw('filename_templates', 'OBS_POINT_STAT_INPUT_TEMPLATE') c_dict['FCST_INPUT_DATATYPE'] = \ self.config.getstr('config', 'FCST_POINT_STAT_INPUT_DATATYPE', '') c_dict['OBS_INPUT_DATATYPE'] = \ self.config.getstr('config', 'OBS_POINT_STAT_INPUT_DATATYPE', '') c_dict['FCST_INPUT_DIR'] = self.config.getdir( 'FCST_POINT_STAT_INPUT_DIR') c_dict['OBS_INPUT_DIR'] = self.config.getdir( 'OBS_POINT_STAT_INPUT_DIR') c_dict['OUTPUT_DIR'] = \ self.config.getdir('POINT_STAT_OUTPUT_DIR') # Configuration c_dict['CONFIG_FILE'] = \ self.config.getstr('config', 'POINT_STAT_CONFIG_FILE') c_dict['MODEL'] = self.config.getstr('config', 'MODEL') c_dict['POINT_STAT_CONFIG_FILE'] = \ self.config.getstr('config', 'POINT_STAT_CONFIG_FILE') regrid = self.config.getstr('config', 'POINT_STAT_REGRID_TO_GRID') # if not surrounded by quotes and not NONE, add quotes if regrid[0] != '"' and regrid != 'NONE': regrid = '"' + regrid + '"' c_dict['REGRID_TO_GRID'] = regrid c_dict['POINT_STAT_GRID'] = self.config.getstr('config', 'POINT_STAT_GRID') c_dict['POINT_STAT_POLY'] = util.getlist( self.config.getstr('config', 'POINT_STAT_POLY', '')) c_dict['POINT_STAT_STATION_ID'] = util.getlist( self.config.getstr('config', 'POINT_STAT_STATION_ID', '')) c_dict['POINT_STAT_MESSAGE_TYPE'] = util.getlist( self.config.getstr('config', 'POINT_STAT_MESSAGE_TYPE', '')) # handle window variables [FCST/OBS]_[FILE_]_WINDOW_[BEGIN/END] self.handle_window_variables(c_dict, 'point_stat') c_dict['NEIGHBORHOOD_WIDTH'] = self.config.getstr( 'config', 'POINT_STAT_NEIGHBORHOOD_WIDTH', '') c_dict['NEIGHBORHOOD_SHAPE'] = self.config.getstr( 'config', 'POINT_STAT_NEIGHBORHOOD_SHAPE', '') c_dict['VERIFICATION_MASK_TEMPLATE'] = \ self.config.getraw('filename_templates', 'POINT_STAT_VERIFICATION_MASK_TEMPLATE') c_dict['VERIFICATION_MASK'] = '' return c_dict