def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST') if self.p.has_option('config', 'FCST_MIN_FORECAST'): min_forecast = self.p.getint('config', 'FCST_MIN_FORECAST') else: min_forecast = 0 #lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) if init_time == -1: #Create a list of files to loop over gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ')) init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL') valid_hr = int(valid_time[8:10]) #Find lead times lead_seq = [] for gs in gen_seq: if valid_hr >= gs: current_lead = valid_hr - gs elif valid_hr < gs: current_lead = valid_hr + gs while current_lead <= max_forecast: lead_seq.append(current_lead) current_lead = current_lead + 24 for lead in lead_seq: if lead < min_forecast: continue task_info.lead = lead for var_info in var_list: self.run_at_time_once(task_info, var_info)
def test_parse_var_list_fcst_and_obs(): conf = metplus_config() cu = ConfigWrapper(conf, None) conf.set('config', 'FCST_VAR1_NAME', "FNAME1") conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") conf.set('config', 'FCST_VAR2_NAME', "FNAME2") conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22") conf.set('config', 'OBS_VAR1_NAME', "ONAME1") conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") conf.set('config', 'OBS_VAR2_NAME', "ONAME2") conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22") var_list = util.parse_var_list(cu) assert(var_list[0]['fcst_name'] == "FNAME1" and \ var_list[0]['obs_name'] == "ONAME1" and \ var_list[1]['fcst_name'] == "FNAME1" and \ var_list[1]['obs_name'] == "ONAME1" and \ var_list[2]['fcst_name'] == "FNAME2" and \ var_list[2]['obs_name'] == "ONAME2" and \ var_list[3]['fcst_name'] == "FNAME2" and \ var_list[3]['obs_name'] == "ONAME2" and \ var_list[0]['fcst_level'] == "FLEVELS11" and \ var_list[0]['obs_level'] == "OLEVELS11" and \ var_list[1]['fcst_level'] == "FLEVELS12" and \ var_list[1]['obs_level'] == "OLEVELS12" and \ var_list[2]['fcst_level'] == "FLEVELS21" and \ var_list[2]['obs_level'] == "OLEVELS21" and \ var_list[3]['fcst_level'] == "FLEVELS22" and \ var_list[3]['obs_level'] == "OLEVELS22")
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) for lead in lead_seq: task_info.lead = lead for var_info in var_list: self.run_at_time_once(task_info, var_info)
def test_reformat_fields_for_met_conus_sfc(): """! THIS RUNS ONLY FOR CONUS SFC skips if config file is set up for upper air Verify that the fcst_field and obs_field text in the MET config field dictionary are well-formed. Test is based on the point_stat_test_conus_sfc.conf file. """ ps = point_stat_wrapper() # Set up the appropriate input directories fcst_input_dir = '/d1/METplus_Mallory/data/gfs' obs_input_dir = '/d1/minnawin/pb2nc_output/nam/conus_sfc' ps.ps_dict['FCST_INPUT_DIR'] = fcst_input_dir ps.ps_dict['OBS_INPUT_DIR'] = obs_input_dir ps.ps_dict['OBS_INPUT_FILE_REGEX'] = \ '.*prepbufr.nam.(2[0-9]{7}).t([0-9]{2})z.tm([0-9]{2}).nc' ps.ps_dict['FCST_INPUT_FILE_REGEX'] = '.*pgbf([0-9]{1,3}).gfs.(2[0-9]{9})' met_config_file = ps.ps_dict['POINT_STAT_CONFIG_FILE'] match = re.match(r'.*conus.*', met_config_file) if not match: pytest.skip("The current test config file is not set up for conus_sfc") all_vars_list = util.parse_var_list(ps.p) logger = logging.getLogger("temp_log") fields = util.reformat_fields_for_met(all_vars_list, logger) # The following fields were defined in the MET+ config file: # TMP, RH, DPT, UGRD, VGRD, TCDC, PRMSL fcst_str = fields.fcst_field expected_fcst_str = '{ name = "TMP"; level = [ "Z2" ]; }, ' \ '{ name = "RH"; level = [ "Z2" ]; }, ' \ '{ name = "DPT"; level = [ "Z2" ]; }, ' \ '{ name = "UGRD"; level = [ "Z10" ]; }, ' \ '{ name = "VGRD"; level = [ "Z10" ]; }, ' \ '{ name = "TCDC"; level = [ "L0" ]; GRIB_lvl_typ = ' \ '200; }, ' \ '{ name = "PRMSL"; level = [ "Z0" ]; }' print("expected: ", expected_fcst_str) print("fcst str: ", fcst_str) obs_str = fields.obs_field expected_obs_str = '{ name = "TMP"; level = [ "Z2" ]; }, ' \ '{ name = "RH"; level = [ "Z2" ]; }, ' \ '{ name = "DPT"; level = [ "Z2" ]; }, ' \ '{ name = "UGRD"; level = [ "Z10" ]; }, ' \ '{ name = "VGRD"; level = [ "Z10" ]; }, ' \ '{ name = "TCDC"; level = [ "L0" ]; }, ' \ '{ name = "PRMSL"; level = [ "Z0" ]; }' print("expected: ", expected_obs_str) print("obs str: ", obs_str) if fcst_str == expected_fcst_str: assert True is True assert obs_str == expected_obs_str
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) for lead in lead_seq: task_info.lead = lead task_info.valid_time = -1 for var_info in var_list: level = var_info.obs_level if level[0].isalpha(): level = var_info.obs_level[1:] self.run_at_time_once(task_info.getValidTime(), level, var_info.obs_name)
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) # want to combine fcst data files to get total accum matching obs? # obs_level = self.p.getstr('config', 'OBS_LEVEL') fcst_level = self.p.getstr('config', 'FCST_LEVEL') # TODO: should use getpath or something? in_dir = self.p.getdir('FCST_PCP_COMBINE_INPUT_DIR') in_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_INPUT_TEMPLATE') out_dir = self.p.getdir('FCST_PCP_COMBINE_OUTPUT_DIR') out_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_OUTPUT_TEMPLATE') for lead in lead_seq: task_info.lead = lead for var_info in var_list: out_level = var_info.obs_level if out_level[0].isalpha(): out_level = out_level[1:] if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \ self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD": self.run_add_method(task_info.getValidTime(), task_info.getInitTime(), out_level, var_info.obs_name, "FCST", True) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM": self.run_sum_method(task_info.getValidTime(), task_info.getInitTime(), fcst_level, out_level, in_dir, out_dir, out_template) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUBTRACT": self.run_subtract_method(task_info, var_info, int(out_level), in_dir, out_dir, in_template, out_template) else: self.logger.error("Invalid PCP_COMBINE_METHOD specified") exit(1)
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) model_templates = util.getlist( self.p.getraw('filename_templates', 'FCST_GRID_STAT_INPUT_TEMPLATE')) model_types = util.getlist(self.p.getraw('config', 'MODEL_TYPE')) max_forecasts = util.getlist( self.p.getraw('config', 'FCST_MAX_FORECAST')) self.check_model(model_templates, model_types, max_forecasts) lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) for md in range(len(model_templates)): cur_model = [ model_templates[md], model_types[md], max_forecasts[md] ] for lead in lead_seq: task_info.lead = lead for var_info in var_list: self.run_at_time_once(task_info, var_info, cur_model)
def test_parse_var_list_fcst_only(): conf = metplus_config() cu = ConfigWrapper(conf, None) conf.set('config', 'FCST_VAR1_NAME', "NAME1") conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'FCST_VAR1_THRESH', ">1, >2") conf.set('config', 'OBS_VAR1_OPTIONS', "OOPTIONS11") var_list = util.parse_var_list(cu) assert(var_list[0]['fcst_name'] == "NAME1" and \ var_list[0]['obs_name'] == "NAME1" and \ var_list[1]['fcst_name'] == "NAME1" and \ var_list[1]['obs_name'] == "NAME1" and \ var_list[0]['fcst_level'] == "LEVELS11" and \ var_list[0]['obs_level'] == "LEVELS11" and \ var_list[1]['fcst_level'] == "LEVELS12" and \ var_list[1]['obs_level'] == "LEVELS12" and \ var_list[0]['fcst_thresh'] == var_list[0]['obs_thresh'] and \ var_list[1]['fcst_thresh'] == var_list[1]['obs_thresh'] and \ var_list[0]['fcst_extra'] == "" and \ var_list[0]['obs_extra'] == "OOPTIONS11" and \ var_list[1]['fcst_extra'] == "" and \ var_list[1]['obs_extra'] == "OOPTIONS11" )
def test_parse_var_list_fcst_and_obs_and_both(): conf = metplus_config() cu = ConfigWrapper(conf, None) conf.set('config', 'OBS_VAR1_NAME', "ONAME1") conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") conf.set('config', 'FCST_VAR2_NAME', "FNAME2") conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22") conf.set('config', 'FCST_VAR3_NAME', "FNAME3") conf.set('config', 'FCST_VAR3_LEVELS', "FLEVELS31, FLEVELS32") conf.set('config', 'OBS_VAR3_NAME', "ONAME3") conf.set('config', 'OBS_VAR3_LEVELS', "OLEVELS31, OLEVELS32") var_list = util.parse_var_list(cu) assert(var_list[0].fcst_name == "ONAME1" and \ var_list[0].obs_name == "ONAME1" and \ var_list[1].fcst_name == "ONAME1" and \ var_list[1].obs_name == "ONAME1" and \ var_list[2].fcst_name == "FNAME2" and \ var_list[2].obs_name == "FNAME2" and \ var_list[3].fcst_name == "FNAME2" and \ var_list[3].obs_name == "FNAME2" and \ var_list[4].fcst_name == "FNAME3" and \ var_list[4].obs_name == "ONAME3" and \ var_list[5].fcst_name == "FNAME3" and \ var_list[5].obs_name == "ONAME3" and \ var_list[0].fcst_level == "OLEVELS11" and \ var_list[0].obs_level == "OLEVELS11" and \ var_list[1].fcst_level == "OLEVELS12" and \ var_list[1].obs_level == "OLEVELS12" and \ var_list[2].fcst_level == "FLEVELS21" and \ var_list[2].obs_level == "FLEVELS21" and \ var_list[3].fcst_level == "FLEVELS22" and \ var_list[3].obs_level == "FLEVELS22" and \ var_list[4].fcst_level == "FLEVELS31" and \ var_list[4].obs_level == "OLEVELS31" and \ var_list[5].fcst_level == "FLEVELS32" and \ var_list[5].obs_level == "OLEVELS32" )
def test_parse_var_list_obs(): conf = metplus_config() cu = ConfigWrapper(conf, None) conf.set('config', 'OBS_VAR1_NAME', "NAME1") conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'OBS_VAR2_NAME', "NAME2") conf.set('config', 'OBS_VAR2_LEVELS', "LEVELS21, LEVELS22") var_list = util.parse_var_list(cu) assert(var_list[0].fcst_name == "NAME1" and \ var_list[0].obs_name == "NAME1" and \ var_list[1].fcst_name == "NAME1" and \ var_list[1].obs_name == "NAME1" and \ var_list[2].fcst_name == "NAME2" and \ var_list[2].obs_name == "NAME2" and \ var_list[3].fcst_name == "NAME2" and \ var_list[3].obs_name == "NAME2" and \ var_list[0].fcst_level == "LEVELS11" and \ var_list[0].obs_level == "LEVELS11" and \ var_list[1].fcst_level == "LEVELS12" and \ var_list[1].obs_level == "LEVELS12" and \ var_list[2].fcst_level == "LEVELS21" and \ var_list[2].obs_level == "LEVELS21" and \ var_list[3].fcst_level == "LEVELS22" and \ var_list[3].obs_level == "LEVELS22")
def grid2grid_sfc_plot_format(self): self.logger.info("Formatting for plotting for grid2grid-sfc") #read config use_init = self.p.getbool('config', 'LOOP_BY_INIT', True) if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_INIT_BEG", start_t) self.add_env_var("FCST_INIT_END", end_t) else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t) self.add_env_var("FCST_VALID_END", end_t) self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR') stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR') var_list = util.parse_var_list(self.p) region_list = util.getlist(self.p.getstr('config', 'REGION_LIST')) lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST')) model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST')) self.add_env_var('INTERP', 'NEAREST') if use_init: loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR') loop_end_hour = self.p.getint('config', 'INIT_END_HOUR') loop_inc = self.p.getint('config', 'INIT_INC') else: loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR') loop_end_hour = self.p.getint('config', 'VALID_END_HOUR') loop_inc = self.p.getint('config', 'VALID_INC') loop_hour = loop_beg_hour while loop_hour <= loop_end_hour: loop_hour_str = str(loop_hour).zfill(2) #filtering times based on if files made based on init_time or valid_time if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_VALID_HOUR", "") self.add_env_var("FCST_INIT_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"') else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"') self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") self.add_env_var("FCST_INIT_HOUR", "") for model in model_list: self.add_env_var('MODEL', model) #build -lookin directory self.set_lookin_dir( os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z', model)) for var_info in var_list: fcst_var_name = var_info.fcst_name fcst_var_level = var_info.fcst_level #fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";") obs_var_name = var_info.obs_name obs_var_level = var_info.obs_level #obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";") self.add_env_var('FCST_VAR_NAME', fcst_var_name) self.add_env_var('FCST_VAR_LEVEL', fcst_var_level) self.add_env_var('OBS_VAR_NAME', obs_var_name) self.add_env_var('OBS_VAR_LEVEL', obs_var_level) for region in region_list: self.add_env_var('REGION', region) for lead in lead_list: if lead < 10: lead_string = '0' + str(lead) else: lead_string = str(lead) self.add_env_var('LEAD', lead_string) if not os.path.exists( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)): os.makedirs( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)) ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat") dump_row_file = os.path.join( stat_analysis_out_dir, loop_hour_str + "Z", model, region, model + "_f" + lead_string + "_fcst" + fcst_var_name + fcst_var_level + "_obs" + obs_var_name + obs_var_level + ".stat") ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat") job = "-job filter -dump_row " + dump_row_file self.add_env_var("JOB", job) #get stat_analysis config file self.set_param_file( self.p.getstr('config', 'STAT_ANALYSIS_CONFIG')) #environment self.logger.debug("") self.logger.debug("ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") self.logger.debug( "COPYABLE ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") #build command cmd = self.get_command() if cmd is None: print( "ERROR: stat_analysis could not generate command" ) return self.logger.info("") self.build() self.clear() loop_hour += loop_inc
def grid2grid_anom_plot_format(self): self.logger.info("Formatting for plotting for grid2grid-anom") #read config use_init = self.p.getbool('config', 'LOOP_BY_INIT', True) if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_INIT_BEG", start_t) self.add_env_var("FCST_INIT_END", end_t) else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t) self.add_env_var("FCST_VALID_END", end_t) self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR') stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR') var_list = util.parse_var_list(self.p) region_list = util.getlist(self.p.getstr('config', 'REGION_LIST')) lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST')) model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST')) if use_init: loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR') loop_end_hour = self.p.getint('config', 'INIT_END_HOUR') loop_inc = self.p.getint('config', 'INIT_INC') else: loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR') loop_end_hour = self.p.getint('config', 'VALID_END_HOUR') loop_inc = self.p.getint('config', 'VALID_INC') loop_hour = loop_beg_hour while loop_hour <= loop_end_hour: loop_hour_str = str(loop_hour).zfill(2) #filtering times based on if files made based on init_time or valid_time if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') self.add_env_var("FCST_VALID_BEG", "") self.add_env_var("FCST_VALID_END", "") self.add_env_var("FCST_VALID_HOUR", "") self.add_env_var("FCST_INIT_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"') else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') self.add_env_var("FCST_VALID_BEG", start_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_END", end_t + "_" + loop_hour_str + "0000") self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"') self.add_env_var("FCST_INIT_BEG", "") self.add_env_var("FCST_INIT_END", "") self.add_env_var("FCST_INIT_HOUR", "") for model in model_list: self.add_env_var('MODEL', model) #build -lookin directory self.set_lookin_dir( os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z', model)) for var_info in var_list: fcst_var_name = var_info.fcst_name fcst_var_level = var_info.fcst_level #fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";") obs_var_name = var_info.obs_name obs_var_level = var_info.obs_level #obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";") self.add_env_var('FCST_VAR_NAME', fcst_var_name) self.add_env_var('FCST_VAR_LEVEL', fcst_var_level) self.add_env_var('OBS_VAR_NAME', obs_var_name) self.add_env_var('OBS_VAR_LEVEL', obs_var_level) interp_mthd = [] if var_info.fcst_name == 'HGT' or var_info.obs_name == 'HGT': fourier_decomp_height = self.p.getbool( 'config', 'FOURIER_HEIGHT_DECOMP') if fourier_decomp_height: wave_num_beg_list = util.getlist( self.p.getstr('config', 'WAVE_NUM_BEG_LIST')) wave_num_end_list = util.getlist( self.p.getstr('config', 'WAVE_NUM_END_LIST')) if len(wave_num_beg_list) != len( wave_num_end_list): print( "ERROR: WAVE_NUM_BEG_LIST and WAVE_NUM_END_LIST do not have the same number of elements" ) exit(1) else: interp_mthd.append("NEAREST") for wn in range(len(wave_num_beg_list)): wb = wave_num_beg_list[wn] we = wave_num_end_list[wn] wave_num_pairing = "WV1_" + wb + "-" + we interp_mthd.append(wave_num_pairing) else: interp_mthd.append("NEAREST") else: interp_mthd.append("NEAREST") for region in region_list: self.add_env_var('REGION', region) for lead in lead_list: if lead < 10: lead_string = '0' + str(lead) else: lead_string = str(lead) self.add_env_var('LEAD', lead_string) if not os.path.exists( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)): os.makedirs( os.path.join(stat_analysis_out_dir, loop_hour_str + "Z", model, region)) for im in interp_mthd: self.add_env_var('INTERP', im) if im == "NEAREST": ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat") dump_row_file = os.path.join( stat_analysis_out_dir, loop_hour_str + "Z", model, region, model + "_f" + lead_string + "_fcst" + fcst_var_name + fcst_var_level + "_obs" + obs_var_name + obs_var_level + ".stat") ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat") else: ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat") dump_row_file = os.path.join( stat_analysis_out_dir, loop_hour_str + "Z", model, region, model + "_f" + lead_string + "_fcst" + fcst_var_name + fcst_var_level + "_obs" + obs_var_name + obs_var_level + "_" + im + ".stat") ##dump_row_file = os.path.join(stat_analysis_out_dir, ## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+"_"+im+".stat") job = "-job filter -dump_row " + dump_row_file self.add_env_var("JOB", job) #get stat_analysis config file self.set_param_file( self.p.getstr('config', 'STAT_ANALYSIS_CONFIG')) #environment self.logger.debug("") self.logger.debug( "ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") self.logger.debug( "COPYABLE ENVIRONMENT FOR NEXT COMMAND: ") self.logger.debug("") #build command cmd = self.get_command() if cmd is None: print( "ERROR: stat_analysis could not generate command" ) return self.logger.info("") self.build() self.clear() loop_hour += loop_inc
def grid2grid_sfc_plots(self): logging_filename = self.logger.handlers[0].baseFilename self.add_env_var("LOGGING_FILENAME", logging_filename) plotting_scripts_dir = self.p.getdir('PLOTTING_SCRIPTS_DIR') #read config use_init = self.p.getbool('config', 'LOOP_BY_INIT', True) if use_init: start_t = self.p.getstr('config', 'INIT_BEG') end_t = self.p.getstr('config', 'INIT_END') loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR') loop_end_hour = self.p.getint('config', 'INIT_END_HOUR') loop_inc = self.p.getint('config', 'INIT_INC') date_filter_method = "Initialization" self.add_env_var("START_T", start_t) self.add_env_var("END_T", end_t) self.add_env_var("DATE_FILTER_METHOD", date_filter_method) else: start_t = self.p.getstr('config', 'VALID_BEG') end_t = self.p.getstr('config', 'VALID_END') loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR') loop_end_hour = self.p.getint('config', 'VALID_END_HOUR') loop_inc = self.p.getint('config', 'VALID_INC') date_filter_method = "Valid" self.add_env_var("START_T", start_t) self.add_env_var("END_T", end_t) self.add_env_var("DATE_FILTER_METHOD", date_filter_method) stat_files_input_dir = self.p.getdir('STAT_FILES_INPUT_DIR') plotting_out_dir = self.p.getdir('PLOTTING_OUT_DIR') if os.path.exists(plotting_out_dir): self.logger.info(plotting_out_dir + " exist, removing") util.rmtree(plotting_out_dir) region_list = util.getlist(self.p.getstr('config', 'REGION_LIST')) lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST')) model_list = self.p.getstr('config', 'MODEL_LIST') plot_stats_list = self.p.getstr('config', 'PLOT_STATS_LIST') self.add_env_var("STAT_FILES_INPUT_DIR", stat_files_input_dir) self.add_env_var("PLOTTING_OUT_DIR", plotting_out_dir) self.add_env_var("MODEL_LIST", model_list) self.add_env_var("PLOT_STATS_LIST", plot_stats_list) var_list = util.parse_var_list(self.p) loop_hour = loop_beg_hour while loop_hour <= loop_end_hour: loop_hour_str = str(loop_hour).zfill(2) self.add_env_var('CYCLE', loop_hour_str) for var_info in var_list: fcst_var_name = var_info.fcst_name fcst_var_level = var_info.fcst_level #fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";") obs_var_name = var_info.obs_name obs_var_level = var_info.obs_level #obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";") self.add_env_var('FCST_VAR_NAME', fcst_var_name) self.add_env_var('FCST_VAR_LEVEL', fcst_var_level) self.add_env_var('OBS_VAR_NAME', obs_var_name) self.add_env_var('OBS_VAR_LEVEL', obs_var_level) for region in region_list: self.add_env_var('REGION', region) for lead in lead_list: if lead < 10: lead_string = '0' + str(lead) else: lead_string = str(lead) self.add_env_var('LEAD', lead_string) py_cmd = os.path.join("python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_sfc_ts.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") self.add_env_var("LEAD_LIST", self.p.getstr('config', 'LEAD_LIST')) py_cmd = os.path.join("python") + " " + os.path.join( plotting_scripts_dir, "plot_grid2grid_sfc_tsmean.py") process = subprocess.Popen(py_cmd, env=self.env, shell=True) process.wait() print("") loop_hour += loop_inc
def create_c_dict(self): c_dict = dict() c_dict['VAR_LIST'] = util.parse_var_list(self.config) return c_dict
def set_environment_variables(self): """! Set all the environment variables in the MET config file to the corresponding values in the MET+ config file. Args: Returns: None - invokes parent class, CommandBuilder add_env_var to add each environment variable to run the """ # pylint:disable=protected-access # Need to call sys.__getframe() to get the filename and method/func # for logging information. # Used for logging. cur_filename = sys._getframe().f_code.co_filename cur_function = sys._getframe().f_code.co_name self.logger.info("INFO|:" + cur_function + '|' + cur_filename + '| ' + "Setting all environment variables specified in " "the MET config file...") # Set the environment variables self.add_env_var(b'MODEL_NAME', str(self.ps_dict['MODEL_NAME'])) regrid_to_grid = str(self.ps_dict['REGRID_TO_GRID']) self.add_env_var(b'REGRID_TO_GRID', regrid_to_grid) os.environ['REGRID_TO_GRID'] = regrid_to_grid # MET accepts a list of values for POINT_STAT_POLY, POINT_STAT_GRID, POINT_STAT_STATION_ID, # and POINT_STAT_MESSAGE_TYPE. If these values are not set in the MET+ config file, assign them to "[]" so # MET recognizes that these are empty lists, resulting in the expected behavior. poly_str = str(self.ps_dict['POINT_STAT_POLY']) if not poly_str: self.add_env_var(b'POINT_STAT_POLY', "[]") else: poly = poly_str.replace("\'", "\"") self.add_env_var(b'POINT_STAT_POLY', poly) grid_str = str(self.ps_dict['POINT_STAT_GRID']) if not grid_str: self.add_env_var(b'POINT_STAT_GRID', "[]") else: # grid = grid_str.replace("\'", "\"") grid = '"' + grid_str + '"' self.add_env_var(b'POINT_STAT_GRID', grid) sid_str = str(self.ps_dict['POINT_STAT_STATION_ID']) if not sid_str: self.add_env_var(b'POINT_STAT_STATION_ID', "[]") else: sid = sid_str.replace("\'", "\"") self.add_env_var(b'POINT_STAT_STATION_ID', sid) tmp_message_type = str(self.ps_dict['POINT_STAT_MESSAGE_TYPE']) # Check for "empty" POINT_STAT_MESSAGE_TYPE in MET+ config file and # set the POINT_STAT_MESSAGE_TYPE environment variable appropriately. if not tmp_message_type: self.add_env_var('POINT_STAT_MESSAGE_TYPE', "[]") else: # Not empty, set the POINT_STAT_MESSAGE_TYPE environment variable to the # message types specified in the MET+ config file. tmp_message_type = str(tmp_message_type).replace("\'", "\"") # Remove all whitespace tmp_message_type = ''.join(tmp_message_type.split()) self.add_env_var(b'POINT_STAT_MESSAGE_TYPE', tmp_message_type) # Retrieve all the fcst and obs field values (name, level, options) # from the MET+ config file, passed into the MET config file via # the FCST_FIELD and OBS_FIELD environment variables. all_vars_list = util.parse_var_list(self.p) met_fields = util.reformat_fields_for_met(all_vars_list, self.logger) self.add_env_var(b'FCST_FIELD', met_fields.fcst_field) self.add_env_var(b'OBS_FIELD', met_fields.obs_field) # Set the environment variables corresponding to the obs_window dictionary. self.add_env_var(b'OBS_WINDOW_BEGIN', str(self.ps_dict['OBS_WINDOW_BEGIN'])) self.add_env_var(b'OBS_WINDOW_END', str(self.ps_dict['OBS_WINDOW_END']))
def run_at_time(self, init_time, valid_time): task_info = TaskInfo() task_info.init_time = init_time task_info.valid_time = valid_time var_list = util.parse_var_list(self.p) max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST') if init_time == -1: #Create a list of files to loop over gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ')) init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL') valid_hr = int(valid_time[8:10]) #Find lead times lead_seq = [] for gs in gen_seq: if valid_hr >= gs: current_lead = valid_hr - gs elif valid_hr < gs: current_lead = valid_hr + gs while current_lead <= max_forecast: lead_seq.append(current_lead) current_lead = current_lead + 24 lead_seq = sorted(lead_seq) if valid_time == -1: lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) # want to combine fcst data files to get total accum matching obs? # obs_level = self.p.getstr('config', 'OBS_LEVEL') fcst_level = self.p.getstr('config', 'FCST_LEVEL') # TODO: should use getpath or something? in_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_INPUT_DIR') in_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_INPUT_TEMPLATE') out_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_OUTPUT_DIR') out_template = self.p.getraw('filename_templates', 'FCST_PCP_COMBINE_OUTPUT_TEMPLATE') for lead in lead_seq: task_info.lead = lead for var_info in var_list: out_level = var_info.obs_level if out_level[0].isalpha(): out_level = out_level[1:] if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \ self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD": self.run_add_method(task_info.getValidTime(), task_info.getInitTime(), out_level, var_info.obs_name, "FCST", True) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM": self.run_sum_method(task_info.getValidTime(), task_info.getInitTime(), fcst_level, out_level, in_dir, out_dir, out_template) elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUBTRACT": self.run_subtract_method(task_info, var_info, int(out_level), in_dir, out_dir, in_template, out_template) else: self.logger.error("Invalid PCP_COMBINE_METHOD specified") exit(1)