def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST')
if self.p.has_option('config', 'FCST_MIN_FORECAST'):
min_forecast = self.p.getint('config', 'FCST_MIN_FORECAST')
else:
min_forecast = 0
#lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
if init_time == -1:
#Create a list of files to loop over
gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ'))
init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL')
valid_hr = int(valid_time[8:10])
#Find lead times
lead_seq = []
for gs in gen_seq:
if valid_hr >= gs:
current_lead = valid_hr - gs
elif valid_hr < gs:
current_lead = valid_hr + gs
while current_lead <= max_forecast:
lead_seq.append(current_lead)
current_lead = current_lead + 24
for lead in lead_seq:
if lead < min_forecast:
continue
task_info.lead = lead
for var_info in var_list:
self.run_at_time_once(task_info, var_info)
def test_parse_var_list_fcst_and_obs():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'FCST_VAR1_NAME', "FNAME1")
conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12")
conf.set('config', 'FCST_VAR2_NAME', "FNAME2")
conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22")
conf.set('config', 'OBS_VAR1_NAME', "ONAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12")
conf.set('config', 'OBS_VAR2_NAME', "ONAME2")
conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22")
var_list = util.parse_var_list(cu)
assert(var_list[0]['fcst_name'] == "FNAME1" and \
var_list[0]['obs_name'] == "ONAME1" and \
var_list[1]['fcst_name'] == "FNAME1" and \
var_list[1]['obs_name'] == "ONAME1" and \
var_list[2]['fcst_name'] == "FNAME2" and \
var_list[2]['obs_name'] == "ONAME2" and \
var_list[3]['fcst_name'] == "FNAME2" and \
var_list[3]['obs_name'] == "ONAME2" and \
var_list[0]['fcst_level'] == "FLEVELS11" and \
var_list[0]['obs_level'] == "OLEVELS11" and \
var_list[1]['fcst_level'] == "FLEVELS12" and \
var_list[1]['obs_level'] == "OLEVELS12" and \
var_list[2]['fcst_level'] == "FLEVELS21" and \
var_list[2]['obs_level'] == "OLEVELS21" and \
var_list[3]['fcst_level'] == "FLEVELS22" and \
var_list[3]['obs_level'] == "OLEVELS22")
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
self.run_at_time_once(task_info, var_info)
def test_reformat_fields_for_met_conus_sfc():
"""! THIS RUNS ONLY FOR CONUS SFC skips if config file is set up
for upper air
Verify that the fcst_field and obs_field text in the MET config
field dictionary are well-formed. Test is based on the
point_stat_test_conus_sfc.conf file.
"""
ps = point_stat_wrapper()
# Set up the appropriate input directories
fcst_input_dir = '/d1/METplus_Mallory/data/gfs'
obs_input_dir = '/d1/minnawin/pb2nc_output/nam/conus_sfc'
ps.ps_dict['FCST_INPUT_DIR'] = fcst_input_dir
ps.ps_dict['OBS_INPUT_DIR'] = obs_input_dir
ps.ps_dict['OBS_INPUT_FILE_REGEX'] = \
'.*prepbufr.nam.(2[0-9]{7}).t([0-9]{2})z.tm([0-9]{2}).nc'
ps.ps_dict['FCST_INPUT_FILE_REGEX'] = '.*pgbf([0-9]{1,3}).gfs.(2[0-9]{9})'
met_config_file = ps.ps_dict['POINT_STAT_CONFIG_FILE']
match = re.match(r'.*conus.*', met_config_file)
if not match:
pytest.skip("The current test config file is not set up for conus_sfc")
all_vars_list = util.parse_var_list(ps.p)
logger = logging.getLogger("temp_log")
fields = util.reformat_fields_for_met(all_vars_list, logger)
# The following fields were defined in the MET+ config file:
# TMP, RH, DPT, UGRD, VGRD, TCDC, PRMSL
fcst_str = fields.fcst_field
expected_fcst_str = '{ name = "TMP"; level = [ "Z2" ]; }, ' \
'{ name = "RH"; level = [ "Z2" ]; }, ' \
'{ name = "DPT"; level = [ "Z2" ]; }, ' \
'{ name = "UGRD"; level = [ "Z10" ]; }, ' \
'{ name = "VGRD"; level = [ "Z10" ]; }, ' \
'{ name = "TCDC"; level = [ "L0" ]; GRIB_lvl_typ = ' \
'200; }, ' \
'{ name = "PRMSL"; level = [ "Z0" ]; }'
print("expected: ", expected_fcst_str)
print("fcst str: ", fcst_str)
obs_str = fields.obs_field
expected_obs_str = '{ name = "TMP"; level = [ "Z2" ]; }, ' \
'{ name = "RH"; level = [ "Z2" ]; }, ' \
'{ name = "DPT"; level = [ "Z2" ]; }, ' \
'{ name = "UGRD"; level = [ "Z10" ]; }, ' \
'{ name = "VGRD"; level = [ "Z10" ]; }, ' \
'{ name = "TCDC"; level = [ "L0" ]; }, ' \
'{ name = "PRMSL"; level = [ "Z0" ]; }'
print("expected: ", expected_obs_str)
print("obs str: ", obs_str)
if fcst_str == expected_fcst_str:
assert True is True
assert obs_str == expected_obs_str
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
for lead in lead_seq:
task_info.lead = lead
task_info.valid_time = -1
for var_info in var_list:
level = var_info.obs_level
if level[0].isalpha():
level = var_info.obs_level[1:]
self.run_at_time_once(task_info.getValidTime(), level,
var_info.obs_name)
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
# want to combine fcst data files to get total accum matching obs?
# obs_level = self.p.getstr('config', 'OBS_LEVEL')
fcst_level = self.p.getstr('config', 'FCST_LEVEL')
# TODO: should use getpath or something?
in_dir = self.p.getdir('FCST_PCP_COMBINE_INPUT_DIR')
in_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_INPUT_TEMPLATE')
out_dir = self.p.getdir('FCST_PCP_COMBINE_OUTPUT_DIR')
out_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_OUTPUT_TEMPLATE')
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
out_level = var_info.obs_level
if out_level[0].isalpha():
out_level = out_level[1:]
if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \
self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD":
self.run_add_method(task_info.getValidTime(),
task_info.getInitTime(),
out_level,
var_info.obs_name,
"FCST", True)
elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM":
self.run_sum_method(task_info.getValidTime(),
task_info.getInitTime(),
fcst_level, out_level,
in_dir, out_dir, out_template)
elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUBTRACT":
self.run_subtract_method(task_info, var_info, int(out_level),
in_dir, out_dir, in_template,
out_template)
else:
self.logger.error("Invalid PCP_COMBINE_METHOD specified")
exit(1)
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
model_templates = util.getlist(
self.p.getraw('filename_templates',
'FCST_GRID_STAT_INPUT_TEMPLATE'))
model_types = util.getlist(self.p.getraw('config', 'MODEL_TYPE'))
max_forecasts = util.getlist(
self.p.getraw('config', 'FCST_MAX_FORECAST'))
self.check_model(model_templates, model_types, max_forecasts)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
for md in range(len(model_templates)):
cur_model = [
model_templates[md], model_types[md], max_forecasts[md]
]
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
self.run_at_time_once(task_info, var_info, cur_model)
def test_parse_var_list_fcst_only():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'FCST_VAR1_NAME', "NAME1")
conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12")
conf.set('config', 'FCST_VAR1_THRESH', ">1, >2")
conf.set('config', 'OBS_VAR1_OPTIONS', "OOPTIONS11")
var_list = util.parse_var_list(cu)
assert(var_list[0]['fcst_name'] == "NAME1" and \
var_list[0]['obs_name'] == "NAME1" and \
var_list[1]['fcst_name'] == "NAME1" and \
var_list[1]['obs_name'] == "NAME1" and \
var_list[0]['fcst_level'] == "LEVELS11" and \
var_list[0]['obs_level'] == "LEVELS11" and \
var_list[1]['fcst_level'] == "LEVELS12" and \
var_list[1]['obs_level'] == "LEVELS12" and \
var_list[0]['fcst_thresh'] == var_list[0]['obs_thresh'] and \
var_list[1]['fcst_thresh'] == var_list[1]['obs_thresh'] and \
var_list[0]['fcst_extra'] == "" and \
var_list[0]['obs_extra'] == "OOPTIONS11" and \
var_list[1]['fcst_extra'] == "" and \
var_list[1]['obs_extra'] == "OOPTIONS11"
)
def test_parse_var_list_fcst_and_obs_and_both():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'OBS_VAR1_NAME', "ONAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12")
conf.set('config', 'FCST_VAR2_NAME', "FNAME2")
conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22")
conf.set('config', 'FCST_VAR3_NAME', "FNAME3")
conf.set('config', 'FCST_VAR3_LEVELS', "FLEVELS31, FLEVELS32")
conf.set('config', 'OBS_VAR3_NAME', "ONAME3")
conf.set('config', 'OBS_VAR3_LEVELS', "OLEVELS31, OLEVELS32")
var_list = util.parse_var_list(cu)
assert(var_list[0].fcst_name == "ONAME1" and \
var_list[0].obs_name == "ONAME1" and \
var_list[1].fcst_name == "ONAME1" and \
var_list[1].obs_name == "ONAME1" and \
var_list[2].fcst_name == "FNAME2" and \
var_list[2].obs_name == "FNAME2" and \
var_list[3].fcst_name == "FNAME2" and \
var_list[3].obs_name == "FNAME2" and \
var_list[4].fcst_name == "FNAME3" and \
var_list[4].obs_name == "ONAME3" and \
var_list[5].fcst_name == "FNAME3" and \
var_list[5].obs_name == "ONAME3" and \
var_list[0].fcst_level == "OLEVELS11" and \
var_list[0].obs_level == "OLEVELS11" and \
var_list[1].fcst_level == "OLEVELS12" and \
var_list[1].obs_level == "OLEVELS12" and \
var_list[2].fcst_level == "FLEVELS21" and \
var_list[2].obs_level == "FLEVELS21" and \
var_list[3].fcst_level == "FLEVELS22" and \
var_list[3].obs_level == "FLEVELS22" and \
var_list[4].fcst_level == "FLEVELS31" and \
var_list[4].obs_level == "OLEVELS31" and \
var_list[5].fcst_level == "FLEVELS32" and \
var_list[5].obs_level == "OLEVELS32" )
def test_parse_var_list_obs():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'OBS_VAR1_NAME', "NAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12")
conf.set('config', 'OBS_VAR2_NAME', "NAME2")
conf.set('config', 'OBS_VAR2_LEVELS', "LEVELS21, LEVELS22")
var_list = util.parse_var_list(cu)
assert(var_list[0].fcst_name == "NAME1" and \
var_list[0].obs_name == "NAME1" and \
var_list[1].fcst_name == "NAME1" and \
var_list[1].obs_name == "NAME1" and \
var_list[2].fcst_name == "NAME2" and \
var_list[2].obs_name == "NAME2" and \
var_list[3].fcst_name == "NAME2" and \
var_list[3].obs_name == "NAME2" and \
var_list[0].fcst_level == "LEVELS11" and \
var_list[0].obs_level == "LEVELS11" and \
var_list[1].fcst_level == "LEVELS12" and \
var_list[1].obs_level == "LEVELS12" and \
var_list[2].fcst_level == "LEVELS21" and \
var_list[2].obs_level == "LEVELS21" and \
var_list[3].fcst_level == "LEVELS22" and \
var_list[3].obs_level == "LEVELS22")
def grid2grid_sfc_plot_format(self):
self.logger.info("Formatting for plotting for grid2grid-sfc")
#read config
use_init = self.p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_INIT_BEG", start_t)
self.add_env_var("FCST_INIT_END", end_t)
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG", start_t)
self.add_env_var("FCST_VALID_END", end_t)
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR')
stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR')
var_list = util.parse_var_list(self.p)
region_list = util.getlist(self.p.getstr('config', 'REGION_LIST'))
lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST'))
model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST'))
self.add_env_var('INTERP', 'NEAREST')
if use_init:
loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'INIT_END_HOUR')
loop_inc = self.p.getint('config', 'INIT_INC')
else:
loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'VALID_END_HOUR')
loop_inc = self.p.getint('config', 'VALID_INC')
loop_hour = loop_beg_hour
while loop_hour <= loop_end_hour:
loop_hour_str = str(loop_hour).zfill(2)
#filtering times based on if files made based on init_time or valid_time
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_VALID_HOUR", "")
self.add_env_var("FCST_INIT_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"')
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"')
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
self.add_env_var("FCST_INIT_HOUR", "")
for model in model_list:
self.add_env_var('MODEL', model)
#build -lookin directory
self.set_lookin_dir(
os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z',
model))
for var_info in var_list:
fcst_var_name = var_info.fcst_name
fcst_var_level = var_info.fcst_level
#fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";")
obs_var_name = var_info.obs_name
obs_var_level = var_info.obs_level
#obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";")
self.add_env_var('FCST_VAR_NAME', fcst_var_name)
self.add_env_var('FCST_VAR_LEVEL', fcst_var_level)
self.add_env_var('OBS_VAR_NAME', obs_var_name)
self.add_env_var('OBS_VAR_LEVEL', obs_var_level)
for region in region_list:
self.add_env_var('REGION', region)
for lead in lead_list:
if lead < 10:
lead_string = '0' + str(lead)
else:
lead_string = str(lead)
self.add_env_var('LEAD', lead_string)
if not os.path.exists(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region)):
os.makedirs(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region))
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat")
dump_row_file = os.path.join(
stat_analysis_out_dir, loop_hour_str + "Z",
model, region,
model + "_f" + lead_string + "_fcst" +
fcst_var_name + fcst_var_level + "_obs" +
obs_var_name + obs_var_level + ".stat")
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat")
job = "-job filter -dump_row " + dump_row_file
self.add_env_var("JOB", job)
#get stat_analysis config file
self.set_param_file(
self.p.getstr('config',
'STAT_ANALYSIS_CONFIG'))
#environment
self.logger.debug("")
self.logger.debug("ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
self.logger.debug(
"COPYABLE ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
#build command
cmd = self.get_command()
if cmd is None:
print(
"ERROR: stat_analysis could not generate command"
)
return
self.logger.info("")
self.build()
self.clear()
loop_hour += loop_inc
def grid2grid_anom_plot_format(self):
self.logger.info("Formatting for plotting for grid2grid-anom")
#read config
use_init = self.p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_INIT_BEG", start_t)
self.add_env_var("FCST_INIT_END", end_t)
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG", start_t)
self.add_env_var("FCST_VALID_END", end_t)
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR')
stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR')
var_list = util.parse_var_list(self.p)
region_list = util.getlist(self.p.getstr('config', 'REGION_LIST'))
lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST'))
model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST'))
if use_init:
loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'INIT_END_HOUR')
loop_inc = self.p.getint('config', 'INIT_INC')
else:
loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'VALID_END_HOUR')
loop_inc = self.p.getint('config', 'VALID_INC')
loop_hour = loop_beg_hour
while loop_hour <= loop_end_hour:
loop_hour_str = str(loop_hour).zfill(2)
#filtering times based on if files made based on init_time or valid_time
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_VALID_HOUR", "")
self.add_env_var("FCST_INIT_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"')
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"')
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
self.add_env_var("FCST_INIT_HOUR", "")
for model in model_list:
self.add_env_var('MODEL', model)
#build -lookin directory
self.set_lookin_dir(
os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z',
model))
for var_info in var_list:
fcst_var_name = var_info.fcst_name
fcst_var_level = var_info.fcst_level
#fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";")
obs_var_name = var_info.obs_name
obs_var_level = var_info.obs_level
#obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";")
self.add_env_var('FCST_VAR_NAME', fcst_var_name)
self.add_env_var('FCST_VAR_LEVEL', fcst_var_level)
self.add_env_var('OBS_VAR_NAME', obs_var_name)
self.add_env_var('OBS_VAR_LEVEL', obs_var_level)
interp_mthd = []
if var_info.fcst_name == 'HGT' or var_info.obs_name == 'HGT':
fourier_decomp_height = self.p.getbool(
'config', 'FOURIER_HEIGHT_DECOMP')
if fourier_decomp_height:
wave_num_beg_list = util.getlist(
self.p.getstr('config', 'WAVE_NUM_BEG_LIST'))
wave_num_end_list = util.getlist(
self.p.getstr('config', 'WAVE_NUM_END_LIST'))
if len(wave_num_beg_list) != len(
wave_num_end_list):
print(
"ERROR: WAVE_NUM_BEG_LIST and WAVE_NUM_END_LIST do not have the same number of elements"
)
exit(1)
else:
interp_mthd.append("NEAREST")
for wn in range(len(wave_num_beg_list)):
wb = wave_num_beg_list[wn]
we = wave_num_end_list[wn]
wave_num_pairing = "WV1_" + wb + "-" + we
interp_mthd.append(wave_num_pairing)
else:
interp_mthd.append("NEAREST")
else:
interp_mthd.append("NEAREST")
for region in region_list:
self.add_env_var('REGION', region)
for lead in lead_list:
if lead < 10:
lead_string = '0' + str(lead)
else:
lead_string = str(lead)
self.add_env_var('LEAD', lead_string)
if not os.path.exists(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region)):
os.makedirs(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region))
for im in interp_mthd:
self.add_env_var('INTERP', im)
if im == "NEAREST":
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat")
dump_row_file = os.path.join(
stat_analysis_out_dir,
loop_hour_str + "Z", model, region,
model + "_f" + lead_string + "_fcst" +
fcst_var_name + fcst_var_level +
"_obs" + obs_var_name + obs_var_level +
".stat")
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat")
else:
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat")
dump_row_file = os.path.join(
stat_analysis_out_dir,
loop_hour_str + "Z", model, region,
model + "_f" + lead_string + "_fcst" +
fcst_var_name + fcst_var_level +
"_obs" + obs_var_name + obs_var_level +
"_" + im + ".stat")
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+"_"+im+".stat")
job = "-job filter -dump_row " + dump_row_file
self.add_env_var("JOB", job)
#get stat_analysis config file
self.set_param_file(
self.p.getstr('config',
'STAT_ANALYSIS_CONFIG'))
#environment
self.logger.debug("")
self.logger.debug(
"ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
self.logger.debug(
"COPYABLE ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
#build command
cmd = self.get_command()
if cmd is None:
print(
"ERROR: stat_analysis could not generate command"
)
return
self.logger.info("")
self.build()
self.clear()
loop_hour += loop_inc
def grid2grid_sfc_plots(self):
logging_filename = self.logger.handlers[0].baseFilename
self.add_env_var("LOGGING_FILENAME", logging_filename)
plotting_scripts_dir = self.p.getdir('PLOTTING_SCRIPTS_DIR')
#read config
use_init = self.p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'INIT_END_HOUR')
loop_inc = self.p.getint('config', 'INIT_INC')
date_filter_method = "Initialization"
self.add_env_var("START_T", start_t)
self.add_env_var("END_T", end_t)
self.add_env_var("DATE_FILTER_METHOD", date_filter_method)
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'VALID_END_HOUR')
loop_inc = self.p.getint('config', 'VALID_INC')
date_filter_method = "Valid"
self.add_env_var("START_T", start_t)
self.add_env_var("END_T", end_t)
self.add_env_var("DATE_FILTER_METHOD", date_filter_method)
stat_files_input_dir = self.p.getdir('STAT_FILES_INPUT_DIR')
plotting_out_dir = self.p.getdir('PLOTTING_OUT_DIR')
if os.path.exists(plotting_out_dir):
self.logger.info(plotting_out_dir + " exist, removing")
util.rmtree(plotting_out_dir)
region_list = util.getlist(self.p.getstr('config', 'REGION_LIST'))
lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST'))
model_list = self.p.getstr('config', 'MODEL_LIST')
plot_stats_list = self.p.getstr('config', 'PLOT_STATS_LIST')
self.add_env_var("STAT_FILES_INPUT_DIR", stat_files_input_dir)
self.add_env_var("PLOTTING_OUT_DIR", plotting_out_dir)
self.add_env_var("MODEL_LIST", model_list)
self.add_env_var("PLOT_STATS_LIST", plot_stats_list)
var_list = util.parse_var_list(self.p)
loop_hour = loop_beg_hour
while loop_hour <= loop_end_hour:
loop_hour_str = str(loop_hour).zfill(2)
self.add_env_var('CYCLE', loop_hour_str)
for var_info in var_list:
fcst_var_name = var_info.fcst_name
fcst_var_level = var_info.fcst_level
#fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";")
obs_var_name = var_info.obs_name
obs_var_level = var_info.obs_level
#obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";")
self.add_env_var('FCST_VAR_NAME', fcst_var_name)
self.add_env_var('FCST_VAR_LEVEL', fcst_var_level)
self.add_env_var('OBS_VAR_NAME', obs_var_name)
self.add_env_var('OBS_VAR_LEVEL', obs_var_level)
for region in region_list:
self.add_env_var('REGION', region)
for lead in lead_list:
if lead < 10:
lead_string = '0' + str(lead)
else:
lead_string = str(lead)
self.add_env_var('LEAD', lead_string)
py_cmd = os.path.join("python") + " " + os.path.join(
plotting_scripts_dir, "plot_grid2grid_sfc_ts.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
self.add_env_var("LEAD_LIST",
self.p.getstr('config', 'LEAD_LIST'))
py_cmd = os.path.join("python") + " " + os.path.join(
plotting_scripts_dir, "plot_grid2grid_sfc_tsmean.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
loop_hour += loop_inc
def create_c_dict(self):
c_dict = dict()
c_dict['VAR_LIST'] = util.parse_var_list(self.config)
return c_dict
def set_environment_variables(self):
"""! Set all the environment variables in the MET config
file to the corresponding values in the MET+ config file.
Args:
Returns: None - invokes parent class, CommandBuilder add_env_var
to add each environment variable to run the
"""
# pylint:disable=protected-access
# Need to call sys.__getframe() to get the filename and method/func
# for logging information.
# Used for logging.
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
self.logger.info("INFO|:" + cur_function + '|' + cur_filename + '| ' +
"Setting all environment variables specified in "
"the MET config file...")
# Set the environment variables
self.add_env_var(b'MODEL_NAME', str(self.ps_dict['MODEL_NAME']))
regrid_to_grid = str(self.ps_dict['REGRID_TO_GRID'])
self.add_env_var(b'REGRID_TO_GRID', regrid_to_grid)
os.environ['REGRID_TO_GRID'] = regrid_to_grid
# MET accepts a list of values for POINT_STAT_POLY, POINT_STAT_GRID, POINT_STAT_STATION_ID,
# and POINT_STAT_MESSAGE_TYPE. If these values are not set in the MET+ config file, assign them to "[]" so
# MET recognizes that these are empty lists, resulting in the expected behavior.
poly_str = str(self.ps_dict['POINT_STAT_POLY'])
if not poly_str:
self.add_env_var(b'POINT_STAT_POLY', "[]")
else:
poly = poly_str.replace("\'", "\"")
self.add_env_var(b'POINT_STAT_POLY', poly)
grid_str = str(self.ps_dict['POINT_STAT_GRID'])
if not grid_str:
self.add_env_var(b'POINT_STAT_GRID', "[]")
else:
# grid = grid_str.replace("\'", "\"")
grid = '"' + grid_str + '"'
self.add_env_var(b'POINT_STAT_GRID', grid)
sid_str = str(self.ps_dict['POINT_STAT_STATION_ID'])
if not sid_str:
self.add_env_var(b'POINT_STAT_STATION_ID', "[]")
else:
sid = sid_str.replace("\'", "\"")
self.add_env_var(b'POINT_STAT_STATION_ID', sid)
tmp_message_type = str(self.ps_dict['POINT_STAT_MESSAGE_TYPE'])
# Check for "empty" POINT_STAT_MESSAGE_TYPE in MET+ config file and
# set the POINT_STAT_MESSAGE_TYPE environment variable appropriately.
if not tmp_message_type:
self.add_env_var('POINT_STAT_MESSAGE_TYPE', "[]")
else:
# Not empty, set the POINT_STAT_MESSAGE_TYPE environment variable to the
# message types specified in the MET+ config file.
tmp_message_type = str(tmp_message_type).replace("\'", "\"")
# Remove all whitespace
tmp_message_type = ''.join(tmp_message_type.split())
self.add_env_var(b'POINT_STAT_MESSAGE_TYPE', tmp_message_type)
# Retrieve all the fcst and obs field values (name, level, options)
# from the MET+ config file, passed into the MET config file via
# the FCST_FIELD and OBS_FIELD environment variables.
all_vars_list = util.parse_var_list(self.p)
met_fields = util.reformat_fields_for_met(all_vars_list, self.logger)
self.add_env_var(b'FCST_FIELD', met_fields.fcst_field)
self.add_env_var(b'OBS_FIELD', met_fields.obs_field)
# Set the environment variables corresponding to the obs_window dictionary.
self.add_env_var(b'OBS_WINDOW_BEGIN',
str(self.ps_dict['OBS_WINDOW_BEGIN']))
self.add_env_var(b'OBS_WINDOW_END',
str(self.ps_dict['OBS_WINDOW_END']))
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST')
if init_time == -1:
#Create a list of files to loop over
gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ'))
init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL')
valid_hr = int(valid_time[8:10])
#Find lead times
lead_seq = []
for gs in gen_seq:
if valid_hr >= gs:
current_lead = valid_hr - gs
elif valid_hr < gs:
current_lead = valid_hr + gs
while current_lead <= max_forecast:
lead_seq.append(current_lead)
current_lead = current_lead + 24
lead_seq = sorted(lead_seq)
if valid_time == -1:
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
# want to combine fcst data files to get total accum matching obs?
# obs_level = self.p.getstr('config', 'OBS_LEVEL')
fcst_level = self.p.getstr('config', 'FCST_LEVEL')
# TODO: should use getpath or something?
in_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_INPUT_DIR')
in_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_INPUT_TEMPLATE')
out_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_OUTPUT_DIR')
out_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_OUTPUT_TEMPLATE')
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
out_level = var_info.obs_level
if out_level[0].isalpha():
out_level = out_level[1:]
if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \
self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD":
self.run_add_method(task_info.getValidTime(),
task_info.getInitTime(), out_level,
var_info.obs_name, "FCST", True)
elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM":
self.run_sum_method(task_info.getValidTime(),
task_info.getInitTime(), fcst_level,
out_level, in_dir, out_dir,
out_template)
elif self.p.getstr('config',
'PCP_COMBINE_METHOD') == "SUBTRACT":
self.run_subtract_method(task_info, var_info,
int(out_level), in_dir, out_dir,
in_template, out_template)
else:
self.logger.error("Invalid PCP_COMBINE_METHOD specified")
exit(1)
