def find_model(self, model_type, lead, init_time):
model_dir = self.p.getstr('config', model_type + '_INPUT_DIR')
# max_forecast = self.p.getint('config', model_type+'_MAX_FORECAST')
forecasts = model_type + '_FORECASTS'
max_forecast = util.getlistint(self.p.getstr('config', forecasts))[-1]
init_interval = self.p.getint('config', model_type + '_INIT_INTERVAL')
lead_check = lead
time_check = init_time
time_offset = 0
found = False
while lead_check <= max_forecast:
native_template = self.p.getraw('filename_templates',
model_type + '_NATIVE_TEMPLATE')
model_ss = sts.StringSub(self.logger,
native_template,
init=time_check,
lead=str(lead_check).zfill(2))
model_file = model_ss.doStringSub()
print("model file: " + model_file)
model_path = os.path.join(model_dir, model_file)
if os.path.exists(model_path):
found = True
break
time_check = util.shift_time(time_check, -init_interval)
lead_check = lead_check + init_interval
if found:
return model_path
else:
return ''
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST')
if self.p.has_option('config', 'FCST_MIN_FORECAST'):
min_forecast = self.p.getint('config', 'FCST_MIN_FORECAST')
else:
min_forecast = 0
#lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
if init_time == -1:
#Create a list of files to loop over
gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ'))
init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL')
valid_hr = int(valid_time[8:10])
#Find lead times
lead_seq = []
for gs in gen_seq:
if valid_hr >= gs:
current_lead = valid_hr - gs
elif valid_hr < gs:
current_lead = valid_hr + gs
while current_lead <= max_forecast:
lead_seq.append(current_lead)
current_lead = current_lead + 24
for lead in lead_seq:
if lead < min_forecast:
continue
task_info.lead = lead
for var_info in var_list:
self.run_at_time_once(task_info, var_info)
def run_at_time(self, init_time):
task_info = TaskInfo()
task_info.init_time = init_time
fcst_vars = util.getlist(self.p.getstr('config', 'FCST_VARS'))
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
for lead in lead_seq:
task_info.lead = lead
for fcst_var in fcst_vars:
task_info.fcst_var = fcst_var
# loop over models to compare
accums = util.getlist(
self.p.getstr('config', fcst_var + "_ACCUM"))
ob_types = util.getlist(
self.p.getstr('config', fcst_var + "_OBTYPE"))
for accum in accums:
task_info.level = accum
for ob_type in ob_types:
task_info.ob_type = ob_type
if lead < int(accum):
continue
# self.run_at_time_fcst(task_info)
self.run_at_time_once(task_info.getValidTime(),
task_info.level,
task_info.ob_type,
task_info.fcst_var)
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
self.run_at_time_once(task_info, var_info)
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
for lead in lead_seq:
task_info.lead = lead
task_info.valid_time = -1
for var_info in var_list:
level = var_info.obs_level
if level[0].isalpha():
level = var_info.obs_level[1:]
self.run_at_time_once(task_info.getValidTime(), level,
var_info.obs_name)
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
# want to combine fcst data files to get total accum matching obs?
# obs_level = self.p.getstr('config', 'OBS_LEVEL')
fcst_level = self.p.getstr('config', 'FCST_LEVEL')
# TODO: should use getpath or something?
in_dir = self.p.getdir('FCST_PCP_COMBINE_INPUT_DIR')
in_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_INPUT_TEMPLATE')
out_dir = self.p.getdir('FCST_PCP_COMBINE_OUTPUT_DIR')
out_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_OUTPUT_TEMPLATE')
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
out_level = var_info.obs_level
if out_level[0].isalpha():
out_level = out_level[1:]
if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \
self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD":
self.run_add_method(task_info.getValidTime(),
task_info.getInitTime(),
out_level,
var_info.obs_name,
"FCST", True)
elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM":
self.run_sum_method(task_info.getValidTime(),
task_info.getInitTime(),
fcst_level, out_level,
in_dir, out_dir, out_template)
elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUBTRACT":
self.run_subtract_method(task_info, var_info, int(out_level),
in_dir, out_dir, in_template,
out_template)
else:
self.logger.error("Invalid PCP_COMBINE_METHOD specified")
exit(1)
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
model_templates = util.getlist(
self.p.getraw('filename_templates',
'FCST_GRID_STAT_INPUT_TEMPLATE'))
model_types = util.getlist(self.p.getraw('config', 'MODEL_TYPE'))
max_forecasts = util.getlist(
self.p.getraw('config', 'FCST_MAX_FORECAST'))
self.check_model(model_templates, model_types, max_forecasts)
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
for md in range(len(model_templates)):
cur_model = [
model_templates[md], model_types[md], max_forecasts[md]
]
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
self.run_at_time_once(task_info, var_info, cur_model)
def create_c_dict(self):
"""! Create a data structure (dictionary) that contains all the
values set in the configuration files
Args:
Returns:
c_dict - a dictionary containing the settings in the
configuration files (that aren't in the
metplus_data, metplus_system, and metplus_runtime
config files.
"""
c_dict = super(PB2NCWrapper, self).create_c_dict()
c_dict['SKIP_IF_OUTPUT_EXISTS'] = self.config.getbool('config', 'PB2NC_SKIP_IF_OUTPUT_EXISTS', False)
c_dict['OFFSETS'] = util.getlistint(self.config.getstr('config', 'PB2NC_OFFSETS', '0'))
# Directories
c_dict['OBS_INPUT_DIR'] = self.config.getdir('PB2NC_INPUT_DIR')
c_dict['OUTPUT_DIR'] = self.config.getdir('PB2NC_OUTPUT_DIR')
c_dict['OBS_INPUT_TEMPLATE'] = self.config.getraw('filename_templates', 'PB2NC_INPUT_TEMPLATE')
c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('filename_templates', 'PB2NC_OUTPUT_TEMPLATE')
c_dict['OBS_INPUT_DATATYPE'] = self.config.getstr('config', 'PB2NC_INPUT_DATATYPE', '')
# Configuration
c_dict['CONFIG_FILE'] = self.config.getstr('config',
'PB2NC_CONFIG_FILE')
c_dict['MESSAGE_TYPE'] = util.getlist(
self.config.getstr('config', 'PB2NC_MESSAGE_TYPE', '[]'))
tmp_message_type = str(c_dict['MESSAGE_TYPE']).replace("\'", "\"")
c_dict['MESSAGE_TYPE'] = ''.join(tmp_message_type)
c_dict['STATION_ID'] = util.getlist(
self.config.getstr('config', 'PB2NC_STATION_ID', '[]'))
tmp_message_type = str(c_dict['STATION_ID']).replace("\'", "\"")
c_dict['STATION_ID'] = ''.join(tmp_message_type.split())
grid_id = self.config.getstr('config', 'PB2NC_GRID')
if grid_id.startswith('G'):
# Reformat grid ids that begin with 'G' ( G10, G1, etc.) to format
# Gnnn
c_dict['GRID'] = self.reformat_grid_id(grid_id)
else:
c_dict['GRID'] = grid_id
c_dict['POLY'] = self.config.getstr('config', 'PB2NC_POLY')
c_dict['BUFR_VAR_LIST'] = util.getlist(
self.config.getstr('config', 'PB2NC_OBS_BUFR_VAR_LIST', '[]'))
c_dict['TIME_SUMMARY_FLAG'] = self.config.getbool('config',
'PB2NC_TIME_SUMMARY_FLAG')
c_dict['TIME_SUMMARY_BEG'] = self.config.getstr('config',
'PB2NC_TIME_SUMMARY_BEG')
c_dict['TIME_SUMMARY_END'] = self.config.getstr('config',
'PB2NC_TIME_SUMMARY_END')
c_dict['TIME_SUMMARY_VAR_NAMES'] = util.getlist(
self.config.getstr('config', 'PB2NC_TIME_SUMMARY_VAR_NAMES'))
c_dict['TIME_SUMMARY_TYPES'] = util.getlist(
self.config.getstr('config', 'PB2NC_TIME_SUMMARY_TYPES'))
c_dict['OBS_WINDOW_BEGIN'] = \
self.config.getseconds('config', 'PB2NC_WINDOW_BEGIN',
self.config.getseconds('config',
'OBS_WINDOW_BEGIN', 0))
c_dict['OBS_WINDOW_END'] = \
self.config.getseconds('config', 'PB2NC_WINDOW_END',
self.config.getseconds('config',
'OBS_WINDOW_END', 0))
c_dict['OBS_FILE_WINDOW_BEGIN'] = \
self.config.getseconds('config', 'PB2NC_FILE_WINDOW_BEGIN',
self.config.getseconds('config',
'OBS_FILE_WINDOW_BEGIN', 0))
c_dict['OBS_FILE_WINDOW_END'] = \
self.config.getseconds('config', 'PB2NC_FILE_WINDOW_END',
self.config.getseconds('config',
'OBS_FILE_WINDOW_END', 0))
c_dict['ALLOW_MULTIPLE_FILES'] = True
return c_dict
def grid2grid_sfc_plot_format(self):
self.logger.info("Formatting for plotting for grid2grid-sfc")
#read config
use_init = self.p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_INIT_BEG", start_t)
self.add_env_var("FCST_INIT_END", end_t)
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG", start_t)
self.add_env_var("FCST_VALID_END", end_t)
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR')
stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR')
var_list = util.parse_var_list(self.p)
region_list = util.getlist(self.p.getstr('config', 'REGION_LIST'))
lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST'))
model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST'))
self.add_env_var('INTERP', 'NEAREST')
if use_init:
loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'INIT_END_HOUR')
loop_inc = self.p.getint('config', 'INIT_INC')
else:
loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'VALID_END_HOUR')
loop_inc = self.p.getint('config', 'VALID_INC')
loop_hour = loop_beg_hour
while loop_hour <= loop_end_hour:
loop_hour_str = str(loop_hour).zfill(2)
#filtering times based on if files made based on init_time or valid_time
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_VALID_HOUR", "")
self.add_env_var("FCST_INIT_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"')
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"')
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
self.add_env_var("FCST_INIT_HOUR", "")
for model in model_list:
self.add_env_var('MODEL', model)
#build -lookin directory
self.set_lookin_dir(
os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z',
model))
for var_info in var_list:
fcst_var_name = var_info.fcst_name
fcst_var_level = var_info.fcst_level
#fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";")
obs_var_name = var_info.obs_name
obs_var_level = var_info.obs_level
#obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";")
self.add_env_var('FCST_VAR_NAME', fcst_var_name)
self.add_env_var('FCST_VAR_LEVEL', fcst_var_level)
self.add_env_var('OBS_VAR_NAME', obs_var_name)
self.add_env_var('OBS_VAR_LEVEL', obs_var_level)
for region in region_list:
self.add_env_var('REGION', region)
for lead in lead_list:
if lead < 10:
lead_string = '0' + str(lead)
else:
lead_string = str(lead)
self.add_env_var('LEAD', lead_string)
if not os.path.exists(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region)):
os.makedirs(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region))
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat")
dump_row_file = os.path.join(
stat_analysis_out_dir, loop_hour_str + "Z",
model, region,
model + "_f" + lead_string + "_fcst" +
fcst_var_name + fcst_var_level + "_obs" +
obs_var_name + obs_var_level + ".stat")
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat")
job = "-job filter -dump_row " + dump_row_file
self.add_env_var("JOB", job)
#get stat_analysis config file
self.set_param_file(
self.p.getstr('config',
'STAT_ANALYSIS_CONFIG'))
#environment
self.logger.debug("")
self.logger.debug("ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
self.logger.debug(
"COPYABLE ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
#build command
cmd = self.get_command()
if cmd is None:
print(
"ERROR: stat_analysis could not generate command"
)
return
self.logger.info("")
self.build()
self.clear()
loop_hour += loop_inc
def grid2grid_anom_plot_format(self):
self.logger.info("Formatting for plotting for grid2grid-anom")
#read config
use_init = self.p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_INIT_BEG", start_t)
self.add_env_var("FCST_INIT_END", end_t)
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG", start_t)
self.add_env_var("FCST_VALID_END", end_t)
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
stat_analysis_lookin_dir = self.p.getdir('STAT_ANALYSIS_LOOKIN_DIR')
stat_analysis_out_dir = self.p.getdir('STAT_ANALYSIS_OUT_DIR')
var_list = util.parse_var_list(self.p)
region_list = util.getlist(self.p.getstr('config', 'REGION_LIST'))
lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST'))
model_list = util.getlist(self.p.getstr('config', 'MODEL_LIST'))
if use_init:
loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'INIT_END_HOUR')
loop_inc = self.p.getint('config', 'INIT_INC')
else:
loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'VALID_END_HOUR')
loop_inc = self.p.getint('config', 'VALID_INC')
loop_hour = loop_beg_hour
while loop_hour <= loop_end_hour:
loop_hour_str = str(loop_hour).zfill(2)
#filtering times based on if files made based on init_time or valid_time
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
self.add_env_var("FCST_VALID_BEG", "")
self.add_env_var("FCST_VALID_END", "")
self.add_env_var("FCST_VALID_HOUR", "")
self.add_env_var("FCST_INIT_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_INIT_HOUR", '"' + loop_hour_str + '"')
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
self.add_env_var("FCST_VALID_BEG",
start_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_END",
end_t + "_" + loop_hour_str + "0000")
self.add_env_var("FCST_VALID_HOUR", '"' + loop_hour_str + '"')
self.add_env_var("FCST_INIT_BEG", "")
self.add_env_var("FCST_INIT_END", "")
self.add_env_var("FCST_INIT_HOUR", "")
for model in model_list:
self.add_env_var('MODEL', model)
#build -lookin directory
self.set_lookin_dir(
os.path.join(stat_analysis_lookin_dir, loop_hour_str + 'Z',
model))
for var_info in var_list:
fcst_var_name = var_info.fcst_name
fcst_var_level = var_info.fcst_level
#fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";")
obs_var_name = var_info.obs_name
obs_var_level = var_info.obs_level
#obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";")
self.add_env_var('FCST_VAR_NAME', fcst_var_name)
self.add_env_var('FCST_VAR_LEVEL', fcst_var_level)
self.add_env_var('OBS_VAR_NAME', obs_var_name)
self.add_env_var('OBS_VAR_LEVEL', obs_var_level)
interp_mthd = []
if var_info.fcst_name == 'HGT' or var_info.obs_name == 'HGT':
fourier_decomp_height = self.p.getbool(
'config', 'FOURIER_HEIGHT_DECOMP')
if fourier_decomp_height:
wave_num_beg_list = util.getlist(
self.p.getstr('config', 'WAVE_NUM_BEG_LIST'))
wave_num_end_list = util.getlist(
self.p.getstr('config', 'WAVE_NUM_END_LIST'))
if len(wave_num_beg_list) != len(
wave_num_end_list):
print(
"ERROR: WAVE_NUM_BEG_LIST and WAVE_NUM_END_LIST do not have the same number of elements"
)
exit(1)
else:
interp_mthd.append("NEAREST")
for wn in range(len(wave_num_beg_list)):
wb = wave_num_beg_list[wn]
we = wave_num_end_list[wn]
wave_num_pairing = "WV1_" + wb + "-" + we
interp_mthd.append(wave_num_pairing)
else:
interp_mthd.append("NEAREST")
else:
interp_mthd.append("NEAREST")
for region in region_list:
self.add_env_var('REGION', region)
for lead in lead_list:
if lead < 10:
lead_string = '0' + str(lead)
else:
lead_string = str(lead)
self.add_env_var('LEAD', lead_string)
if not os.path.exists(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region)):
os.makedirs(
os.path.join(stat_analysis_out_dir,
loop_hour_str + "Z", model,
region))
for im in interp_mthd:
self.add_env_var('INTERP', im)
if im == "NEAREST":
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat")
dump_row_file = os.path.join(
stat_analysis_out_dir,
loop_hour_str + "Z", model, region,
model + "_f" + lead_string + "_fcst" +
fcst_var_name + fcst_var_level +
"_obs" + obs_var_name + obs_var_level +
".stat")
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+".stat")
else:
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_"+fcst_var_name+fcst_var_level+".stat")
dump_row_file = os.path.join(
stat_analysis_out_dir,
loop_hour_str + "Z", model, region,
model + "_f" + lead_string + "_fcst" +
fcst_var_name + fcst_var_level +
"_obs" + obs_var_name + obs_var_level +
"_" + im + ".stat")
##dump_row_file = os.path.join(stat_analysis_out_dir,
## loop_hour_str+"Z", model, region, model+"_f"+lead_string+"_fcst"+fcst_var_name+fcst_var_level+fcst_var_extra+"_obs"+obs_var_name+obs_var_level+obs_var_extra+"_"+im+".stat")
job = "-job filter -dump_row " + dump_row_file
self.add_env_var("JOB", job)
#get stat_analysis config file
self.set_param_file(
self.p.getstr('config',
'STAT_ANALYSIS_CONFIG'))
#environment
self.logger.debug("")
self.logger.debug(
"ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
self.logger.debug(
"COPYABLE ENVIRONMENT FOR NEXT COMMAND: ")
self.logger.debug("")
#build command
cmd = self.get_command()
if cmd is None:
print(
"ERROR: stat_analysis could not generate command"
)
return
self.logger.info("")
self.build()
self.clear()
loop_hour += loop_inc
def grid2grid_pres_plots(self):
logging_filename = self.logger.handlers[0].baseFilename
self.add_env_var("LOGGING_FILENAME", logging_filename)
plotting_scripts_dir = self.p.getdir('PLOTTING_SCRIPTS_DIR')
#read config
use_init = self.p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'INIT_END_HOUR')
loop_inc = self.p.getint('config', 'INIT_INC')
date_filter_method = "Initialization"
self.add_env_var("START_T", start_t)
self.add_env_var("END_T", end_t)
self.add_env_var("DATE_FILTER_METHOD", date_filter_method)
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'VALID_END_HOUR')
loop_inc = self.p.getint('config', 'VALID_INC')
date_filter_method = "Valid"
self.add_env_var("START_T", start_t)
self.add_env_var("END_T", end_t)
self.add_env_var("DATE_FILTER_METHOD", date_filter_method)
stat_files_input_dir = self.p.getdir('STAT_FILES_INPUT_DIR')
plotting_out_dir = self.p.getdir('PLOTTING_OUT_DIR')
if os.path.exists(plotting_out_dir):
self.logger.info(plotting_out_dir + " exist, removing")
util.rmtree(plotting_out_dir)
region_list = util.getlist(self.p.getstr('config', 'REGION_LIST'))
lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST'))
model_list = self.p.getstr('config', 'MODEL_LIST')
plot_stats_list = self.p.getstr('config', 'PLOT_STATS_LIST')
self.add_env_var("STAT_FILES_INPUT_DIR", stat_files_input_dir)
self.add_env_var("PLOTTING_OUT_DIR", plotting_out_dir)
self.add_env_var("MODEL_LIST", model_list)
self.add_env_var("PLOT_STATS_LIST", plot_stats_list)
#need to grab var info in special way that differs from util.parse_var_list
#need variables with cooresponding list of levels; logic derived from util.parse_var_list
var_info_list = []
# find all FCST_VARn_NAME keys in the conf files
all_conf = self.p.keys('config')
fcst_indices = []
regex = re.compile("FCST_VAR(\d+)_NAME")
for conf in all_conf:
result = regex.match(conf)
if result is not None:
fcst_indices.append(result.group(1))
# loop over all possible variables and add them to list
for n in fcst_indices:
# get fcst var info if available
if self.p.has_option('config', "FCST_VAR" + n + "_NAME"):
fcst_name = self.p.getstr('config', "FCST_VAR" + n + "_NAME")
fcst_extra = ""
if self.p.has_option('config', "FCST_VAR" + n + "_OPTIONS"):
fcst_extra = self.p.getraw('config',
"FCST_VAR" + n + "_OPTIONS")
fcst_levels = util.getlist(
self.p.getstr('config', "FCST_VAR" + n + "_LEVELS"))
# if OBS_VARn_X does not exist, use FCST_VARn_X
if self.p.has_option('config', "OBS_VAR" + n + "_NAME"):
obs_name = self.p.getstr('config', "OBS_VAR" + n + "_NAME")
else:
obs_name = fcst_name
obs_extra = ""
if self.p.has_option('config', "OBS_VAR" + n + "_OPTIONS"):
obs_extra = self.p.getraw('config', "OBS_VAR" + n + "_OPTIONS")
##else:
## obs_extra = fcst_extra
##fcst_levels = util.getlist(self.p.getstr('config', "FCST_VAR"+n+"_LEVELS"))
if self.p.has_option('config', "OBS_VAR" + n + "_LEVELS"):
obs_levels = util.getlist(
self.p.getstr('config', "FCST_VAR" + n + "_LEVELS"))
else:
obs_levels = fcst_levels
if len(fcst_levels) != len(obs_levels):
print("ERROR: FCST_VAR"+n+"_LEVELS and OBS_VAR"+n+\
"_LEVELS do not have the same number of elements")
exit(1)
fo = util.FieldObj()
fo.fcst_name = fcst_name
fo.obs_name = obs_name
fo.fcst_extra = fcst_extra
fo.obs_extra = obs_extra
fo.fcst_level = fcst_levels
fo.obs_level = obs_levels
var_info_list.append(fo)
loop_hour = loop_beg_hour
while loop_hour <= loop_end_hour:
loop_hour_str = str(loop_hour).zfill(2)
self.add_env_var('CYCLE', loop_hour_str)
for v in var_info_list:
fcst_var_levels_list = v.fcst_level
self.add_env_var('FCST_VAR_NAME', v.fcst_name)
#self.add_env_var('FCST_VAR_EXTRA', v.fcst_extra)
self.add_env_var(
'FCST_VAR_LEVELS_LIST',
''.join(fcst_var_levels_list).replace(
"P", " P").lstrip().replace(" P", ", P"))
obs_var_levels_list = v.obs_level
self.add_env_var('OBS_VAR_NAME', v.obs_name)
#self.add_env_var('OBS_VAR_EXTRA', v.obs_extra)
self.add_env_var(
'OBS_VAR_LEVELS_LIST',
''.join(obs_var_levels_list).replace(
"P", " P").lstrip().replace(" P", ", P"))
for region in region_list:
self.add_env_var('REGION', region)
for lead in lead_list:
if lead < 10:
lead_string = '0' + str(lead)
else:
lead_string = str(lead)
self.add_env_var('LEAD', lead_string)
for vl in range(len(fcst_var_levels_list)):
self.add_env_var('FCST_VAR_LEVEL',
fcst_var_levels_list[vl])
self.add_env_var('OBS_VAR_LEVEL',
obs_var_levels_list[vl])
py_cmd = os.path.join(
"python") + " " + os.path.join(
plotting_scripts_dir,
"plot_grid2grid_pres_ts.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
####py_cmd = os.path.join("python3")+" "+os.path.join(plotting_scripts_dir, "plot_grid2grid_pres_tp.py") #add python3 at top of script
py_cmd = os.path.join("python") + " " + os.path.join(
plotting_scripts_dir, "plot_grid2grid_pres_tp.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
self.add_env_var("LEAD_LIST",
self.p.getstr('config', 'LEAD_LIST'))
py_cmd = os.path.join("python") + " " + os.path.join(
plotting_scripts_dir, "plot_grid2grid_pres_tsmean.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
####py_cmd = os.path.join("python3")+" "+os.path.join(plotting_scripts_dir, "plot_grid2grid_pres_tpmean.py") #add python3 at top of script
py_cmd = os.path.join("python") + " " + os.path.join(
plotting_scripts_dir, "plot_grid2grid_pres_tpmean.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
loop_hour += loop_inc
def grid2grid_sfc_plots(self):
logging_filename = self.logger.handlers[0].baseFilename
self.add_env_var("LOGGING_FILENAME", logging_filename)
plotting_scripts_dir = self.p.getdir('PLOTTING_SCRIPTS_DIR')
#read config
use_init = self.p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
start_t = self.p.getstr('config', 'INIT_BEG')
end_t = self.p.getstr('config', 'INIT_END')
loop_beg_hour = self.p.getint('config', 'INIT_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'INIT_END_HOUR')
loop_inc = self.p.getint('config', 'INIT_INC')
date_filter_method = "Initialization"
self.add_env_var("START_T", start_t)
self.add_env_var("END_T", end_t)
self.add_env_var("DATE_FILTER_METHOD", date_filter_method)
else:
start_t = self.p.getstr('config', 'VALID_BEG')
end_t = self.p.getstr('config', 'VALID_END')
loop_beg_hour = self.p.getint('config', 'VALID_BEG_HOUR')
loop_end_hour = self.p.getint('config', 'VALID_END_HOUR')
loop_inc = self.p.getint('config', 'VALID_INC')
date_filter_method = "Valid"
self.add_env_var("START_T", start_t)
self.add_env_var("END_T", end_t)
self.add_env_var("DATE_FILTER_METHOD", date_filter_method)
stat_files_input_dir = self.p.getdir('STAT_FILES_INPUT_DIR')
plotting_out_dir = self.p.getdir('PLOTTING_OUT_DIR')
if os.path.exists(plotting_out_dir):
self.logger.info(plotting_out_dir + " exist, removing")
util.rmtree(plotting_out_dir)
region_list = util.getlist(self.p.getstr('config', 'REGION_LIST'))
lead_list = util.getlistint(self.p.getstr('config', 'LEAD_LIST'))
model_list = self.p.getstr('config', 'MODEL_LIST')
plot_stats_list = self.p.getstr('config', 'PLOT_STATS_LIST')
self.add_env_var("STAT_FILES_INPUT_DIR", stat_files_input_dir)
self.add_env_var("PLOTTING_OUT_DIR", plotting_out_dir)
self.add_env_var("MODEL_LIST", model_list)
self.add_env_var("PLOT_STATS_LIST", plot_stats_list)
var_list = util.parse_var_list(self.p)
loop_hour = loop_beg_hour
while loop_hour <= loop_end_hour:
loop_hour_str = str(loop_hour).zfill(2)
self.add_env_var('CYCLE', loop_hour_str)
for var_info in var_list:
fcst_var_name = var_info.fcst_name
fcst_var_level = var_info.fcst_level
#fcst_var_extra = var_info.fcst_extra.replace(" = ", "").rstrip(";")
obs_var_name = var_info.obs_name
obs_var_level = var_info.obs_level
#obs_var_extra = var_info.obs_extra.replace(" = ", "").rstrip(";")
self.add_env_var('FCST_VAR_NAME', fcst_var_name)
self.add_env_var('FCST_VAR_LEVEL', fcst_var_level)
self.add_env_var('OBS_VAR_NAME', obs_var_name)
self.add_env_var('OBS_VAR_LEVEL', obs_var_level)
for region in region_list:
self.add_env_var('REGION', region)
for lead in lead_list:
if lead < 10:
lead_string = '0' + str(lead)
else:
lead_string = str(lead)
self.add_env_var('LEAD', lead_string)
py_cmd = os.path.join("python") + " " + os.path.join(
plotting_scripts_dir, "plot_grid2grid_sfc_ts.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
self.add_env_var("LEAD_LIST",
self.p.getstr('config', 'LEAD_LIST'))
py_cmd = os.path.join("python") + " " + os.path.join(
plotting_scripts_dir, "plot_grid2grid_sfc_tsmean.py")
process = subprocess.Popen(py_cmd,
env=self.env,
shell=True)
process.wait()
print("")
loop_hour += loop_inc
def test_getlist_int():
l = '6, 7, 42'
test_list = util.getlistint(l)
assert (test_list == [6, 7, 42])
def main():
logger = logging.getLogger('run_example')
init_time = 0
start_time = 0
end_time = 0
time_interval = 1
short_opts = "c:r:h"
long_opts = ["config=", "help", "runtime="]
# All command line input, get options and arguments
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts)
except getopt.GetoptError as err:
print(str(err))
usage('SCRIPT IS EXITING DUE TO UNRECOGNIZED COMMAND LINE OPTION')
for k, v in opts:
if k in ('-c', '--config'):
# adds the conf file to the list of arguments.
args.append(config_launcher.set_conf_file_path(v))
elif k in ('-h', '--help'):
usage()
exit()
elif k in ('-r', '--runtime'):
start_time = v
end_time = v
else:
assert False, "UNHANDLED OPTION"
if not args:
args = None
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(args, usage, None, logger)
p = config_launcher.launch(infiles, moreopt)
logger = util.get_logger(p)
logger.setLevel(logging.DEBUG)
if start_time == 0:
start_time = p.getstr('config', 'START_TIME')
end_time = p.getstr('config', 'END_TIME')
time_interval = p.getstr('config', 'TIME_INTERVAL')
# Get the list of processes to call
process_list = util.getlist(p.getstr('config', 'PROCESS_LIST'))
model_type = p.getstr('config', 'MODEL_TYPE')
fcst_vars = util.getlist(p.getstr('config', 'FCST_VARS'))
lead_seq = util.getlistint(p.getstr('config', 'LEAD_SEQ'))
init_time = start_time
while init_time <= end_time:
print("")
print("****************************************")
print("* RUNNING MET+")
print("* EVALUATING " + model_type + " at init time: " + init_time)
print("****************************************")
logger.info("****************************************")
logger.info("* RUNNING MET+")
logger.info("* EVALUATING " + model_type + " at init time: " +
init_time)
logger.info("****************************************")
for lead in lead_seq:
for fcst_var in fcst_vars:
# loop over models to compare
accums = util.getlist(p.getstr('config', fcst_var + "_ACCUM"))
ob_types = util.getlist(
p.getstr('config', fcst_var + "_OBTYPE"))
for accum in accums:
for ob_type in ob_types:
if lead < int(accum):
continue
obs_var = p.getstr('config', ob_type + "_VAR")
logger.info("")
logger.info("")
logger.info("For " + init_time + " F" + str(lead) +
", processing " + model_type + "_" +
fcst_var + "_" + accum + " vs " + ob_type +
" " + obs_var + "_" + accum)
valid_time = util.shift_time(init_time, lead)
data_interval = p.getint('config',
ob_type + '_DATA_INTERVAL')
if int(valid_time[8:10]) % data_interval != 0:
logger.warning("No observation for valid time: " +
valid_time + ". Skipping...")
continue
for process in process_list:
if process == "pcp_combine":
run_pcp = CG_pcp_combine(p, logger)
run_pcp.run_at_time(valid_time, accum, ob_type,
fcst_var)
elif process == "regrid_data_plane":
run_regrid = CG_regrid_data_plane(p, logger)
run_regrid.run_at_time(valid_time, accum,
ob_type)
elif process == "grid_stat":
run_grid_stat = CG_grid_stat(p, logger)
run_grid_stat.run_at_time(
init_time, lead, accum, ob_type, fcst_var)
else:
print("ERROR: Invalid process in process list")
exit(1)
init_time = util.shift_time(init_time, int(time_interval))
(logger).info("END OF EXECUTION")
def run_at_time(self, init_time, valid_time):
task_info = TaskInfo()
task_info.init_time = init_time
task_info.valid_time = valid_time
var_list = util.parse_var_list(self.p)
max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST')
if init_time == -1:
#Create a list of files to loop over
gen_seq = util.getlistint(self.p.getstr('config', 'GEN_SEQ'))
init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL')
valid_hr = int(valid_time[8:10])
#Find lead times
lead_seq = []
for gs in gen_seq:
if valid_hr >= gs:
current_lead = valid_hr - gs
elif valid_hr < gs:
current_lead = valid_hr + gs
while current_lead <= max_forecast:
lead_seq.append(current_lead)
current_lead = current_lead + 24
lead_seq = sorted(lead_seq)
if valid_time == -1:
lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ'))
# want to combine fcst data files to get total accum matching obs?
# obs_level = self.p.getstr('config', 'OBS_LEVEL')
fcst_level = self.p.getstr('config', 'FCST_LEVEL')
# TODO: should use getpath or something?
in_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_INPUT_DIR')
in_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_INPUT_TEMPLATE')
out_dir = self.p.getstr('config', 'FCST_PCP_COMBINE_OUTPUT_DIR')
out_template = self.p.getraw('filename_templates',
'FCST_PCP_COMBINE_OUTPUT_TEMPLATE')
for lead in lead_seq:
task_info.lead = lead
for var_info in var_list:
out_level = var_info.obs_level
if out_level[0].isalpha():
out_level = out_level[1:]
if not self.p.has_option('config', 'PCP_COMBINE_METHOD') or \
self.p.getstr('config', 'PCP_COMBINE_METHOD') == "ADD":
self.run_add_method(task_info.getValidTime(),
task_info.getInitTime(), out_level,
var_info.obs_name, "FCST", True)
elif self.p.getstr('config', 'PCP_COMBINE_METHOD') == "SUM":
self.run_sum_method(task_info.getValidTime(),
task_info.getInitTime(), fcst_level,
out_level, in_dir, out_dir,
out_template)
elif self.p.getstr('config',
'PCP_COMBINE_METHOD') == "SUBTRACT":
self.run_subtract_method(task_info, var_info,
int(out_level), in_dir, out_dir,
in_template, out_template)
else:
self.logger.error("Invalid PCP_COMBINE_METHOD specified")
exit(1)
def create_c_dict(self):
"""! Create a dictionary that holds all the values set in the
METplus config file for the point-stat wrapper.
Returns:
c_dict - A dictionary containing the key-value pairs set
in the METplus configuration file.
"""
c_dict = super(PointStatWrapper, self).create_c_dict()
c_dict['ALLOW_MULTIPLE_FILES'] = True
c_dict['OFFSETS'] = util.getlistint(
self.config.getstr('config', 'POINT_STAT_OFFSETS', '0'))
c_dict['FCST_INPUT_TEMPLATE'] = \
self.config.getraw('filename_templates',
'FCST_POINT_STAT_INPUT_TEMPLATE')
c_dict['OBS_INPUT_TEMPLATE'] = \
self.config.getraw('filename_templates',
'OBS_POINT_STAT_INPUT_TEMPLATE')
c_dict['FCST_INPUT_DATATYPE'] = \
self.config.getstr('config', 'FCST_POINT_STAT_INPUT_DATATYPE', '')
c_dict['OBS_INPUT_DATATYPE'] = \
self.config.getstr('config', 'OBS_POINT_STAT_INPUT_DATATYPE', '')
c_dict['FCST_INPUT_DIR'] = self.config.getdir(
'FCST_POINT_STAT_INPUT_DIR')
c_dict['OBS_INPUT_DIR'] = self.config.getdir(
'OBS_POINT_STAT_INPUT_DIR')
c_dict['OUTPUT_DIR'] = \
self.config.getdir('POINT_STAT_OUTPUT_DIR')
# Configuration
c_dict['CONFIG_FILE'] = \
self.config.getstr('config', 'POINT_STAT_CONFIG_FILE')
c_dict['MODEL'] = self.config.getstr('config', 'MODEL')
c_dict['POINT_STAT_CONFIG_FILE'] = \
self.config.getstr('config', 'POINT_STAT_CONFIG_FILE')
regrid = self.config.getstr('config', 'POINT_STAT_REGRID_TO_GRID')
# if not surrounded by quotes and not NONE, add quotes
if regrid[0] != '"' and regrid != 'NONE':
regrid = '"' + regrid + '"'
c_dict['REGRID_TO_GRID'] = regrid
c_dict['POINT_STAT_GRID'] = self.config.getstr('config',
'POINT_STAT_GRID')
c_dict['POINT_STAT_POLY'] = util.getlist(
self.config.getstr('config', 'POINT_STAT_POLY', ''))
c_dict['POINT_STAT_STATION_ID'] = util.getlist(
self.config.getstr('config', 'POINT_STAT_STATION_ID', ''))
c_dict['POINT_STAT_MESSAGE_TYPE'] = util.getlist(
self.config.getstr('config', 'POINT_STAT_MESSAGE_TYPE', ''))
# handle window variables [FCST/OBS]_[FILE_]_WINDOW_[BEGIN/END]
self.handle_window_variables(c_dict, 'point_stat')
c_dict['NEIGHBORHOOD_WIDTH'] = self.config.getstr(
'config', 'POINT_STAT_NEIGHBORHOOD_WIDTH', '')
c_dict['NEIGHBORHOOD_SHAPE'] = self.config.getstr(
'config', 'POINT_STAT_NEIGHBORHOOD_SHAPE', '')
c_dict['VERIFICATION_MASK_TEMPLATE'] = \
self.config.getraw('filename_templates',
'POINT_STAT_VERIFICATION_MASK_TEMPLATE')
c_dict['VERIFICATION_MASK'] = ''
return c_dict