def run_at_time(self, input_dict):
"""! Runs the MET application for a given run time. This function loops
over the list of forecast leads and runs the application for each.
Args:
@param input_dict dictionary containing time information
"""
# loop of forecast leads and process each
lead_seq = util.get_lead_sequence(self.config, input_dict)
for lead in lead_seq:
input_dict['lead_hours'] = lead
self.logger.info("Processing forecast lead {}".format(lead))
# set current lead time config and environment variables
self.config.set('config', 'CURRENT_LEAD_TIME', lead)
os.environ['METPLUS_CURRENT_LEAD_TIME'] = str(lead)
time_info = time_util.ti_calculate(input_dict)
if util.skip_time(time_info, self.config):
self.logger.debug('Skipping run time')
continue
# Run for given init/valid time and forecast lead combination
self.run_at_time_once(time_info)
def run_at_time(self, input_dict):
"""! Runs the MET application for a given run time. This function loops
over the list of forecast leads and runs the application for each.
Overrides run_at_time in compare_gridded_wrapper.py
Args:
@param input_dict dictionary containing timing information
"""
# max_lookback = self.c_dict['MAX_LOOKBACK']
# file_interval = self.c_dict['FILE_INTERVAL']
lead_seq = util.get_lead_sequence(self.config, input_dict)
for var_info in self.c_dict['VAR_LIST']:
if self.c_dict['SINGLE_RUN']:
self.run_single_mode(input_dict, var_info)
continue
model_list = []
obs_list = []
# find files for each forecast lead time
tasks = []
for lead in lead_seq:
input_dict['lead_hours'] = lead
time_info = time_util.ti_calculate(input_dict)
tasks.append(time_info)
for current_task in tasks:
# call find_model/obs as needed
model_file = self.find_model(current_task, var_info)
obs_file = self.find_obs(current_task, var_info)
if model_file is None and obs_file is None:
self.logger.warning('Obs and fcst files were not found for init {} and lead {}'.
format(current_task['init_fmt'], current_task['lead_hours']))
continue
if model_file is None:
self.logger.warning('Forecast file was not found for init {} and lead {}'.
format(current_task['init_fmt'], current_task['lead_hours']))
continue
if obs_file is None:
self.logger.warning('Observation file was not found for init {} and lead {}'.
format(current_task['init_fmt'], current_task['lead_hours']))
continue
model_list.append(model_file)
obs_list.append(obs_file)
if len(model_list) == 0:
return
# write ascii file with list of files to process
input_dict['lead_hours'] = 0
time_info = time_util.ti_calculate(input_dict)
model_outfile = time_info['valid_fmt'] + '_mtd_fcst_' + var_info['fcst_name'] + '.txt'
obs_outfile = time_info['valid_fmt'] + '_mtd_obs_' + var_info['obs_name'] + '.txt'
model_list_path = self.write_list_file(model_outfile, model_list)
obs_list_path = self.write_list_file(obs_outfile, obs_list)
arg_dict = {'obs_path' : obs_list_path,
'model_path' : model_list_path }
self.process_fields_one_thresh(current_task, var_info, **arg_dict)
def test_get_lead_sequence_lead_list(key, value):
input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'LEAD_SEQ', key)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = value
assert (test_seq == lead_seq)
def test_get_lead_sequence_lead():
input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'LEAD_SEQ', "3,6,9,12")
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = [3, 6, 9, 12]
assert (test_seq == lead_seq)
def test_get_lead_sequence_init(key, value):
input_dict = {'valid': datetime.datetime(2019, 02, 01, key)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'INIT_SEQ', "0, 12")
conf.set('config', 'LEAD_SEQ_MAX', 36)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = value
assert (test_seq == lead_seq)
def test_get_lead_sequence_init_min_10():
input_dict = {'valid': datetime.datetime(2019, 2, 1, 12)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'INIT_SEQ', "0, 12")
conf.set('config', 'LEAD_SEQ_MAX', 24)
conf.set('config', 'LEAD_SEQ_MIN', 10)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = [12, 24]
assert (test_seq == lead_seq)
def run_at_time(self, input_dict):
"""! Stub, not yet implemented """
# loop of forecast leads and process each
lead_seq = util.get_lead_sequence(self.config, input_dict)
for lead in lead_seq:
input_dict['lead_hours'] = lead
self.logger.info("Processing forecast lead {}".format(lead))
# set current lead time config and environment variables
self.config.set('config', 'CURRENT_LEAD_TIME', lead)
os.environ['METPLUS_CURRENT_LEAD_TIME'] = str(lead)
# Run for given init/valid time and forecast lead combination
self.run_at_time_once(input_dict)
def run_at_time(self, input_dict):
"""! Runs the MET application for a given run time. Processing forecast
or observation data is determined by conf variables. This function
loops over the list of forecast leads and runs the application for
each.
Args:
@param input_dict dictionary containing timing information
"""
lead_seq = util.get_lead_sequence(self.config, input_dict)
for lead in lead_seq:
self.clear()
input_dict['lead_hours'] = lead
self.config.set('config', 'CURRENT_LEAD_TIME', lead)
os.environ['METPLUS_CURRENT_LEAD_TIME'] = str(lead)
time_info = time_util.ti_calculate(input_dict)
self.run_at_time_once(time_info)
def run_at_time(self, input_dict):
"""! Do some processing for the current run time (init or valid)
Args:
@param input_dict dictionary containing time information of current run
generally contains 'now' (current) time and 'init' or 'valid' time
"""
# fill in time info dictionary
time_info = time_util.ti_calculate(input_dict)
# check if looping by valid or init and log time for run
loop_by = time_info['loop_by']
self.logger.info('Running ExampleWrapper at {} time {}'.format(loop_by,
time_info[loop_by+'_fmt']))
# read input directory and template from config dictionary
input_dir = self.c_dict['INPUT_DIR']
input_template = self.c_dict['INPUT_TEMPLATE']
self.logger.info('Input directory is {}'.format(input_dir))
self.logger.info('Input template is {}'.format(input_template))
# get forecast leads to loop over
lead_seq = util.get_lead_sequence(self.config, input_dict)
for lead in lead_seq:
# set forecast lead time in hours
time_info['lead_hours'] = lead
# recalculate time info items
time_info = time_util.ti_calculate(time_info)
# log init, valid, and forecast lead times for current loop iteration
self.logger.info('Processing forecast lead {} initialized at {} and valid at {}'
.format(lead, time_info['init'].strftime('%Y-%m-%d %HZ'),
time_info['valid'].strftime('%Y-%m-%d %HZ')))
# perform string substitution to find filename based on template and current run time
# pass in logger, then template, then any items to use to fill in template
# pass time info with ** in front to expand each dictionary item to a variable
# i.e. time_info['init'] becomes init=init_value
filename = StringSub(self.logger,
input_template,
**time_info).do_string_sub()
self.logger.info('Looking in input directory for file: {}'.format(filename))
return True
def run_single_mode(self, input_dict, var_info):
single_list = []
if self.c_dict['SINGLE_DATA_SRC'] == 'OBS':
find_method = self.find_obs
s_name = var_info['obs_name']
s_level = var_info['obs_level']
else:
find_method = self.find_model
s_name = var_info['fcst_name']
s_level = var_info['fcst_level']
lead_seq = util.get_lead_sequence(self.config, input_dict)
for lead in lead_seq:
input_dict['lead_hours'] = lead
current_task = time_util.ti_calculate(input_dict)
single_file = find_method(current_task, var_info)
if single_file is None:
self.logger.warning('Single file was not found for init {} and lead {}'.
format(current_task['init_fmt'], current_task['lead_hours']))
continue
single_list.append(single_file)
if len(single_list) == 0:
return
# write ascii file with list of files to process
input_dict['lead_hours'] = 0
time_info = time_util.ti_calculate(input_dict)
single_outfile = time_info['valid_fmt'] + '_mtd_single_' + s_name + '.txt'
single_list_path = self.write_list_file(single_outfile, single_list)
arg_dict = {}
if self.c_dict['SINGLE_DATA_SRC'] == 'OBS':
arg_dict['obs_path'] = single_list_path
arg_dict['model_path'] = None
else:
arg_dict['model_path'] = single_list_path
arg_dict['obs_path'] = None
self.process_fields_one_thresh(current_task, var_info, **arg_dict)
def run_at_time(self, input_dict):
"""! Runs the MET application for a given run time. Processing forecast
or observation data is determined by conf variables. This function
loops over the list of forecast leads and runs the application for
each.
Args:
@param init_time initialization time to run. -1 if not set
@param valid_time valid time to run. -1 if not set
"""
app_name_caps = self.app_name.upper()
class_name = self.__class__.__name__[0:-7]
lead_seq = util.get_lead_sequence(self.config, input_dict)
run_list = []
if self.config.getbool('config', 'FCST_' + app_name_caps + '_RUN',
False):
run_list.append("FCST")
if self.config.getbool('config', 'OBS_' + app_name_caps + '_RUN',
False):
run_list.append("OBS")
if len(run_list) == 0:
self.logger.error(class_name+" specified in process_list, but "+\
"FCST_"+app_name_caps+"_RUN and OBS_"+app_name_caps+"_RUN "+\
" are both False. Set one or both to true or "+\
"remove "+class_name+" from the process_list")
exit()
for to_run in run_list:
self.logger.info("Processing {} data".format(to_run))
for lead in lead_seq:
input_dict['lead_hours'] = lead
self.config.set('config', 'CURRENT_LEAD_TIME', lead)
os.environ['METPLUS_CURRENT_LEAD_TIME'] = str(lead)
self.logger.info("Processing forecast lead {}".format(lead))
time_info = time_util.ti_calculate(input_dict)
for var_info in self.c_dict['VAR_LIST']:
self.run_at_time_once(time_info, var_info, to_run)
