def get_params(param_a, param_b):
params_a, params_b = get_param_list(param_a, param_b)
logger = logging.getLogger('master_metplus')
# read A confs
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(params_a, None, None, logger)
p = ConfigWrapper(config_launcher.launch(infiles, moreopt), None)
# read B confs
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(params_b, None, None, logger)
p_b = ConfigWrapper(config_launcher.launch(infiles, moreopt), None)
return p, p_b
def metplus_config():
"""! Create a METplus configuration object that can be
manipulated/modified to
reflect different paths, directories, values, etc. for individual
tests.
"""
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='MTDWrapper',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='MTDWrapper')
produtil.log.postmsg('mtd_wrapper is starting')
# Read in the configuration object CONFIG
config = config_metplus.setup()
logger = util.get_logger(config)
config = ConfigWrapper(config, logger)
return config
except Exception as e:
produtil.log.jlogger.critical('mtd_wrapper failed: %s' % (str(e), ),
exc_info=True)
sys.exit(2)
def test_parse_var_list_fcst_and_obs():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'FCST_VAR1_NAME', "FNAME1")
conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12")
conf.set('config', 'FCST_VAR2_NAME', "FNAME2")
conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22")
conf.set('config', 'OBS_VAR1_NAME', "ONAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12")
conf.set('config', 'OBS_VAR2_NAME', "ONAME2")
conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22")
var_list = util.parse_var_list(cu)
assert(var_list[0]['fcst_name'] == "FNAME1" and \
var_list[0]['obs_name'] == "ONAME1" and \
var_list[1]['fcst_name'] == "FNAME1" and \
var_list[1]['obs_name'] == "ONAME1" and \
var_list[2]['fcst_name'] == "FNAME2" and \
var_list[2]['obs_name'] == "ONAME2" and \
var_list[3]['fcst_name'] == "FNAME2" and \
var_list[3]['obs_name'] == "ONAME2" and \
var_list[0]['fcst_level'] == "FLEVELS11" and \
var_list[0]['obs_level'] == "OLEVELS11" and \
var_list[1]['fcst_level'] == "FLEVELS12" and \
var_list[1]['obs_level'] == "OLEVELS12" and \
var_list[2]['fcst_level'] == "FLEVELS21" and \
var_list[2]['obs_level'] == "OLEVELS21" and \
var_list[3]['fcst_level'] == "FLEVELS22" and \
var_list[3]['obs_level'] == "OLEVELS22")
def __init__(self, config, logger=None):
"""!Class for Creating and Running External Programs.
It was intended to handle the MET executables but
can be used by other executables."""
self.logger = logger
self.config = ConfigWrapper(config, self.logger)
self.verbose = self.config.getstr('config', 'LOG_MET_VERBOSITY', '2')
def test_get_lead_sequence_lead_list(key, value):
input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'LEAD_SEQ', key)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = value
assert (test_seq == lead_seq)
def test_get_lead_sequence_lead():
input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'LEAD_SEQ', "3,6,9,12")
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = [3, 6, 9, 12]
assert (test_seq == lead_seq)
def test_preprocess_file_unzipped():
conf = ConfigWrapper(metplus_config(), None)
stage_dir = conf.getdir('STAGING_DIR',
os.path.join(conf.getdir('OUTPUT_BASE'), "stage"))
filepath = conf.getdir(
'METPLUS_BASE') + "/internal_tests/data/zip/testfile4.txt"
outpath = util.preprocess_file(filepath, None, conf)
assert (filepath == outpath and os.path.exists(outpath))
def test_get_lead_sequence_init(key, value):
input_dict = {'valid': datetime.datetime(2019, 02, 01, key)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'INIT_SEQ', "0, 12")
conf.set('config', 'LEAD_SEQ_MAX', 36)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = value
assert (test_seq == lead_seq)
def test_get_lead_sequence_init_min_10():
input_dict = {'valid': datetime.datetime(2019, 2, 1, 12)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'INIT_SEQ', "0, 12")
conf.set('config', 'LEAD_SEQ_MAX', 24)
conf.set('config', 'LEAD_SEQ_MIN', 10)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = [12, 24]
assert (test_seq == lead_seq)
def metplus_config():
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='TCPairsWrapper ',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='TCPairsWrapper ')
produtil.log.postmsg('tc_pairs_wrapper is starting')
# Read in the configuration object CONFIG
config = config_metplus.setup()
logger = util.get_logger(config)
config = ConfigWrapper(config, logger)
return config
except Exception as e:
produtil.log.jlogger.critical('tc_pairs_wrapper failed: %s' %
(str(e), ),
exc_info=True)
sys.exit(2)
def test_parse_var_list_fcst_only():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'FCST_VAR1_NAME', "NAME1")
conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12")
conf.set('config', 'FCST_VAR1_THRESH', ">1, >2")
conf.set('config', 'OBS_VAR1_OPTIONS', "OOPTIONS11")
var_list = util.parse_var_list(cu)
assert(var_list[0]['fcst_name'] == "NAME1" and \
var_list[0]['obs_name'] == "NAME1" and \
var_list[1]['fcst_name'] == "NAME1" and \
var_list[1]['obs_name'] == "NAME1" and \
var_list[0]['fcst_level'] == "LEVELS11" and \
var_list[0]['obs_level'] == "LEVELS11" and \
var_list[1]['fcst_level'] == "LEVELS12" and \
var_list[1]['obs_level'] == "LEVELS12" and \
var_list[0]['fcst_thresh'] == var_list[0]['obs_thresh'] and \
var_list[1]['fcst_thresh'] == var_list[1]['obs_thresh'] and \
var_list[0]['fcst_extra'] == "" and \
var_list[0]['obs_extra'] == "OOPTIONS11" and \
var_list[1]['fcst_extra'] == "" and \
var_list[1]['obs_extra'] == "OOPTIONS11"
)
def test_parse_var_list_fcst_and_obs_and_both():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'OBS_VAR1_NAME', "ONAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12")
conf.set('config', 'FCST_VAR2_NAME', "FNAME2")
conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22")
conf.set('config', 'FCST_VAR3_NAME', "FNAME3")
conf.set('config', 'FCST_VAR3_LEVELS', "FLEVELS31, FLEVELS32")
conf.set('config', 'OBS_VAR3_NAME', "ONAME3")
conf.set('config', 'OBS_VAR3_LEVELS', "OLEVELS31, OLEVELS32")
var_list = util.parse_var_list(cu)
assert(var_list[0].fcst_name == "ONAME1" and \
var_list[0].obs_name == "ONAME1" and \
var_list[1].fcst_name == "ONAME1" and \
var_list[1].obs_name == "ONAME1" and \
var_list[2].fcst_name == "FNAME2" and \
var_list[2].obs_name == "FNAME2" and \
var_list[3].fcst_name == "FNAME2" and \
var_list[3].obs_name == "FNAME2" and \
var_list[4].fcst_name == "FNAME3" and \
var_list[4].obs_name == "ONAME3" and \
var_list[5].fcst_name == "FNAME3" and \
var_list[5].obs_name == "ONAME3" and \
var_list[0].fcst_level == "OLEVELS11" and \
var_list[0].obs_level == "OLEVELS11" and \
var_list[1].fcst_level == "OLEVELS12" and \
var_list[1].obs_level == "OLEVELS12" and \
var_list[2].fcst_level == "FLEVELS21" and \
var_list[2].obs_level == "FLEVELS21" and \
var_list[3].fcst_level == "FLEVELS22" and \
var_list[3].obs_level == "FLEVELS22" and \
var_list[4].fcst_level == "FLEVELS31" and \
var_list[4].obs_level == "OLEVELS31" and \
var_list[5].fcst_level == "FLEVELS32" and \
var_list[5].obs_level == "OLEVELS32" )
def test_parse_var_list_obs():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'OBS_VAR1_NAME', "NAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12")
conf.set('config', 'OBS_VAR2_NAME', "NAME2")
conf.set('config', 'OBS_VAR2_LEVELS', "LEVELS21, LEVELS22")
var_list = util.parse_var_list(cu)
assert(var_list[0].fcst_name == "NAME1" and \
var_list[0].obs_name == "NAME1" and \
var_list[1].fcst_name == "NAME1" and \
var_list[1].obs_name == "NAME1" and \
var_list[2].fcst_name == "NAME2" and \
var_list[2].obs_name == "NAME2" and \
var_list[3].fcst_name == "NAME2" and \
var_list[3].obs_name == "NAME2" and \
var_list[0].fcst_level == "LEVELS11" and \
var_list[0].obs_level == "LEVELS11" and \
var_list[1].fcst_level == "LEVELS12" and \
var_list[1].obs_level == "LEVELS12" and \
var_list[2].fcst_level == "LEVELS21" and \
var_list[2].obs_level == "LEVELS21" and \
var_list[3].fcst_level == "LEVELS22" and \
var_list[3].obs_level == "LEVELS22")
def test_getseconds(key, value):
conf = ConfigWrapper(metplus_config(), None)
conf.set('config', 'TEST_SECONDS', key)
seconds = conf.getseconds('config', 'TEST_SECONDS')
assert (seconds == value)
def main():
"""!Main program.
Master METplus script that invokes the necessary Python scripts
to perform various activities, such as series analysis."""
# Setup Task logger, Until Conf object is created, Task logger is
# only logging to tty, not a file.
logger = logging.getLogger('master_metplus')
logger.info('Starting METplus v%s', util.get_version_number())
# Parse arguments, options and return a config instance.
conf = config_metplus.setup(filename='master_metplus.py')
# NOW we have a conf object p, we can now get the logger
# and set the handler to write to the LOG_METPLUS
# TODO: Frimel setting up logger file handler.
# Setting up handler i.e util.get_logger should be moved to
# the setup wrapper and encapsulated in the config object.
# than you would get it this way logger=p.log(). The config
# object has-a logger we want.
logger = util.get_logger(conf)
logger.info('Running METplus v%s called with command: %s',
util.get_version_number(), ' '.join(sys.argv))
# check for deprecated config items and warn user to remove/replace them
util.check_for_deprecated_config(conf, logger)
config = ConfigWrapper(conf, logger)
# set staging dir to OUTPUT_BASE/stage if not set
if not config.has_option('dir', 'STAGING_DIR'):
config.set('dir', 'STAGING_DIR',
os.path.join(config.getdir('OUTPUT_BASE'), "stage"))
# handle dir to write temporary files
util.handle_tmp_dir(config)
# This is available in each subprocess from os.system BUT
# we also set it in each process since they may be called stand alone.
os.environ['MET_BASE'] = config.getdir('MET_BASE')
config.env = os.environ.copy()
# Use config object to get the list of processes to call
process_list = util.getlist(config.getstr('config', 'PROCESS_LIST'))
# Keep this comment.
# When running commands in the process_list, reprocess the
# original command line using (item))[sys.argv[1:]].
#
# You could call each task (ie. run_tc_pairs.py) without any args since
# the final METPLUS_CONF file was just created from config_metplus.setup,
# and each task, also calls setup, which use an existing final conf
# file over command line args.
#
# both work ...
# Note: Using (item))sys.argv[1:], is preferable since
# it doesn't depend on the conf file existing.
processes = []
for item in process_list:
try:
logger = config.log(item)
command_builder = \
getattr(sys.modules[__name__],
item + "Wrapper")(config, logger)
# if Usage specified in PROCESS_LIST, print usage and exit
if item == 'Usage':
command_builder.run_all_times()
exit(1)
except AttributeError:
raise NameError("Process %s doesn't exist" % item)
processes.append(command_builder)
loop_order = config.getstr('config', 'LOOP_ORDER', '')
if loop_order == '':
loop_order = config.getstr('config', 'LOOP_METHOD')
if loop_order == "processes":
for process in processes:
# referencing using repr(process.app_name) in
# log since it may be None,
# if not set in the command builder subclass' contsructor,
# and no need to generate an exception because of that.
produtil.log.postmsg('master_metplus Calling run_all_times '
'in: %s wrapper.' % repr(process.app_name))
process.run_all_times()
elif loop_order == "times":
util.loop_over_times_and_call(config, processes)
else:
logger.error("Invalid LOOP_METHOD defined. " + \
"Options are processes, times")
exit()
# scrub staging directory if requested
if config.getbool('config', 'SCRUB_STAGING_DIR', False) and\
os.path.exists(config.getdir('STAGING_DIR')):
staging_dir = config.getdir('STAGING_DIR')
logger.info("Scrubbing staging dir: %s", staging_dir)
shutil.rmtree(staging_dir)
# rewrite final conf so it contains all of the default values used
util.write_final_conf(conf, logger)
logger.info('METplus has successfully finished running.')
exit()
def launch(file_list, moreopt, cycle=None, init_dirs=True, prelaunch=None):
for filename in file_list:
if not isinstance(filename, str):
raise TypeError('First input to metplus.config.for_initial_job '
'must be a list of strings.')
conf = METplusLauncher()
logger = conf.log()
cu = ConfigWrapper(conf, logger)
# set config variable for current time
conf.set('config', 'CLOCK_TIME',
datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
# Read in and parse all the conf files.
for filename in file_list:
conf.read(filename)
logger.info("%s: Parsed this file" % (filename, ))
# Overriding or passing in specific conf items on the command line
# ie. config.NEWVAR="a new var" dir.SOMEDIR=/override/dir/path
# If spaces, seems like you need double quotes on command line.
if moreopt:
for section, options in moreopt.items():
if not conf.has_section(section):
conf.add_section(section)
for option, value in options.items():
logger.info('Override: %s.%s=%s' %
(section, option, repr(value)))
conf.set(section, option, value)
# All conf files and command line options have been parsed.
# So lets set and add specific log variables to the conf object
# based on the conf log template values.
util.set_logvars(conf)
# Determine if final METPLUS_CONF file already exists on disk.
# If it does, use it instead.
confloc = conf.getloc('METPLUS_CONF')
# Received feedback: Users want to overwrite the final conf file if it exists.
# Not overwriting is annoying everyone, since when one makes a conf file edit
# you have to remember to remove the final conf file.
# Originally based on a hwrf workflow. since launcher task only runs once,
# and all following tasks use the generated conf file.
#finalconfexists = util.file_exists(confloc)
# Force setting to False, so conf always gets overwritten
finalconfexists = False
if finalconfexists:
logger.warning(
'IGNORING all parsed conf file(s) AND any command line options or arguments, if given.'
)
logger.warning('INSTEAD, Using Existing final conf: %s' % (confloc))
del logger
del conf
conf = METplusLauncher()
logger = conf.log()
conf.read(confloc)
# Set moreopt to None, in case it is not None, We do not want to process
# more options since using existing final conf file.
moreopt = None
# Place holder for when workflow is developed in METplus.
# rocoto does not initialize the dirs, it returns here.
# if not init_dirs:
# if prelaunch is not None:
# prelaunch(conf,logger,cycle)
# return conf
# Initialize the output directories
produtil.fileop.makedirs(cu.getdir('OUTPUT_BASE', logger), logger=logger)
# A user my set the confloc METPLUS_CONF location in a subdir of OUTPUT_BASE
# or even in another parent directory altogether, so make thedirectory
# so the metplus_final.conf file can be written.
if not os.path.exists(realpath(dirname(confloc))):
produtil.fileop.makedirs(realpath(dirname(confloc)), logger=logger)
# set METPLUS_BASE conf to location of scripts used by METplus
# warn if user has set METPLUS_BASE to something different
# in their conf file
user_metplus_base = cu.getdir('METPLUS_BASE', METPLUS_BASE)
if realpath(user_metplus_base) != realpath(METPLUS_BASE):
logger.warning('METPLUS_BASE from the conf files has no effect.'+\
' Overriding to '+METPLUS_BASE)
conf.set('dir', 'METPLUS_BASE', METPLUS_BASE)
# do the same for PARM_BASE
user_parm_base = cu.getdir('PARM_BASE', PARM_BASE)
if realpath(user_parm_base) != realpath(PARM_BASE):
logger.error('PARM_BASE from the config ({}) '.format(user_parm_base) +\
'differs from METplus parm base ({}). '.format(PARM_BASE))
logger.error('Please remove PARM_BASE from any config file. Set the ' +\
'environment variable METPLUS_PARM_BASE to change where ' +\
'the METplus wrappers look for config files.')
exit(1)
conf.set('dir', 'PARM_BASE', PARM_BASE)
version_number = util.get_version_number()
conf.set('config', 'METPLUS_VERSION', version_number)
# print config items that are set automatically
for var in ('METPLUS_BASE', 'PARM_BASE'):
expand = cu.getdir(var)
logger.info('Setting [dir] %s to %s' % (var, expand))
# Place holder for when workflow is developed in METplus.
# if prelaunch is not None:
# prelaunch(conf,logger,cycle)
# writes the METPLUS_CONF used by all tasks.
if not finalconfexists:
logger.info('METPLUS_CONF: %s written here.' % (confloc, ))
with open(confloc, 'wt') as f:
conf.write(f)
return conf
def test_preprocess_file_none():
conf = ConfigWrapper(metplus_config(), None)
outpath = util.preprocess_file(None, None, conf)
assert (outpath is None)
