def run_test_use_case(param_file, run_a, run_b):
metplus_home = "/d1/mccabe/METplus"
a_conf = metplus_home+"/internal_tests/use_cases/system.a.conf"
b_conf = metplus_home+"/internal_tests/use_cases/system.b.conf"
params = param_file.split(",")
# params_a = [ params, a_conf ]
# params_b = [ params, b_conf ]
params_a = params + [a_conf]
params_b = params + [b_conf]
print(params_a)
all_good = True
# params_a = [ param_file, a_conf ]
# params_b = [ param_file, b_conf ]
logger = logging.getLogger('master_metplus')
# read A confs
(parm, infiles, moreopt) = config_launcher.parse_launch_args(params_a,
None, None,
logger)
p = config_launcher.launch(infiles, moreopt)
# read B confs
(parm, infiles, moreopt) = config_launcher.parse_launch_args(params_b,
None, None,
logger)
p_b = config_launcher.launch(infiles, moreopt)
# run A
if run_a:
cmd = os.path.join(p.getstr('config', "METPLUS_BASE"),"ush","master_metplus.py")
for parm in params_a:
cmd += " -c "+parm
print("CMD A:"+cmd)
process = subprocess.Popen(cmd, shell=True)
process.wait()
# run B
if run_b:
cmd = os.path.join(p_b.getstr('config', "METPLUS_BASE"),"ush","master_metplus.py")
for parm in params_b:
cmd += " -c "+parm
print("CMD B:"+cmd)
process = subprocess.Popen(cmd, shell=True)
process.wait()
if not compare_results(p, p_b):
all_good = False
return all_good
def get_params(param_a, param_b):
params_a, params_b = get_param_list(param_a, param_b)
logger = logging.getLogger('master_metplus')
# read A confs
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(params_a, None, None, logger)
p = ConfigWrapper(config_launcher.launch(infiles, moreopt), None)
# read B confs
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(params_b, None, None, logger)
p_b = ConfigWrapper(config_launcher.launch(infiles, moreopt), None)
return p, p_b
def setup(filename=None, logger=None):
"""!The METplus setup function.
@param filename the filename of the calling module.
@param logger a logging.logger for log messages
The setup function that process command line options
and arguements and returns a configuration object."""
# Used for logging and usage statment
if filename is None:
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
filename = cur_filename
# Setup Task logger, Until a Conf object is created, Task logger is
# only logging to tty, not a file.
if logger is None:
logger = logging.getLogger('metplus')
logger.info('Starting METplus configuration setup.')
# Currently METplus command line options are for additional conf file.
# Note options are NOT arguments. ie. -c opt arg arg arg
# command line options. -c <conf_file_X>.
# We will handle them the way produtil handles conf files
# which are ultimately passed as arguments.
# if option is followed by : or = indicates option requires an argument
short_opts = "c:h"
# Note: r: runtime= option is not being used. remove it ?
long_opts = ["config=", "help"]
# All command line input, get options and arguments
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts)
except getopt.GetoptError as err:
logger.critical('Invalid arguments to %s. Exiting.' % (filename))
print(str(err))
usage(filename, logger)
exit(1)
if opts == []:
usage(filename, logger)
opts_conf_files = list()
opts_conf_file = None
for k, v in opts:
if k in ('-c', '--config'):
opts_conf_files.extend(v.split(","))
elif k in ('-h', '--help'):
if logger:
logger.info('Help, printing Usage statement')
usage(filename=filename)
else:
assert False, "UNHANDLED OPTION"
opts_conf_list = list()
for opts_conf_file in opts_conf_files:
opts_conf_list.append(
config_launcher.set_conf_file_path(opts_conf_file))
# append args to opts conf_list, to maintain the same conf
# file order from command line, when confs may be space seperated.
# ie. -c conf1 -c conf2 conf3 would become
# args=[conf3] and opts=[conf1, conf2]
# opts_conf_list = [conf1, conf2, conf3]
opts_conf_list.extend(args)
args = opts_conf_list
# parm, is path to parm directory
# infiles, list of input conf files to be read and processed
# moreopt, dictionary of conf file settings, passed in from command line.
if not args:
args = None
(parm, infiles, moreopt) = \
config_launcher.parse_launch_args(args, usage, filename, logger)
# Currently metplus is not handling cycle.
# Therefore can not use conf.timestrinterp and
# some conf file settings ie. {[a|f]YMDH} time settings.
cycle = None
conf = config_launcher.launch(infiles, moreopt, cycle=cycle)
#conf.sanity_check()
logger.info('Completed METplus configuration setup.')
return conf
def main():
"""!Main program.
Master MET+ script that invokes the necessary Python scripts
to perform various activities, such as series analysis."""
# Job Logger
produtil.log.jlogger.info('Top of master_metplus')
# Setup Task logger, Until Conf object is created, Task logger is
# only logging to tty, not a file.
logger = logging.getLogger('master_metplus')
logger.info('logger Top of master_metplus.')
# Used for logging and usage statment
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
short_opts = "c:r:h"
long_opts = ["config=", "help", "runtime="]
# All command line input, get options and arguments
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts)
except getopt.GetoptError as err:
print(str(err))
usage('SCRIPT IS EXITING DUE TO UNRECOGNIZED COMMAND LINE OPTION')
for k, v in opts:
if k in ('-c', '--config'):
# adds the conf file to the list of arguments.
print("ADDED CONF FILE: " + v)
args.append(config_launcher.set_conf_file_path(v))
elif k in ('-h', '--help'):
usage()
exit()
elif k in ('-r', '--runtime'):
start_time = v
end_time = v
else:
assert False, "UNHANDLED OPTION"
if not args:
args = None
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(args, usage, None, logger)
p = config_launcher.launch(infiles, moreopt)
# NOW I have a conf object p, I can now setup the handler
# to write to the LOG_FILENAME.
logger = util.get_logger(p)
# This is available in each subprocess from os.system BUT
# we also set it in each process since they may be called stand alone.
os.environ['MET_BASE'] = p.getdir('MET_BASE')
# Use config object to get the list of processes to call
process_list = util.getlist(p.getstr('config', 'PROCESS_LIST'))
# Keep this comment.
# When running commands in the process_list, reprocess the
# original command line using (item))[sys.argv[1:]].
#
# You could call each task (ie. run_tc_pairs.py) without any args since
# the final METPLUS_CONF file was just created from config_metplus.setup,
# and each task, also calls setup, which use an existing final conf
# file over command line args.
#
# both work ...
# Note: Using (item))sys.argv[1:], is preferable since
# it doesn't depend on the conf file existing.
processes = []
for item in process_list:
try:
command_builder = getattr(sys.modules[__name__],
item + "Wrapper")(p, logger)
except AttributeError:
raise NameError("Process %s doesn't exist" % item)
exit()
processes.append(command_builder)
if p.getstr('config', 'LOOP_METHOD') == "processes":
for process in processes:
process.run_all_times()
elif p.getstr('config', 'LOOP_METHOD') == "times":
time_format = p.getstr('config', 'INIT_TIME_FMT')
start_t = p.getstr('config', 'INIT_BEG')
end_t = p.getstr('config', 'INIT_END')
time_interval = p.getint('config', 'INIT_INC')
if time_interval < 60:
print(
"ERROR: time_interval parameter must be greater than 60 seconds"
)
exit(1)
init_time = calendar.timegm(time.strptime(start_t, time_format))
end_time = calendar.timegm(time.strptime(end_t, time_format))
while init_time <= end_time:
run_time = time.strftime("%Y%m%d%H%M", time.gmtime(init_time))
print("")
print("****************************************")
print("* RUNNING MET+")
print("* at init time: " + run_time)
print("****************************************")
logger.info("****************************************")
logger.info("* RUNNING MET+")
logger.info("* at init time: " + run_time)
logger.info("****************************************")
for process in processes:
process.run_at_time(run_time)
process.clear()
init_time += time_interval
else:
print("ERROR: Invalid LOOP_METHOD defined. " + \
"Options are processes, times")
exit()
exit()
for item in process_list:
cmd_shell = cmd.to_shell()
logger.info("INFO | [" + cur_filename + ":" + cur_function + "] | " +
"Running: " + cmd_shell)
ret = run(cmd)
if ret != 0:
logger.error("ERROR | [" + cur_filename + ":" + cur_function +
"] | " + "Problem executing: " + cmd_shell)
exit(0)
def main():
logger = logging.getLogger('run_example')
init_time = 0
start_time = 0
end_time = 0
time_interval = 1
short_opts = "c:r:h"
long_opts = ["config=", "help", "runtime="]
# All command line input, get options and arguments
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts)
except getopt.GetoptError as err:
print(str(err))
usage('SCRIPT IS EXITING DUE TO UNRECOGNIZED COMMAND LINE OPTION')
for k, v in opts:
if k in ('-c', '--config'):
# adds the conf file to the list of arguments.
args.append(config_launcher.set_conf_file_path(v))
elif k in ('-h', '--help'):
usage()
exit()
elif k in ('-r', '--runtime'):
start_time = v
end_time = v
else:
assert False, "UNHANDLED OPTION"
if not args:
args = None
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(args, usage, None, logger)
p = config_launcher.launch(infiles, moreopt)
logger = util.get_logger(p)
logger.setLevel(logging.DEBUG)
if start_time == 0:
start_time = p.getstr('config', 'START_TIME')
end_time = p.getstr('config', 'END_TIME')
time_interval = p.getstr('config', 'TIME_INTERVAL')
# Get the list of processes to call
process_list = util.getlist(p.getstr('config', 'PROCESS_LIST'))
model_type = p.getstr('config', 'MODEL_TYPE')
fcst_vars = util.getlist(p.getstr('config', 'FCST_VARS'))
lead_seq = util.getlistint(p.getstr('config', 'LEAD_SEQ'))
init_time = start_time
while init_time <= end_time:
print("")
print("****************************************")
print("* RUNNING MET+")
print("* EVALUATING " + model_type + " at init time: " + init_time)
print("****************************************")
logger.info("****************************************")
logger.info("* RUNNING MET+")
logger.info("* EVALUATING " + model_type + " at init time: " +
init_time)
logger.info("****************************************")
for lead in lead_seq:
for fcst_var in fcst_vars:
# loop over models to compare
accums = util.getlist(p.getstr('config', fcst_var + "_ACCUM"))
ob_types = util.getlist(
p.getstr('config', fcst_var + "_OBTYPE"))
for accum in accums:
for ob_type in ob_types:
if lead < int(accum):
continue
obs_var = p.getstr('config', ob_type + "_VAR")
logger.info("")
logger.info("")
logger.info("For " + init_time + " F" + str(lead) +
", processing " + model_type + "_" +
fcst_var + "_" + accum + " vs " + ob_type +
" " + obs_var + "_" + accum)
valid_time = util.shift_time(init_time, lead)
data_interval = p.getint('config',
ob_type + '_DATA_INTERVAL')
if int(valid_time[8:10]) % data_interval != 0:
logger.warning("No observation for valid time: " +
valid_time + ". Skipping...")
continue
for process in process_list:
if process == "pcp_combine":
run_pcp = CG_pcp_combine(p, logger)
run_pcp.run_at_time(valid_time, accum, ob_type,
fcst_var)
elif process == "regrid_data_plane":
run_regrid = CG_regrid_data_plane(p, logger)
run_regrid.run_at_time(valid_time, accum,
ob_type)
elif process == "grid_stat":
run_grid_stat = CG_grid_stat(p, logger)
run_grid_stat.run_at_time(
init_time, lead, accum, ob_type, fcst_var)
else:
print("ERROR: Invalid process in process list")
exit(1)
init_time = util.shift_time(init_time, int(time_interval))
(logger).info("END OF EXECUTION")