def run_test_use_case(param_file, run_a, run_b): metplus_home = "/d1/mccabe/METplus" a_conf = metplus_home+"/internal_tests/use_cases/system.a.conf" b_conf = metplus_home+"/internal_tests/use_cases/system.b.conf" params = param_file.split(",") # params_a = [ params, a_conf ] # params_b = [ params, b_conf ] params_a = params + [a_conf] params_b = params + [b_conf] print(params_a) all_good = True # params_a = [ param_file, a_conf ] # params_b = [ param_file, b_conf ] logger = logging.getLogger('master_metplus') # read A confs (parm, infiles, moreopt) = config_launcher.parse_launch_args(params_a, None, None, logger) p = config_launcher.launch(infiles, moreopt) # read B confs (parm, infiles, moreopt) = config_launcher.parse_launch_args(params_b, None, None, logger) p_b = config_launcher.launch(infiles, moreopt) # run A if run_a: cmd = os.path.join(p.getstr('config', "METPLUS_BASE"),"ush","master_metplus.py") for parm in params_a: cmd += " -c "+parm print("CMD A:"+cmd) process = subprocess.Popen(cmd, shell=True) process.wait() # run B if run_b: cmd = os.path.join(p_b.getstr('config', "METPLUS_BASE"),"ush","master_metplus.py") for parm in params_b: cmd += " -c "+parm print("CMD B:"+cmd) process = subprocess.Popen(cmd, shell=True) process.wait() if not compare_results(p, p_b): all_good = False return all_good
def get_params(param_a, param_b): params_a, params_b = get_param_list(param_a, param_b) logger = logging.getLogger('master_metplus') # read A confs (parm, infiles, moreopt) = config_launcher.parse_launch_args(params_a, None, None, logger) p = ConfigWrapper(config_launcher.launch(infiles, moreopt), None) # read B confs (parm, infiles, moreopt) = config_launcher.parse_launch_args(params_b, None, None, logger) p_b = ConfigWrapper(config_launcher.launch(infiles, moreopt), None) return p, p_b
def setup(filename=None, logger=None): """!The METplus setup function. @param filename the filename of the calling module. @param logger a logging.logger for log messages The setup function that process command line options and arguements and returns a configuration object.""" # Used for logging and usage statment if filename is None: cur_filename = sys._getframe().f_code.co_filename cur_function = sys._getframe().f_code.co_name filename = cur_filename # Setup Task logger, Until a Conf object is created, Task logger is # only logging to tty, not a file. if logger is None: logger = logging.getLogger('metplus') logger.info('Starting METplus configuration setup.') # Currently METplus command line options are for additional conf file. # Note options are NOT arguments. ie. -c opt arg arg arg # command line options. -c <conf_file_X>. # We will handle them the way produtil handles conf files # which are ultimately passed as arguments. # if option is followed by : or = indicates option requires an argument short_opts = "c:h" # Note: r: runtime= option is not being used. remove it ? long_opts = ["config=", "help"] # All command line input, get options and arguments try: opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts) except getopt.GetoptError as err: logger.critical('Invalid arguments to %s. Exiting.' % (filename)) print(str(err)) usage(filename, logger) exit(1) if opts == []: usage(filename, logger) opts_conf_files = list() opts_conf_file = None for k, v in opts: if k in ('-c', '--config'): opts_conf_files.extend(v.split(",")) elif k in ('-h', '--help'): if logger: logger.info('Help, printing Usage statement') usage(filename=filename) else: assert False, "UNHANDLED OPTION" opts_conf_list = list() for opts_conf_file in opts_conf_files: opts_conf_list.append( config_launcher.set_conf_file_path(opts_conf_file)) # append args to opts conf_list, to maintain the same conf # file order from command line, when confs may be space seperated. # ie. -c conf1 -c conf2 conf3 would become # args=[conf3] and opts=[conf1, conf2] # opts_conf_list = [conf1, conf2, conf3] opts_conf_list.extend(args) args = opts_conf_list # parm, is path to parm directory # infiles, list of input conf files to be read and processed # moreopt, dictionary of conf file settings, passed in from command line. if not args: args = None (parm, infiles, moreopt) = \ config_launcher.parse_launch_args(args, usage, filename, logger) # Currently metplus is not handling cycle. # Therefore can not use conf.timestrinterp and # some conf file settings ie. {[a|f]YMDH} time settings. cycle = None conf = config_launcher.launch(infiles, moreopt, cycle=cycle) #conf.sanity_check() logger.info('Completed METplus configuration setup.') return conf
def main(): """!Main program. Master MET+ script that invokes the necessary Python scripts to perform various activities, such as series analysis.""" # Job Logger produtil.log.jlogger.info('Top of master_metplus') # Setup Task logger, Until Conf object is created, Task logger is # only logging to tty, not a file. logger = logging.getLogger('master_metplus') logger.info('logger Top of master_metplus.') # Used for logging and usage statment cur_filename = sys._getframe().f_code.co_filename cur_function = sys._getframe().f_code.co_name short_opts = "c:r:h" long_opts = ["config=", "help", "runtime="] # All command line input, get options and arguments try: opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts) except getopt.GetoptError as err: print(str(err)) usage('SCRIPT IS EXITING DUE TO UNRECOGNIZED COMMAND LINE OPTION') for k, v in opts: if k in ('-c', '--config'): # adds the conf file to the list of arguments. print("ADDED CONF FILE: " + v) args.append(config_launcher.set_conf_file_path(v)) elif k in ('-h', '--help'): usage() exit() elif k in ('-r', '--runtime'): start_time = v end_time = v else: assert False, "UNHANDLED OPTION" if not args: args = None (parm, infiles, moreopt) = config_launcher.parse_launch_args(args, usage, None, logger) p = config_launcher.launch(infiles, moreopt) # NOW I have a conf object p, I can now setup the handler # to write to the LOG_FILENAME. logger = util.get_logger(p) # This is available in each subprocess from os.system BUT # we also set it in each process since they may be called stand alone. os.environ['MET_BASE'] = p.getdir('MET_BASE') # Use config object to get the list of processes to call process_list = util.getlist(p.getstr('config', 'PROCESS_LIST')) # Keep this comment. # When running commands in the process_list, reprocess the # original command line using (item))[sys.argv[1:]]. # # You could call each task (ie. run_tc_pairs.py) without any args since # the final METPLUS_CONF file was just created from config_metplus.setup, # and each task, also calls setup, which use an existing final conf # file over command line args. # # both work ... # Note: Using (item))sys.argv[1:], is preferable since # it doesn't depend on the conf file existing. processes = [] for item in process_list: try: command_builder = getattr(sys.modules[__name__], item + "Wrapper")(p, logger) except AttributeError: raise NameError("Process %s doesn't exist" % item) exit() processes.append(command_builder) if p.getstr('config', 'LOOP_METHOD') == "processes": for process in processes: process.run_all_times() elif p.getstr('config', 'LOOP_METHOD') == "times": time_format = p.getstr('config', 'INIT_TIME_FMT') start_t = p.getstr('config', 'INIT_BEG') end_t = p.getstr('config', 'INIT_END') time_interval = p.getint('config', 'INIT_INC') if time_interval < 60: print( "ERROR: time_interval parameter must be greater than 60 seconds" ) exit(1) init_time = calendar.timegm(time.strptime(start_t, time_format)) end_time = calendar.timegm(time.strptime(end_t, time_format)) while init_time <= end_time: run_time = time.strftime("%Y%m%d%H%M", time.gmtime(init_time)) print("") print("****************************************") print("* RUNNING MET+") print("* at init time: " + run_time) print("****************************************") logger.info("****************************************") logger.info("* RUNNING MET+") logger.info("* at init time: " + run_time) logger.info("****************************************") for process in processes: process.run_at_time(run_time) process.clear() init_time += time_interval else: print("ERROR: Invalid LOOP_METHOD defined. " + \ "Options are processes, times") exit() exit() for item in process_list: cmd_shell = cmd.to_shell() logger.info("INFO | [" + cur_filename + ":" + cur_function + "] | " + "Running: " + cmd_shell) ret = run(cmd) if ret != 0: logger.error("ERROR | [" + cur_filename + ":" + cur_function + "] | " + "Problem executing: " + cmd_shell) exit(0)
def main(): logger = logging.getLogger('run_example') init_time = 0 start_time = 0 end_time = 0 time_interval = 1 short_opts = "c:r:h" long_opts = ["config=", "help", "runtime="] # All command line input, get options and arguments try: opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts) except getopt.GetoptError as err: print(str(err)) usage('SCRIPT IS EXITING DUE TO UNRECOGNIZED COMMAND LINE OPTION') for k, v in opts: if k in ('-c', '--config'): # adds the conf file to the list of arguments. args.append(config_launcher.set_conf_file_path(v)) elif k in ('-h', '--help'): usage() exit() elif k in ('-r', '--runtime'): start_time = v end_time = v else: assert False, "UNHANDLED OPTION" if not args: args = None (parm, infiles, moreopt) = config_launcher.parse_launch_args(args, usage, None, logger) p = config_launcher.launch(infiles, moreopt) logger = util.get_logger(p) logger.setLevel(logging.DEBUG) if start_time == 0: start_time = p.getstr('config', 'START_TIME') end_time = p.getstr('config', 'END_TIME') time_interval = p.getstr('config', 'TIME_INTERVAL') # Get the list of processes to call process_list = util.getlist(p.getstr('config', 'PROCESS_LIST')) model_type = p.getstr('config', 'MODEL_TYPE') fcst_vars = util.getlist(p.getstr('config', 'FCST_VARS')) lead_seq = util.getlistint(p.getstr('config', 'LEAD_SEQ')) init_time = start_time while init_time <= end_time: print("") print("****************************************") print("* RUNNING MET+") print("* EVALUATING " + model_type + " at init time: " + init_time) print("****************************************") logger.info("****************************************") logger.info("* RUNNING MET+") logger.info("* EVALUATING " + model_type + " at init time: " + init_time) logger.info("****************************************") for lead in lead_seq: for fcst_var in fcst_vars: # loop over models to compare accums = util.getlist(p.getstr('config', fcst_var + "_ACCUM")) ob_types = util.getlist( p.getstr('config', fcst_var + "_OBTYPE")) for accum in accums: for ob_type in ob_types: if lead < int(accum): continue obs_var = p.getstr('config', ob_type + "_VAR") logger.info("") logger.info("") logger.info("For " + init_time + " F" + str(lead) + ", processing " + model_type + "_" + fcst_var + "_" + accum + " vs " + ob_type + " " + obs_var + "_" + accum) valid_time = util.shift_time(init_time, lead) data_interval = p.getint('config', ob_type + '_DATA_INTERVAL') if int(valid_time[8:10]) % data_interval != 0: logger.warning("No observation for valid time: " + valid_time + ". Skipping...") continue for process in process_list: if process == "pcp_combine": run_pcp = CG_pcp_combine(p, logger) run_pcp.run_at_time(valid_time, accum, ob_type, fcst_var) elif process == "regrid_data_plane": run_regrid = CG_regrid_data_plane(p, logger) run_regrid.run_at_time(valid_time, accum, ob_type) elif process == "grid_stat": run_grid_stat = CG_grid_stat(p, logger) run_grid_stat.run_at_time( init_time, lead, accum, ob_type, fcst_var) else: print("ERROR: Invalid process in process list") exit(1) init_time = util.shift_time(init_time, int(time_interval)) (logger).info("END OF EXECUTION")