def metplus_config():
"""! Create a METplus configuration object that can be
manipulated/modified to
reflect different paths, directories, values, etc. for individual
tests.
"""
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='MTDWrapper',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='MTDWrapper')
produtil.log.postmsg('mtd_wrapper is starting')
# Read in the configuration object CONFIG
config = config_metplus.setup()
logger = util.get_logger(config)
config = ConfigWrapper(config, logger)
return config
except Exception as e:
produtil.log.jlogger.critical('mtd_wrapper failed: %s' % (str(e), ),
exc_info=True)
sys.exit(2)
def get_config_obj():
"""! Create the configuration object that is used by all tests"""
file_list = [
"/Users/cyclopath/dev_MET+/METplus/internal_tests/pytests/produtil"
]
config_obj = config_metplus.setup(file_list[0])
return config_obj
def metplus_config():
"""! Create a METplus configuration object that can be
manipulated/modified to
reflect different paths, directories, values, etc. for individual
tests.
"""
# Read in the configuration object CONFIG
config = config_metplus.setup(util.baseinputconfs)
util.get_logger(config)
return config
def get_config():
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='test run_tc_pairs',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='test run_tc_pairs')
produtil.log.postmsg('unit test for run_tc_pairs is starting')
# Read in the configuration object CONFIG_INST
config_instance = config_metplus.setup()
if 'MET_BASE' not in os.environ:
os.environ['MET_BASE'] = config_instance.getdir('MET_BASE')
return config_instance
def metplus_config():
"""! Generate the METplus config object"""
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False, jobname='TcStatWrapper ',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='TcStatWrapper ')
produtil.log.postmsg('tc_stat_wrapper is starting')
# Read in the configuration object CONFIG
config = config_metplus.setup()
return config
except Exception as e:
produtil.log.jlogger.critical(
'tc_stat_wrapper failed: %s' % (str(e),), exc_info=True)
sys.exit(2)
def __init__(self):
# Setup
produtil.setup.setup(send_dbn=False, jobname='TestProdutilForMETplus')
self.config = config_metplus.setup()
# Create Logger
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
# Create Handler for logging the data to a file
logger_handler = logging.FileHandler('/tmp/produtil_for_METplus.log')
logger_handler.setLevel(logging.DEBUG)
# Create a Formatter to the Handler and add it to the Handler, then add the Handler to the Logger
logger_formatter = logging.Formatter(
'%(asctime)s|%(name)s: %(message)s')
logger_handler.setFormatter(logger_formatter)
self.logger.addHandler(logger_handler)
log_level_name = logging.getLevelName(self.logger.getEffectiveLevel())
self.logger.debug('Logger configured at level: ' + log_level_name)
def metplus_config():
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='TCPairsWrapper ',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='TCPairsWrapper ')
produtil.log.postmsg('tc_pairs_wrapper is starting')
# Read in the configuration object CONFIG
config = config_metplus.setup(util.baseinputconfs)
logger = util.get_logger(config)
return config
except Exception as e:
produtil.log.jlogger.critical('tc_pairs_wrapper failed: %s' %
(str(e), ),
exc_info=True)
sys.exit(2)
# Clean up the tmp directory if it exists
if os.path.isdir(tmp_dir):
util.rmtree(tmp_dir)
msg = ("INFO|[" + cur_function + ":" + cur_filename + "]"
"| Finished extract tiles")
self.logger.info(msg)
if __name__ == "__main__":
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False, jobname='extract_tiles',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='extract_tiles')
produtil.log.postmsg('extract_tiles is starting')
# Read in the configuration object CONFIG_INST
CONFIG_INST = config_metplus.setup()
if 'MET_BASE' not in os.environ:
os.environ['MET_BASE'] = CONFIG_INST.getdir('MET_BASE')
ET = ExtractTilesWrapper(CONFIG_INST, logger=None)
ET.run_all_times()
produtil.log.postmsg('extract_tiles completed')
except Exception as exception:
produtil.log.jlogger.critical(
'extract_tiles failed: %s' % (str(exception),), exc_info=True)
sys.exit(2)
"examples/<example_name>.conf \n")
print("Currently available processes are: ")
for process in self.available_processes:
print(" - {}".format(process))
if __name__ == "__main__":
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='SeriesByLeadWrapper',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='SeriesByLeadWrapper')
produtil.log.postmsg('SeriesByLeadWrapper is starting')
# Read in the configuration object CONFIG
CONFIG = config_metplus.setup()
if 'MET_BASE' not in os.environ:
os.environ['MET_BASE'] = CONFIG.getdir('MET_BASE')
UW = UsageWrapper(CONFIG, logger=None)
UW.run_all_times()
produtil.log.postmsg('series analysis by lead time completed')
except Exception as exc:
produtil.log.jlogger.critical('SeriesByLeadWrapper failed: %s' %
(str(exc), ),
exc_info=True)
sys.exit(2)
else:
adj_lon = lon
return adj_lon
if __name__ == "__main__":
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='CyclonePlotter',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='CyclonePlotter')
produtil.log.postmsg('CyclonePlotter is starting')
#pylint:disable=invalid-name
p = config_metplus.setup()
if 'MET_BASE' not in os.environ:
os.environ['MET_BASE'] = p.getdir('MET_BASE')
# Request data extraction and plot generation.
#pylint:disable=invalid-name
cyclone = CyclonePlotterWrapper(p, None)
cyclone.run_all_times()
produtil.log.postmsg('CyclonePlotter completed')
except Exception as e:
produtil.log.jlogger.critical('CyclonePlotter failed: %s' % (str(e), ),
exc_info=True)
sys.exit(2)
def get_test_config():
config_instance = config_metplus.setup()
return config_instance
def main():
"""!Main program.
Master METplus script that invokes the necessary Python scripts
to perform various activities, such as series analysis."""
# Setup Task logger, Until Conf object is created, Task logger is
# only logging to tty, not a file.
logger = logging.getLogger('master_metplus')
logger.info('Starting METplus v%s', util.get_version_number())
# Parse arguments, options and return a config instance.
conf = config_metplus.setup(filename='master_metplus.py')
# NOW we have a conf object p, we can now get the logger
# and set the handler to write to the LOG_METPLUS
# TODO: Frimel setting up logger file handler.
# Setting up handler i.e util.get_logger should be moved to
# the setup wrapper and encapsulated in the config object.
# than you would get it this way logger=p.log(). The config
# object has-a logger we want.
logger = util.get_logger(conf)
logger.info('Running METplus v%s called with command: %s',
util.get_version_number(), ' '.join(sys.argv))
# check for deprecated config items and warn user to remove/replace them
util.check_for_deprecated_config(conf, logger)
config = ConfigWrapper(conf, logger)
# set staging dir to OUTPUT_BASE/stage if not set
if not config.has_option('dir', 'STAGING_DIR'):
config.set('dir', 'STAGING_DIR',
os.path.join(config.getdir('OUTPUT_BASE'), "stage"))
# handle dir to write temporary files
util.handle_tmp_dir(config)
# This is available in each subprocess from os.system BUT
# we also set it in each process since they may be called stand alone.
os.environ['MET_BASE'] = config.getdir('MET_BASE')
config.env = os.environ.copy()
# Use config object to get the list of processes to call
process_list = util.getlist(config.getstr('config', 'PROCESS_LIST'))
# Keep this comment.
# When running commands in the process_list, reprocess the
# original command line using (item))[sys.argv[1:]].
#
# You could call each task (ie. run_tc_pairs.py) without any args since
# the final METPLUS_CONF file was just created from config_metplus.setup,
# and each task, also calls setup, which use an existing final conf
# file over command line args.
#
# both work ...
# Note: Using (item))sys.argv[1:], is preferable since
# it doesn't depend on the conf file existing.
processes = []
for item in process_list:
try:
logger = config.log(item)
command_builder = \
getattr(sys.modules[__name__],
item + "Wrapper")(config, logger)
# if Usage specified in PROCESS_LIST, print usage and exit
if item == 'Usage':
command_builder.run_all_times()
exit(1)
except AttributeError:
raise NameError("Process %s doesn't exist" % item)
processes.append(command_builder)
loop_order = config.getstr('config', 'LOOP_ORDER', '')
if loop_order == '':
loop_order = config.getstr('config', 'LOOP_METHOD')
if loop_order == "processes":
for process in processes:
# referencing using repr(process.app_name) in
# log since it may be None,
# if not set in the command builder subclass' contsructor,
# and no need to generate an exception because of that.
produtil.log.postmsg('master_metplus Calling run_all_times '
'in: %s wrapper.' % repr(process.app_name))
process.run_all_times()
elif loop_order == "times":
util.loop_over_times_and_call(config, processes)
else:
logger.error("Invalid LOOP_METHOD defined. " + \
"Options are processes, times")
exit()
# scrub staging directory if requested
if config.getbool('config', 'SCRUB_STAGING_DIR', False) and\
os.path.exists(config.getdir('STAGING_DIR')):
staging_dir = config.getdir('STAGING_DIR')
logger.info("Scrubbing staging dir: %s", staging_dir)
shutil.rmtree(staging_dir)
# rewrite final conf so it contains all of the default values used
util.write_final_conf(conf, logger)
logger.info('METplus has successfully finished running.')
exit()
def main():
"""!Main program.
Master MET+ script that invokes the necessary Python scripts
to perform various activities, such as series analysis."""
# Job Logger
produtil.log.jlogger.info('Top of master_metplus')
# Used for logging and usage statment
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
# Setup Task logger, Until Conf object is created, Task logger is
# only logging to tty, not a file.
logger = logging.getLogger('master_metplus')
logger.info('logger Top of master_metplus.')
# Parse arguments, options and return a config instance.
p = config_metplus.setup(filename=cur_filename)
# NOW we have a conf object p, we can now get the logger
# and set the handler to write to the LOG_METPLUS
# TODO: Frimel setting up logger file handler.
# Setting up handler i.e util.get_logger should be moved to
# the setup wrapper and encapsulated in the config object.
# than you would get it this way logger=p.log(). The config
# object has-a logger we want.
logger = util.get_logger(p)
#logger.info('Top of master_metplus after conf file setup.')
# This is available in each subprocess from os.system BUT
# we also set it in each process since they may be called stand alone.
os.environ['MET_BASE'] = p.getdir('MET_BASE')
# Use config object to get the list of processes to call
process_list = util.getlist(p.getstr('config', 'PROCESS_LIST'))
# Keep this comment.
# When running commands in the process_list, reprocess the
# original command line using (item))[sys.argv[1:]].
#
# You could call each task (ie. run_tc_pairs.py) without any args since
# the final METPLUS_CONF file was just created from config_metplus.setup,
# and each task, also calls setup, which use an existing final conf
# file over command line args.
#
# both work ...
# Note: Using (item))sys.argv[1:], is preferable since
# it doesn't depend on the conf file existing.
processes = []
for item in process_list:
try:
logger = p.log(item)
command_builder = getattr(sys.modules[__name__], item+"Wrapper")(p, logger)
except AttributeError:
raise NameError("Process %s doesn't exist" % item)
exit()
processes.append(command_builder)
if p.getstr('config', 'LOOP_METHOD') == "processes":
for process in processes:
# referencing using repr(process.app_name) in log since it may be None,
# if not set in the command builder subclass' contsructor, and no need to
# generate an exception because of that.
produtil.log.postmsg('master_metplus Calling run_all_times '
'in: %s wrapper.' % repr(process.app_name))
process.run_all_times()
elif p.getstr('config', 'LOOP_METHOD') == "times":
use_init = p.getbool('config', 'LOOP_BY_INIT', True)
if use_init:
time_format = p.getstr('config', 'INIT_TIME_FMT')
start_t = p.getstr('config', 'INIT_BEG')
end_t = p.getstr('config', 'INIT_END')
time_interval = p.getint('config', 'INIT_INC')
else:
time_format = p.getstr('config', 'VALID_TIME_FMT')
start_t = p.getstr('config', 'VALID_BEG')
end_t = p.getstr('config', 'VALID_END')
time_interval = p.getint('config', 'VALID_INC')
if time_interval < 60:
print("ERROR: time_interval parameter must be greater than 60 seconds")
exit(1)
loop_time = calendar.timegm(time.strptime(start_t, time_format))
end_time = calendar.timegm(time.strptime(end_t, time_format))
while loop_time <= end_time:
run_time = time.strftime("%Y%m%d%H%M", time.gmtime(loop_time))
logger.info("****************************************")
logger.info("* RUNNING MET+")
if use_init:
logger.info("* at init time: " + run_time)
else:
logger.info("* at valid time: " + run_time)
logger.info("****************************************")
for process in processes:
# Set valid time to -1 if using init and vice versa
if use_init:
process.run_at_time(run_time, -1)
else:
process.run_at_time(-1, run_time)
process.clear()
loop_time += time_interval
else:
print("ERROR: Invalid LOOP_METHOD defined. " + \
"Options are processes, times")
exit()
exit()
