def __init__(self, config, logger=None):
"""!Class for Creating and Running External Programs.
It was intended to handle the MET executables but
can be used by other executables."""
self.logger = logger
self.config = ConfigWrapper(config, self.logger)
self.verbose = self.config.getstr('config', 'LOG_MET_VERBOSITY', '2')
def test_preprocess_file_unzipped():
conf = ConfigWrapper(metplus_config(), None)
stage_dir = conf.getdir('STAGING_DIR',
os.path.join(conf.getdir('OUTPUT_BASE'), "stage"))
filepath = conf.getdir(
'METPLUS_BASE') + "/internal_tests/data/zip/testfile4.txt"
outpath = util.preprocess_file(filepath, None, conf)
assert (filepath == outpath and os.path.exists(outpath))
def get_params(param_a, param_b):
params_a, params_b = get_param_list(param_a, param_b)
logger = logging.getLogger('master_metplus')
# read A confs
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(params_a, None, None, logger)
p = ConfigWrapper(config_launcher.launch(infiles, moreopt), None)
# read B confs
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(params_b, None, None, logger)
p_b = ConfigWrapper(config_launcher.launch(infiles, moreopt), None)
return p, p_b
def metplus_config():
"""! Create a METplus configuration object that can be
manipulated/modified to
reflect different paths, directories, values, etc. for individual
tests.
"""
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='MTDWrapper',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='MTDWrapper')
produtil.log.postmsg('mtd_wrapper is starting')
# Read in the configuration object CONFIG
config = config_metplus.setup()
logger = util.get_logger(config)
config = ConfigWrapper(config, logger)
return config
except Exception as e:
produtil.log.jlogger.critical('mtd_wrapper failed: %s' % (str(e), ),
exc_info=True)
sys.exit(2)
def test_parse_var_list_fcst_and_obs():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'FCST_VAR1_NAME', "FNAME1")
conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12")
conf.set('config', 'FCST_VAR2_NAME', "FNAME2")
conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22")
conf.set('config', 'OBS_VAR1_NAME', "ONAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12")
conf.set('config', 'OBS_VAR2_NAME', "ONAME2")
conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22")
var_list = util.parse_var_list(cu)
assert(var_list[0]['fcst_name'] == "FNAME1" and \
var_list[0]['obs_name'] == "ONAME1" and \
var_list[1]['fcst_name'] == "FNAME1" and \
var_list[1]['obs_name'] == "ONAME1" and \
var_list[2]['fcst_name'] == "FNAME2" and \
var_list[2]['obs_name'] == "ONAME2" and \
var_list[3]['fcst_name'] == "FNAME2" and \
var_list[3]['obs_name'] == "ONAME2" and \
var_list[0]['fcst_level'] == "FLEVELS11" and \
var_list[0]['obs_level'] == "OLEVELS11" and \
var_list[1]['fcst_level'] == "FLEVELS12" and \
var_list[1]['obs_level'] == "OLEVELS12" and \
var_list[2]['fcst_level'] == "FLEVELS21" and \
var_list[2]['obs_level'] == "OLEVELS21" and \
var_list[3]['fcst_level'] == "FLEVELS22" and \
var_list[3]['obs_level'] == "OLEVELS22")
def test_get_lead_sequence_lead_list(key, value):
input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'LEAD_SEQ', key)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = value
assert (test_seq == lead_seq)
def test_get_lead_sequence_lead():
input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'LEAD_SEQ', "3,6,9,12")
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = [3, 6, 9, 12]
assert (test_seq == lead_seq)
def test_get_lead_sequence_init(key, value):
input_dict = {'valid': datetime.datetime(2019, 02, 01, key)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'INIT_SEQ', "0, 12")
conf.set('config', 'LEAD_SEQ_MAX', 36)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = value
assert (test_seq == lead_seq)
def test_get_lead_sequence_init_min_10():
input_dict = {'valid': datetime.datetime(2019, 2, 1, 12)}
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'INIT_SEQ', "0, 12")
conf.set('config', 'LEAD_SEQ_MAX', 24)
conf.set('config', 'LEAD_SEQ_MIN', 10)
test_seq = util.get_lead_sequence(cu, input_dict)
lead_seq = [12, 24]
assert (test_seq == lead_seq)
def metplus_config():
try:
if 'JLOGFILE' in os.environ:
produtil.setup.setup(send_dbn=False,
jobname='TCPairsWrapper ',
jlogfile=os.environ['JLOGFILE'])
else:
produtil.setup.setup(send_dbn=False, jobname='TCPairsWrapper ')
produtil.log.postmsg('tc_pairs_wrapper is starting')
# Read in the configuration object CONFIG
config = config_metplus.setup()
logger = util.get_logger(config)
config = ConfigWrapper(config, logger)
return config
except Exception as e:
produtil.log.jlogger.critical('tc_pairs_wrapper failed: %s' %
(str(e), ),
exc_info=True)
sys.exit(2)
def test_parse_var_list_fcst_only():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'FCST_VAR1_NAME', "NAME1")
conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12")
conf.set('config', 'FCST_VAR1_THRESH', ">1, >2")
conf.set('config', 'OBS_VAR1_OPTIONS', "OOPTIONS11")
var_list = util.parse_var_list(cu)
assert(var_list[0]['fcst_name'] == "NAME1" and \
var_list[0]['obs_name'] == "NAME1" and \
var_list[1]['fcst_name'] == "NAME1" and \
var_list[1]['obs_name'] == "NAME1" and \
var_list[0]['fcst_level'] == "LEVELS11" and \
var_list[0]['obs_level'] == "LEVELS11" and \
var_list[1]['fcst_level'] == "LEVELS12" and \
var_list[1]['obs_level'] == "LEVELS12" and \
var_list[0]['fcst_thresh'] == var_list[0]['obs_thresh'] and \
var_list[1]['fcst_thresh'] == var_list[1]['obs_thresh'] and \
var_list[0]['fcst_extra'] == "" and \
var_list[0]['obs_extra'] == "OOPTIONS11" and \
var_list[1]['fcst_extra'] == "" and \
var_list[1]['obs_extra'] == "OOPTIONS11"
)
def test_parse_var_list_fcst_and_obs_and_both():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'OBS_VAR1_NAME', "ONAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12")
conf.set('config', 'FCST_VAR2_NAME', "FNAME2")
conf.set('config', 'FCST_VAR2_LEVELS', "FLEVELS21, FLEVELS22")
conf.set('config', 'FCST_VAR3_NAME', "FNAME3")
conf.set('config', 'FCST_VAR3_LEVELS', "FLEVELS31, FLEVELS32")
conf.set('config', 'OBS_VAR3_NAME', "ONAME3")
conf.set('config', 'OBS_VAR3_LEVELS', "OLEVELS31, OLEVELS32")
var_list = util.parse_var_list(cu)
assert(var_list[0].fcst_name == "ONAME1" and \
var_list[0].obs_name == "ONAME1" and \
var_list[1].fcst_name == "ONAME1" and \
var_list[1].obs_name == "ONAME1" and \
var_list[2].fcst_name == "FNAME2" and \
var_list[2].obs_name == "FNAME2" and \
var_list[3].fcst_name == "FNAME2" and \
var_list[3].obs_name == "FNAME2" and \
var_list[4].fcst_name == "FNAME3" and \
var_list[4].obs_name == "ONAME3" and \
var_list[5].fcst_name == "FNAME3" and \
var_list[5].obs_name == "ONAME3" and \
var_list[0].fcst_level == "OLEVELS11" and \
var_list[0].obs_level == "OLEVELS11" and \
var_list[1].fcst_level == "OLEVELS12" and \
var_list[1].obs_level == "OLEVELS12" and \
var_list[2].fcst_level == "FLEVELS21" and \
var_list[2].obs_level == "FLEVELS21" and \
var_list[3].fcst_level == "FLEVELS22" and \
var_list[3].obs_level == "FLEVELS22" and \
var_list[4].fcst_level == "FLEVELS31" and \
var_list[4].obs_level == "OLEVELS31" and \
var_list[5].fcst_level == "FLEVELS32" and \
var_list[5].obs_level == "OLEVELS32" )
def test_parse_var_list_obs():
conf = metplus_config()
cu = ConfigWrapper(conf, None)
conf.set('config', 'OBS_VAR1_NAME', "NAME1")
conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12")
conf.set('config', 'OBS_VAR2_NAME', "NAME2")
conf.set('config', 'OBS_VAR2_LEVELS', "LEVELS21, LEVELS22")
var_list = util.parse_var_list(cu)
assert(var_list[0].fcst_name == "NAME1" and \
var_list[0].obs_name == "NAME1" and \
var_list[1].fcst_name == "NAME1" and \
var_list[1].obs_name == "NAME1" and \
var_list[2].fcst_name == "NAME2" and \
var_list[2].obs_name == "NAME2" and \
var_list[3].fcst_name == "NAME2" and \
var_list[3].obs_name == "NAME2" and \
var_list[0].fcst_level == "LEVELS11" and \
var_list[0].obs_level == "LEVELS11" and \
var_list[1].fcst_level == "LEVELS12" and \
var_list[1].obs_level == "LEVELS12" and \
var_list[2].fcst_level == "LEVELS21" and \
var_list[2].obs_level == "LEVELS21" and \
var_list[3].fcst_level == "LEVELS22" and \
var_list[3].obs_level == "LEVELS22")
def __init__(self,
info_basedir,
phase='',
split='',
to_read=(),
seq_len=1,
n_seg=3,
seq_strides=('uniform', ),
aug_video=True,
transformer=collections.defaultdict(lambda: None),
run_n_sample=0,
shuffle=True,
classes=None):
# config handler
attrs = locals()
ConfigWrapper.__init__(self, attrs)
# info loader
with open(os.path.join(info_basedir, 'dataset_info.json'), 'r') as f:
self.dataset_info = json.load(f)
self.n_class = self.dataset_info['n_class']
infos = dict()
t_n_sample = None
for mod_name in self.dataset_info['modality'].keys():
mod = self.dataset_info['modality'][mod_name]
if os.path.exists(
os.path.join(
info_basedir,
'{}_{}_{}.json'.format(phase, mod_name, split))):
with open(
os.path.join(
info_basedir,
'{}_{}_{}.json'.format(phase, mod_name, split)),
'r') as f:
infos[mod_name] = json.load(f)
# keep classes
if classes:
infos[mod_name] = [
x for x in infos[mod_name] if x['label'] in classes
]
if t_n_sample is None:
t_n_sample = len(infos[mod_name])
else:
if len(infos[mod_name]) != t_n_sample:
RuntimeError(
'sample number wrong for modality {}'.format(
mod_name))
# else:
# print('warning, missing info for modal:{}'.format(mod_name))
# register mode loader
def get_factory(mod_name):
def getter(item, context):
return self._get_mode(mod_name, item, context)
return getter
self.__setattr__('get_{}'.format(mod_name), get_factory(mod_name))
self.infos = infos
# sample iterator
if run_n_sample == 0:
run_n_sample = t_n_sample
self.run_n_sample = run_n_sample
self.n_sample = t_n_sample
n_epoch = int(math.ceil(self.run_n_sample * 1.0 / self.n_sample))
n_sample_ind_iter = []
for _ in range(n_epoch):
if shuffle:
iter_epoch = np.random.permutation(self.n_sample).tolist()
else:
iter_epoch = range(self.n_sample)
n_sample_ind_iter = n_sample_ind_iter + list(iter_epoch)
n_sample_ind_iter = n_sample_ind_iter[:self.run_n_sample]
self.n_sample_ind_iter = n_sample_ind_iter
self.righthand = 4
self.lefthand = 7
self.upperbody = [0, 1, 2, 3, 4, 5, 6, 7, 8, 11, 14, 15, 16, 17]
self.fullbody = [i for i in range(18)]
self.box_length = 40
#average images
import scipy.io as sio
vgg_model = sio.loadmat(
'/data6/SRIP18-7/codes/PCNN/pretrained_models/imagenet-vgg-f.mat')
flownet_model = sio.loadmat(
'/data6/SRIP18-7/codes/PCNN/pretrained_models/flow_net.mat')
self.rgb_avg = torch.from_numpy(vgg_model['normalization'][0][0][0] /
255.0).float().permute(2, 0, 1)
self.flow_avg = torch.from_numpy(
flownet_model['normalization'][0][0][0] / 255.0).float().permute(
2, 0, 1)
def test_preprocess_file_none():
conf = ConfigWrapper(metplus_config(), None)
outpath = util.preprocess_file(None, None, conf)
assert (outpath is None)
def test_getseconds(key, value):
conf = ConfigWrapper(metplus_config(), None)
conf.set('config', 'TEST_SECONDS', key)
seconds = conf.getseconds('config', 'TEST_SECONDS')
assert (seconds == value)
def main():
"""!Main program.
Master METplus script that invokes the necessary Python scripts
to perform various activities, such as series analysis."""
# Setup Task logger, Until Conf object is created, Task logger is
# only logging to tty, not a file.
logger = logging.getLogger('master_metplus')
logger.info('Starting METplus v%s', util.get_version_number())
# Parse arguments, options and return a config instance.
conf = config_metplus.setup(filename='master_metplus.py')
# NOW we have a conf object p, we can now get the logger
# and set the handler to write to the LOG_METPLUS
# TODO: Frimel setting up logger file handler.
# Setting up handler i.e util.get_logger should be moved to
# the setup wrapper and encapsulated in the config object.
# than you would get it this way logger=p.log(). The config
# object has-a logger we want.
logger = util.get_logger(conf)
logger.info('Running METplus v%s called with command: %s',
util.get_version_number(), ' '.join(sys.argv))
# check for deprecated config items and warn user to remove/replace them
util.check_for_deprecated_config(conf, logger)
config = ConfigWrapper(conf, logger)
# set staging dir to OUTPUT_BASE/stage if not set
if not config.has_option('dir', 'STAGING_DIR'):
config.set('dir', 'STAGING_DIR',
os.path.join(config.getdir('OUTPUT_BASE'), "stage"))
# handle dir to write temporary files
util.handle_tmp_dir(config)
# This is available in each subprocess from os.system BUT
# we also set it in each process since they may be called stand alone.
os.environ['MET_BASE'] = config.getdir('MET_BASE')
config.env = os.environ.copy()
# Use config object to get the list of processes to call
process_list = util.getlist(config.getstr('config', 'PROCESS_LIST'))
# Keep this comment.
# When running commands in the process_list, reprocess the
# original command line using (item))[sys.argv[1:]].
#
# You could call each task (ie. run_tc_pairs.py) without any args since
# the final METPLUS_CONF file was just created from config_metplus.setup,
# and each task, also calls setup, which use an existing final conf
# file over command line args.
#
# both work ...
# Note: Using (item))sys.argv[1:], is preferable since
# it doesn't depend on the conf file existing.
processes = []
for item in process_list:
try:
logger = config.log(item)
command_builder = \
getattr(sys.modules[__name__],
item + "Wrapper")(config, logger)
# if Usage specified in PROCESS_LIST, print usage and exit
if item == 'Usage':
command_builder.run_all_times()
exit(1)
except AttributeError:
raise NameError("Process %s doesn't exist" % item)
processes.append(command_builder)
loop_order = config.getstr('config', 'LOOP_ORDER', '')
if loop_order == '':
loop_order = config.getstr('config', 'LOOP_METHOD')
if loop_order == "processes":
for process in processes:
# referencing using repr(process.app_name) in
# log since it may be None,
# if not set in the command builder subclass' contsructor,
# and no need to generate an exception because of that.
produtil.log.postmsg('master_metplus Calling run_all_times '
'in: %s wrapper.' % repr(process.app_name))
process.run_all_times()
elif loop_order == "times":
util.loop_over_times_and_call(config, processes)
else:
logger.error("Invalid LOOP_METHOD defined. " + \
"Options are processes, times")
exit()
# scrub staging directory if requested
if config.getbool('config', 'SCRUB_STAGING_DIR', False) and\
os.path.exists(config.getdir('STAGING_DIR')):
staging_dir = config.getdir('STAGING_DIR')
logger.info("Scrubbing staging dir: %s", staging_dir)
shutil.rmtree(staging_dir)
# rewrite final conf so it contains all of the default values used
util.write_final_conf(conf, logger)
logger.info('METplus has successfully finished running.')
exit()
def launch(file_list, moreopt, cycle=None, init_dirs=True, prelaunch=None):
for filename in file_list:
if not isinstance(filename, str):
raise TypeError('First input to metplus.config.for_initial_job '
'must be a list of strings.')
conf = METplusLauncher()
logger = conf.log()
cu = ConfigWrapper(conf, logger)
# set config variable for current time
conf.set('config', 'CLOCK_TIME',
datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
# Read in and parse all the conf files.
for filename in file_list:
conf.read(filename)
logger.info("%s: Parsed this file" % (filename, ))
# Overriding or passing in specific conf items on the command line
# ie. config.NEWVAR="a new var" dir.SOMEDIR=/override/dir/path
# If spaces, seems like you need double quotes on command line.
if moreopt:
for section, options in moreopt.items():
if not conf.has_section(section):
conf.add_section(section)
for option, value in options.items():
logger.info('Override: %s.%s=%s' %
(section, option, repr(value)))
conf.set(section, option, value)
# All conf files and command line options have been parsed.
# So lets set and add specific log variables to the conf object
# based on the conf log template values.
util.set_logvars(conf)
# Determine if final METPLUS_CONF file already exists on disk.
# If it does, use it instead.
confloc = conf.getloc('METPLUS_CONF')
# Received feedback: Users want to overwrite the final conf file if it exists.
# Not overwriting is annoying everyone, since when one makes a conf file edit
# you have to remember to remove the final conf file.
# Originally based on a hwrf workflow. since launcher task only runs once,
# and all following tasks use the generated conf file.
#finalconfexists = util.file_exists(confloc)
# Force setting to False, so conf always gets overwritten
finalconfexists = False
if finalconfexists:
logger.warning(
'IGNORING all parsed conf file(s) AND any command line options or arguments, if given.'
)
logger.warning('INSTEAD, Using Existing final conf: %s' % (confloc))
del logger
del conf
conf = METplusLauncher()
logger = conf.log()
conf.read(confloc)
# Set moreopt to None, in case it is not None, We do not want to process
# more options since using existing final conf file.
moreopt = None
# Place holder for when workflow is developed in METplus.
# rocoto does not initialize the dirs, it returns here.
# if not init_dirs:
# if prelaunch is not None:
# prelaunch(conf,logger,cycle)
# return conf
# Initialize the output directories
produtil.fileop.makedirs(cu.getdir('OUTPUT_BASE', logger), logger=logger)
# A user my set the confloc METPLUS_CONF location in a subdir of OUTPUT_BASE
# or even in another parent directory altogether, so make thedirectory
# so the metplus_final.conf file can be written.
if not os.path.exists(realpath(dirname(confloc))):
produtil.fileop.makedirs(realpath(dirname(confloc)), logger=logger)
# set METPLUS_BASE conf to location of scripts used by METplus
# warn if user has set METPLUS_BASE to something different
# in their conf file
user_metplus_base = cu.getdir('METPLUS_BASE', METPLUS_BASE)
if realpath(user_metplus_base) != realpath(METPLUS_BASE):
logger.warning('METPLUS_BASE from the conf files has no effect.'+\
' Overriding to '+METPLUS_BASE)
conf.set('dir', 'METPLUS_BASE', METPLUS_BASE)
# do the same for PARM_BASE
user_parm_base = cu.getdir('PARM_BASE', PARM_BASE)
if realpath(user_parm_base) != realpath(PARM_BASE):
logger.error('PARM_BASE from the config ({}) '.format(user_parm_base) +\
'differs from METplus parm base ({}). '.format(PARM_BASE))
logger.error('Please remove PARM_BASE from any config file. Set the ' +\
'environment variable METPLUS_PARM_BASE to change where ' +\
'the METplus wrappers look for config files.')
exit(1)
conf.set('dir', 'PARM_BASE', PARM_BASE)
version_number = util.get_version_number()
conf.set('config', 'METPLUS_VERSION', version_number)
# print config items that are set automatically
for var in ('METPLUS_BASE', 'PARM_BASE'):
expand = cu.getdir(var)
logger.info('Setting [dir] %s to %s' % (var, expand))
# Place holder for when workflow is developed in METplus.
# if prelaunch is not None:
# prelaunch(conf,logger,cycle)
# writes the METPLUS_CONF used by all tasks.
if not finalconfexists:
logger.info('METPLUS_CONF: %s written here.' % (confloc, ))
with open(confloc, 'wt') as f:
conf.write(f)
return conf
class CommandRunner(object):
"""! Class for Creating and Running External Programs
"""
def __init__(self, config, logger=None):
"""!Class for Creating and Running External Programs.
It was intended to handle the MET executables but
can be used by other executables."""
self.logger = logger
self.config = ConfigWrapper(config, self.logger)
self.verbose = self.config.getstr('config', 'LOG_MET_VERBOSITY', '2')
def run_cmd(self,
cmd,
env=None,
ismetcmd=True,
app_name=None,
run_inshell=False,
log_theoutput=False,
**kwargs):
"""!The command cmd is a string which is converted to a produtil
exe Runner object and than run. Output of the command may also
be redirected to either METplus log, MET log, or TTY.
Some subclasses of CommandBuilder ie. series_by_init_wrapper, run
non MET commands ie. convert, in addition to MET binary commands,
ie. regrid_data_plane.
Args:
@param cmd: A string, Command used in the produtil exe Runner object.
@param env: Default None, environment for run to pass in, uses
os.environ if not set.
@param ismetcmd: Default True, Will direct output to METplus log,
Metlog , or TTY. Set to False and use the other keywords as needed.
@param app_name: Used only when ismetcmd=True, The name of the exectable
being run.
@param run_inshell: Used only when ismetcmd=False, will Create a
runner object with the cmd being run through a shell, exe('sh')['-c', cmd]
This is required by commands, such as ncdump that are redirecting
output to a file, and other commands such as the convert command
when creating animated gifs.
@param log_theoutput: Used only when ismetcmd=False, will redirect
the stderr and stdout to a the METplus log file or tty.
DO Not set to True if the command is redirecting output to a file.
@param kwargs Other options sent to the produtil Run constructor
"""
if cmd is None:
return cmd
# if env not set, use os.environ
if env is None:
env = os.environ
self.logger.info("RUNNING: %s" % cmd)
if ismetcmd:
# self.app_name MUST be defined in the subclass' constructor,
# this code block is a safety net in case that was not done.
# self.app_name is used to generate the MET log output name,
# if output is directed there, based on the conf settings.
#
# cmd.split()[0] 'should' be the /path/to/<some_met_binary>
if not app_name:
app_name = os.path.basename(cmd.split()[0])
self.logger.warning('MISSING self.app_name, '
'setting name to: %s' % repr(app_name))
self.logger.warning(
'Fix the code and edit the following objects'
' contructor: %s, ' % repr(self))
# Determine where to send the output from the MET command.
log_dest = self.cmdlog_destination(cmdlog=app_name + '.log')
# KEEP This comment as a reference note.
# Run the executable in a new process instead of through a shell.
# FYI. We were originally running the command through a shell
# which also works just fine. IF we go back to running through
# a shell, The string ,cmd, is formatted exactly as is needed.
# By formatted, it is as it would be when typed at the shell prompt.
# This includes, for example, quoting or backslash escaping filenames
# with spaces in them.
# Run the executable and pass the arguments as a sequence.
# Split the command in to a sequence using shell syntax.
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_dest:
self.logger.info("app_name is: %s, output sent to: %s" %
(app_name, log_dest))
#cmd = exe('sh')['-c', cmd].err2out() >> log_dest
cmd = exe(the_exe)[the_args].env(**env).err2out() >> log_dest
else:
#cmd = exe('sh')['-c', cmd].err2out()
cmd = exe(the_exe)[the_args].env(**env).err2out()
else:
# This block is for all the Non-MET commands
# Some commands still need to be run in a shell in order to work.
#
# There are currently 3 cases of non met commnds that need to be handled.
# case 1. Redirecting to a file in cmd string, which must be through a shell
# ie. cmd = ncdump ...series_F006/min.nc > .../min.txt
# cmd = exe('sh')['-c', cmd]
# case 2. Running the executable directly, w/ arguments, NO redirection
# ie. ncap2, cmd = exe(the_exe)[the_args]
# case 3. Runnng the command and logging the output to
# log_dest
if run_inshell:
# set the_exe to log command has finished running
the_exe = shlex.split(cmd)[0]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd = exe('sh')['-c', cmd].env(**env).err2out() >> log_dest
else:
cmd = exe('sh')['-c', cmd].env(**env)
else:
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd = exe(the_exe)[the_args].env(
**env).err2out() >> log_dest
else:
cmd = exe(the_exe)[the_args].env(**env)
ret = 0
# run app unless DO_NOT_RUN_EXE is set to True
if not self.config.getbool('config', 'DO_NOT_RUN_EXE', False):
ret = run(cmd, **kwargs)
self.logger.debug('Finished running {}'.format(the_exe))
return (ret, cmd)
# TODO: Refactor seriesbylead.
# For now we are back to running through a shell.
# Can not run its cmd string, unless we run through a shell.
# TODO refactor commands that are redirecting to a file.
# They should be brought in to the produtil framework
# the commands should not be required to run through a shell
# in order to work. we can either capture the output and write
# to a file or pass in the output_file and build the runner
# object usng exe('ncump')['file.nc'].out(output_file,append=False)
# The non met command where ismetcmd=False and log_theoutput=False block above
# was created specfically for running any command that is capturing its
# output to a file. Such as in series by lead.
# ie. cmd = '/usr/local/bin/ncdump ...series_F006/min.nc > .../min.txt'
# The way those commands,as of 2018Apr20, are being defined,
# the > redirection is in the command. We don't want
# to send the command cmd output to a log file, it will mess up the
# intention. Also, cmd can not be run when decontructed with shlex
# You can't have a redirect symbol as an argument, it gets quoted
# and I'm not sure how to get around that ...
# /usr/local/bin/ncdump ...series_F006/min.nc ">" .../min.txt
# We don't want to use run_cmd since that function may redirect
# output to a log file.
# if cmdlog=None. The returned value is either the METplus log
# or None
def cmdlog_destination(self, cmdlog=None):
"""!Returns the location of where the command output will be sent.
The METplus log, the MET log, or tty.
Args:
@param cmdlog: The cmdlog is a filename, any path info is removed.
It is joined with LOG_DIR. If cmdlog is None,
output is sent to either the METplus log or TTY.
@returns log_dest: The destination of where to send the command output.
"""
# Check the cmdlog argument.
# ie. if cmdlog = '', or '/', or trailing slash /path/blah.log/ etc...,
# os.path.basename returns '', and we can't write to '',
# so set cmdlog to None.
if cmdlog:
cmdlog = os.path.basename(cmdlog)
if not cmdlog: cmdlog = None
# Set the default destination to None, which will be TTY
cmdlog_dest = None
# metpluslog is the setting used to determine if output is sent to either
# a log file or tty.
# metpluslog includes /path/filename.
metpluslog = self.config.getstr('config', 'LOG_METPLUS', '')
# This block determines where to send the command output, cmdlog_dest.
# To the METplus log, a MET log, or tty.
# If no metpluslog, cmlog_dest is None, which should be interpreted as tty.
if metpluslog:
log_met_output_to_metplus = self.config.getbool(
'config', 'LOG_MET_OUTPUT_TO_METPLUS')
# If cmdlog is None send output to metpluslog.
if log_met_output_to_metplus or not cmdlog:
cmdlog_dest = metpluslog
else:
log_timestamp = self.config.getstr('config', 'LOG_TIMESTAMP',
'')
if log_timestamp:
cmdlog_dest = os.path.join(self.config.getdir('LOG_DIR'),
cmdlog + '_' + log_timestamp)
else:
cmdlog_dest = os.path.join(self.config.getdir('LOG_DIR'),
cmdlog)
# If cmdlog_dest None we will not redirect output to a log file
# when building the Runner object, so it will end up going to tty.
return cmdlog_dest
# This method SHOULD ONLY BE USED by wrappers that build their cmd
# outside of the command_builder.py get_command() method
# ie. such as tc_pairs wrapper. Objects that fully use the CommandBuilder
# already have the metverbosity set in the command.
def insert_metverbosity_opt(self, cmd=None):
"""!Returns the cmd with the verbosity option inserted
and set after the first space found in the cmd string or
after the cmd string if there are no spaces.
There is NO CHECKING to see if the verbosity is already
inserted in the command. If cmd is None, None is returned.
Args:
@param cmd: One string, The cmd string to insert the -v option.
@returns cmd: The cmd string w/ -v <level:1-5> inserted
after the first white space or end if no
spaces. If cmd is None, None is returned.
"""
if cmd:
verbose_opt = " -v " + str(self.verbose) + " "
# None splits on whitespace space, tab, newline, return, formfeed
cmd_split = cmd.split(None, 1)
# Handle two cases of splitting.
# /path/to/cmd
# /path/to/cmd blah blah blah ....
if len(cmd_split) == 1:
cmd = cmd_split[0] + verbose_opt
elif len(cmd_split) == 2:
cmd = cmd_split[0] + verbose_opt + cmd_split[1]
else:
self.logger.debug('Can not Insert MET verbosity option, '
'command unchanged, using: %s .' % repr(cmd))
return cmd