def main_tape():
postmsg('hwrf_archive tape step starting')
conf = hwrf_expt.conf
logger = conf.log('archive.disk')
archive = conf.getloc('archive', 'NONE')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hwrf_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
logger.info('Disk archiving enabled.')
logger.info('Nothing to do in the HPSS archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hwrf_archive tape step does nothing when using disk '
'archives.')
return
elif archive[0:5] != 'hpss:' and archive[0:5] != 'hpsz:':
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
if makethedir:
adir = os.path.dirname(archive[5:])
logger.info('%s: make this HPSS directory, even if it exists' %
(adir, ))
mcmd = exe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mcmd, logger=logger)
if archive[0:5] == 'hpss:':
path = archive[5:]
flags = '-cvp'
cmd = exe(conf.getexe('htar'))[flags + 'f', path, '-L',
'-'] << files
elif archive[0:5] == 'hpsz:':
topath = archive[5:]
frompath = conf.strinterp('config',
'{WORKhwrf}/stage-archive.tar.gz')
cmd = exe(conf.getexe('hsi'))['put', frompath, ':', topath]
checkrun(cmd, logger=logger)
postmsg('hwrf_archive tape step completed')
def find_dbn_alert():
"""!Locates the dbn_alert executable based on environment
variables, and returns it as a produtil.prog.Runner object."""
global no_DBNROOT_warn
ENV=os.environ
if ENV.get('DBNROOT','')!='':
return alias(exe(os.path.join(ENV['DBNROOT'],'bin/dbn_alert')))
else:
if not no_DBNROOT_warn:
log.warning("$DBNROOT is not set. Will search for dbn_alert "
"in your $PATH.")
no_DBNROOT_warn=True
return alias(exe('dbn_alert'))
def main_disk():
postmsg('hwrf_archive disk step starting')
conf = hwrf_expt.conf
logger = conf.log('archive.disk')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
archive = conf.getloc('archive', 'NONE')
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hwrf_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
path = archive[5:]
if makethedir:
adir = os.path.dirname(path)
if not os.path.exists(adir):
produtil.fileop.makedirs(adir, logger=logger)
flags = '-cvp'
if path[-3:] == '.gz' or path[-4:] == '.tgz':
flags += 'z'
cmd = exe(conf.getexe('tar'))[flags + 'f', path, '-T',
'-'] << files
elif archive[0:5] == 'hpss:':
logger.info('HPSS archiving enabled.')
logger.info('Nothing to do in the disk archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hwrf_archive disk step does nothing when using htar '
'archives.')
return
elif archive[0:5] == 'hpsz:':
path = conf.strinterp('config', '{WORKhwrf}/stage-archive.tar.gz')
cmd = exe(conf.getexe('tar'))['-cvpf', path, '-T', '-'] << files
else:
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
checkrun(cmd, logger=logger)
postmsg('hwrf_archive disk step completed')
def main():
hwrf_expt.init_module()
conf = hwrf_expt.conf
if not conf.has_option('archive', 'wrfout'):
jlogger.info(
'No wrfout option in [archive] section. Will not make wrfout archive.'
)
sys.exit(0)
logger = conf.log()
files = list()
dt = hwrf.numerics.to_timedelta('6:00:00')
t0 = conf.cycle
wrf = hwrf_expt.runwrf.wrf()
with produtil.cd.NamedDir(hwrf_expt.runwrf.location):
for i in xrange(22):
for dom in wrf:
t = t0 + dt * i
out = dom.get_output('auxhist3', t)
if out is None:
out = dom.get_output('history', t)
if out is None:
out = dom.get_output('auxhist2', t)
if out is None:
logger.error('%s: could not determine wrfout for '
'domain %s' %
(t.strftime('%Y%m%d%H'), str(dom)))
if not os.path.exists(out.path()):
logger.error('%s: does not exist' % (out.path(), ))
if not produtil.fileop.isnonempty(out.path(), ):
logger.error('%s: is empty' % (out.path(), ))
files.append(out.path())
thearchive = conf.timestrinterp('archive', '{wrfout}', 0)
if thearchive[0:5] != 'hpss:':
logger.error('The wrfout archive path must begin with "hpss:": ' +
thearchive)
sys.exit(1)
thearchive = thearchive[5:]
adir = os.path.dirname(thearchive)
mkdir = exe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mkdir, logger=logger)
cmd = exe(conf.getexe('htar'))['-cpf', thearchive][files]
checkrun(cmd, logger=logger)
def main():
import hwrf_expt
hwrf_expt.init_module(make_ensemble_da=True)
conf = hwrf_expt.conf
cycle = hwrf_expt.conf.cycle
input_catalog = conf.get('config', 'input_catalog')
input_sources = conf.get('config', 'input_sources')
logger = conf.log('exhwrf_input')
WORKhwrf = conf.getdir('WORKhwrf')
if input_catalog != 'hwrfdata':
jlogger.info("Input catalog is %s, not \"hwrfdata\" so data should "
"be staged on disk already. I have nothing to do, so "
"I'll just exit. This is not an error.")
sys.exit(0)
# Make sure we're in the cycle's work directory, otherwise we might
# pull archives and other big things to $HOME.
produtil.fileop.chdir(WORKhwrf, logger=logger)
# Figure out how to run htar:
htar = exe(conf.getexe('htar'))
# Figure out how to run hsi:
hsi = exe(conf.getexe('hsi'))
# Get the list of data to pull:
data = list(d for d in hwrf_expt.inputiter())
# Decide where to put the data:
cat = hwrf.input.DataCatalog(conf, "hwrfdata", cycle)
# Now pull the data:
getem = hwrf.input.InputSource(conf,
input_sources,
conf.cycle,
htar=htar,
hsi=hsi,
logger=logger)
bad = not getem.get(data, cat)
if bad:
jlogger.error('Missing data in exhwrf_input. Workflow may fail.')
sys.exit(1)
def __call__(self,**kwargs):
"""!Expands strings specified in the constructor and calls
dbn_alert with the results. If dbn alerts are disabled, then
the fact that a dbn alert would be run is logged, but
dbn_alert is NOT called.
@param kwargs string formatting variables for the dbn alert arguments"""
assert(job is not None)
kwargs['job']=str(job)
alert_args=[ s.format(**kwargs) for s in self.alert_args ]
if send_dbn_alerts:
if isinstance(self.alert_exe,basestring):
cmd=exe(self.alert_exe)[alert_args]
else:
cmd=self.alert_exe[alert_args]
log.log(self.loglevel,'DBN Alert: '+repr(cmd))
ret=run(cmd)
log.log(self.loglevel,'Exit status %s from dbn_alert.'%(repr(ret), ))
else:
log.log(self.loglevel,'dbn_alert is disabled')
log.log(self.loglevel,'would run: dbn_alert '+( " ".join(alert_args) ))
def run_all_times(self):
"""! Builds the command for invoking tcmpr.R plot script.
Args:
Returns:
"""
self.logger.debug("TCMPR input " + self.input_data)
self.logger.debug("TCMPR config file " + self.plot_config_file)
self.logger.debug("output " + self.output_base_dir)
# Create a dictionary of all the "optional" options and flags.
cmds_dict = self.retrieve_optionals()
# Create the TCMPR output base directory, where the final plots
# will be saved.
util.mkdir_p(self.output_base_dir)
# If input data is a file, create a single command and invoke R script.
if os.path.isfile(self.input_data):
self.logger.debug("Currently plotting " + self.input_data)
cmds_dict[' -lookin '] = self.input_data
# Special treatment of the "optional" output_base_dir option
# because we are supporting the plotting of multiple tcst files
# in a directory.
if self.output_base_dir:
# dated_output_dir = self.create_output_subdir(self.input_data)
cmds_dict[' -outdir '] = self.output_base_dir
# Generate the list, where the -args are separated by their
# values.
full_cmd_list = ['Rscript' + self.tcmpr_script]
for key, value in cmds_dict.items():
full_cmd_list.append(key)
full_cmd_list.append(value)
# Separate the 'Rscript' portion from the args, to conform to
# produtil's exe syntax.
cmd = exe(full_cmd_list[0])[full_cmd_list[1:]] > '/dev/null'
self.logger.debug("Command run " + cmd.to_shell())
self.logger.info("Generating requested plots for " +
self.input_data)
# pylint:disable=unnecessary-pass
# If a tc file is empty, continue to the next, thus the pass
# isn't unnecessary.
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
self.logger.warn("plot_tcmpr.R returned non-zero"
" exit status, "
"tcst file may be missing data, "
"continuing: " + repr(ese))
# If the input data is a directory, create a list of all the
# files in the directory and invoke the R script for this list
# of files.
elif os.path.isdir(self.input_data):
self.logger.debug("plot all files in directory " + self.input_data)
cmds_dict = self.retrieve_optionals()
all_tcst_files_list = util.get_files(self.input_data, ".*.tcst",
self.logger)
all_tcst_files = ' '.join(all_tcst_files_list)
self.logger.debug("num of files " + str(len(all_tcst_files)))
# Append the mandatory -lookin option to the base command.
cmds_dict['-lookin'] = all_tcst_files
if self.output_base_dir:
cmds_dict['-outdir'] = self.output_base_dir
self.logger.debug("Creating dated output dir " +
self.output_base_dir)
# Create the full_cmd_list from the keys and values of the
# cmds_dict and then form one command list.
full_cmd_list = list()
full_cmd_list.append("Rscript")
full_cmd_list.append(self.tcmpr_script)
for key, value in cmds_dict.items():
full_cmd_list.append(key)
if key == '-lookin':
# treat the list of dirs in -lookin differently,
# append each individual directory to replicate original
# implementation's behavior of splitting the commands
# by whitespace and assigning each command to an item
# in a list.
for tcst_file in all_tcst_files_list:
full_cmd_list.append(tcst_file)
elif key == '-plot':
# plot types list is also appended as a single string,
# delimited by ','.
full_cmd_list.append(','.join(value))
elif key == '-dep':
# dependant variables list items are appended
# as one string. Convert list into a string delimited
# by ','.
full_cmd_list.append(','.join(value))
else:
full_cmd_list.append(value)
# Separate the 'Rscript' portion from the args, to conform to
# produtil's exe syntax.
cmd = exe(full_cmd_list[0])[full_cmd_list[1:]] > '/dev/null'
# This can be a very long command if the user has
# indicated a directory. Only log this if necessary.
# self.logger.debug("DEBUG: Command run " + cmd.to_shell())
# cmd_str = ' '.join(full_cmd_list)
# cmd_list = 'Rscript ' + cmd_str
# self.logger.debug('TCMPR Command run: ' + cmd_str)
# Now run the command via produtil
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
# If the tcst file is empty (with the exception of the
# header), or there is some other problem, then
# plot_tcmpr.R will return with a non-zero exit status of 1
self.logger.error("plot_tcmpr.R returned non-zero"
" exit status, tcst file may be missing"
" data... continuing: " + str(ese))
sys.exit(1)
else:
self.logger.error("Expected input is neither a file nor directory,"
"exiting...")
sys.exit(1)
self.logger.info("Plotting complete")
def run_cmd(self,
cmd,
env=None,
log_name=None,
copyable_env=None,
**kwargs):
"""!The command cmd is a string which is converted to a produtil
exe Runner object and than run. Output of the command may also
be redirected to either METplus log, MET log, or TTY.
Some subclasses of CommandBuilder ie. series_by_init_wrapper, run
non MET commands ie. convert, in addition to MET binary commands,
ie. regrid_data_plane.
Args:
@param cmd: A string, Command used in the produtil exe Runner object.
@param env: Default None, environment for run to pass in, uses
os.environ if not set.
@param log_name: Used only when ismetcmd=True, The name of the exectable
being run.
@param kwargs Other options sent to the produtil Run constructor
"""
if cmd is None:
return cmd
# if env not set, use os.environ
if env is None:
env = os.environ
self.logger.info("COMMAND: %s" % cmd)
# don't run app if DO_NOT_RUN_EXE is set to True
if self.skip_run:
self.logger.info("Not running command (DO_NOT_RUN_EXE = True)")
return 0, cmd
# self.log_name MUST be defined in the subclass' constructor,
# this code block is a safety net in case that was not done.
# self.log_name is used to generate the MET log output name,
# if output is directed there, based on the conf settings.
#
# cmd.split()[0] 'should' be the /path/to/<some_met_binary>
if not log_name:
log_name = os.path.basename(cmd.split()[0])
self.logger.warning('MISSING self.log_name, '
'setting name to: %s' % repr(log_name))
self.logger.warning('Fix the code and edit the following objects'
' contructor: %s, ' % repr(self))
# Determine where to send the output from the MET command.
log_dest = self.cmdlog_destination(cmdlog=log_name + '.log')
# determine if command must be run in a shell
run_inshell = False
if '*' in cmd or ';' in cmd or '<' in cmd or '>' in cmd:
run_inshell = True
# KEEP This comment as a reference note.
# Run the executable in a new process instead of through a shell.
# FYI. We were originally running the command through a shell
# which also works just fine. IF we go back to running through
# a shell, The string ,cmd, is formatted exactly as is needed.
# By formatted, it is as it would be when typed at the shell prompt.
# This includes, for example, quoting or backslash escaping filenames
# with spaces in them.
# Run the executable and pass the arguments as a sequence.
# Split the command in to a sequence using shell syntax.
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_dest:
self.logger.debug("log_name is: %s, output sent to: %s" %
(log_name, log_dest))
self.log_header_info(log_dest, copyable_env, cmd)
if run_inshell:
cmd_exe = exe('sh')['-c', cmd].env(**env).err2out() >> log_dest
else:
cmd_exe = exe(the_exe)[the_args].env(
**env).err2out() >> log_dest
else:
if run_inshell:
cmd_exe = exe('sh')['-c', cmd].env(**env)
else:
cmd_exe = exe(the_exe)[the_args].env(**env).err2out()
# get current time to calculate total time to run command
start_cmd_time = datetime.now()
# run command
try:
ret = run(cmd_exe, **kwargs)
except:
ret = -1
else:
# calculate time to run
end_cmd_time = datetime.now()
total_cmd_time = end_cmd_time - start_cmd_time
self.logger.debug(f'Finished running {the_exe} '
f'in {total_cmd_time}')
return ret, cmd
import produtil.setup
import produtil.run
from produtil.run import run, checkrun, exe, mpirun, mpi, alias, \
ExitStatusException, runstr
# Initialize produtil. The arguments I send are not necessary, but
# they do make the output more elegant.
produtil.setup.setup(
send_dbn=False, # eliminate two "I can't find dbn_alert" warnings
jobname="example") # Set the name of the job
logger = logging.getLogger('my/logging/domain')
logger.info('hello world examples')
logger.info('single argument to echo')
run(exe("echo")["hello world"]) # prints => hello world
logger.info('two arguments to echo')
run(exe("echo")["hello", "world"]) # prints => hello world
logger.info('generate command step-by-step')
cmd = exe("echo")
for s in ["hello", "world"]:
cmd = cmd[s]
run(cmd) # prints => hello world
logger.info('pass environment variables to sh')
cmd = exe("sh")['-c', '$COMMAND $ENVAR1 $ENVAR2'].env(COMMAND='echo',
ENVAR1='hello',
ENVAR2='world')
run(cmd) # runs sh to print => hello world
def run_cmd(self,
cmd,
env=None,
ismetcmd=True,
app_name=None,
run_inshell=False,
log_theoutput=False,
copyable_env=None,
**kwargs):
"""!The command cmd is a string which is converted to a produtil
exe Runner object and than run. Output of the command may also
be redirected to either METplus log, MET log, or TTY.
Some subclasses of CommandBuilder ie. series_by_init_wrapper, run
non MET commands ie. convert, in addition to MET binary commands,
ie. regrid_data_plane.
Args:
@param cmd: A string, Command used in the produtil exe Runner object.
@param env: Default None, environment for run to pass in, uses
os.environ if not set.
@param ismetcmd: Default True, Will direct output to METplus log,
Metlog , or TTY. Set to False and use the other keywords as needed.
@param app_name: Used only when ismetcmd=True, The name of the exectable
being run.
@param run_inshell: Used only when ismetcmd=False, will Create a
runner object with the cmd being run through a shell, exe('sh')['-c', cmd]
This is required by commands, such as ncdump that are redirecting
output to a file, and other commands such as the convert command
when creating animated gifs.
@param log_theoutput: Used only when ismetcmd=False, will redirect
the stderr and stdout to a the METplus log file or tty.
DO Not set to True if the command is redirecting output to a file.
@param kwargs Other options sent to the produtil Run constructor
"""
if cmd is None:
return cmd
# if env not set, use os.environ
if env is None:
env = os.environ
self.logger.info("COMMAND: %s" % cmd)
if ismetcmd:
# self.app_name MUST be defined in the subclass' constructor,
# this code block is a safety net in case that was not done.
# self.app_name is used to generate the MET log output name,
# if output is directed there, based on the conf settings.
#
# cmd.split()[0] 'should' be the /path/to/<some_met_binary>
if not app_name:
app_name = os.path.basename(cmd.split()[0])
self.logger.warning('MISSING self.app_name, '
'setting name to: %s' % repr(app_name))
self.logger.warning(
'Fix the code and edit the following objects'
' contructor: %s, ' % repr(self))
# Determine where to send the output from the MET command.
log_dest = self.cmdlog_destination(cmdlog=app_name + '.log')
# KEEP This comment as a reference note.
# Run the executable in a new process instead of through a shell.
# FYI. We were originally running the command through a shell
# which also works just fine. IF we go back to running through
# a shell, The string ,cmd, is formatted exactly as is needed.
# By formatted, it is as it would be when typed at the shell prompt.
# This includes, for example, quoting or backslash escaping filenames
# with spaces in them.
# Run the executable and pass the arguments as a sequence.
# Split the command in to a sequence using shell syntax.
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_dest:
self.logger.debug("app_name is: %s, output sent to: %s" %
(app_name, log_dest))
with open(log_dest, 'a+') as log_file_handle:
# if logging MET command to its own log file, add command that was run to that log
if self.log_command_to_met_log:
# if environment variables were set and available, write them to MET tool log
if copyable_env:
log_file_handle.write(
"\nCOPYABLE ENVIRONMENT FOR NEXT COMMAND:\n")
log_file_handle.write(f"{copyable_env}\n\n")
else:
log_file_handle.write('\n')
log_file_handle.write(f"COMMAND:\n{cmd}\n\n")
# write line to designate where MET tool output starts
log_file_handle.write("MET OUTPUT:\n")
cmd_exe = exe(the_exe)[the_args].env(
**env).err2out() >> log_dest
else:
cmd_exe = exe(the_exe)[the_args].env(**env).err2out()
else:
# This block is for all the Non-MET commands
# Some commands still need to be run in a shell in order to work.
#
# There are currently 3 cases of non met commnds that need to be handled.
# case 1. Redirecting to a file in cmd string, which must be through a shell
# ie. cmd = ncdump ...series_F006/min.nc > .../min.txt
# cmd = exe('sh')['-c', cmd]
# case 2. Running the executable directly, w/ arguments, NO redirection
# ie. ncap2, cmd = exe(the_exe)[the_args]
# case 3. Runnng the command and logging the output to
# log_dest
if run_inshell:
# set the_exe to log command has finished running
the_exe = shlex.split(cmd)[0]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd_exe = exe('sh')['-c',
cmd].env(**env).err2out() >> log_dest
else:
cmd_exe = exe('sh')['-c', cmd].env(**env)
else:
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd_exe = exe(the_exe)[the_args].env(
**env).err2out() >> log_dest
else:
cmd_exe = exe(the_exe)[the_args].env(**env)
ret = 0
# run app unless DO_NOT_RUN_EXE is set to True
if not self.config.getbool('config', 'DO_NOT_RUN_EXE', False):
# get current time to calculate total time to run command
start_cmd_time = datetime.now()
# run command
try:
ret = run(cmd_exe, **kwargs)
except:
ret = -1
else:
# calculate time to run
end_cmd_time = datetime.now()
total_cmd_time = end_cmd_time - start_cmd_time
self.logger.debug(
f'Finished running {the_exe} in {total_cmd_time}')
return (ret, cmd)
def run_cmd(self, cmd, ismetcmd = True, app_name=None, run_inshell=False,
log_theoutput=False, **kwargs):
"""!The command cmd is a string which is converted to a produtil
exe Runner object and than run. Output of the command may also
be redirected to either METplus log, MET log, or TTY.
Some subclasses of CommandBuilder ie. series_by_init_wrapper, run
non MET commands ie. convert, in addition to MET binary commands,
ie. regrid_data_plane.
Args:
@param cmd: A string, Command used in the produtil exe Runner object.
@param ismetcmd: Default True, Will direct output to METplus log,
Metlog , or TTY. Set to False and use the other keywords as needed.
@param app_name: Used only when ismetcmd=True, The name of the exectable
being run.
@param run_inshell: Used only when ismetcmd=False, will Create a
runner object with the cmd being run through a shell, exe('sh')['-c', cmd]
This is required by commands, such as ncdump that are redirecting
output to a file, and other commands such as the convert command
when creating animated gifs.
@param log_theoutput: Used only when ismetcmd=False, will redirect
the stderr and stdout to a the METplus log file or tty.
DO Not set to True if the command is redirecting output to a file.
@param kwargs Other options sent to the produtil Run constructor
"""
if cmd is None:
return cmd
if ismetcmd:
# self.app_name MUST be defined in the subclass' constructor,
# this code block is a safety net in case that was not done.
# self.app_name is used to generate the MET log output name,
# if output is directed there, based on the conf settings.
#
# cmd.split()[0] 'should' be the /path/to/<some_met_binary>
if not app_name:
app_name = os.path.basename(cmd.split()[0])
self.logger.warning('MISSING self.app_name, '
'setting name to: %s' % repr(app_name))
self.logger.warning('Fix the code and edit the following objects'
' contructor: %s, ' % repr(self))
# Determine where to send the output from the MET command.
log_dest = self.cmdlog_destination(cmdlog=app_name+'.log')
# KEEP This comment as a reference note.
# Run the executable in a new process instead of through a shell.
# FYI. We were originally running the command through a shell
# which also works just fine. IF we go back to running through
# a shell, The string ,cmd, is formatted exactly as is needed.
# By formatted, it is as it would be when typed at the shell prompt.
# This includes, for example, quoting or backslash escaping filenames
# with spaces in them.
# Run the executable and pass the arguments as a sequence.
# Split the command in to a sequence using shell syntax.
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_dest:
self.logger.info("app_name is: %s, output sent to: %s" % (app_name, log_dest))
#cmd = exe('sh')['-c', cmd].err2out() >> log_dest
cmd = exe(the_exe)[the_args].err2out() >> log_dest
else:
#cmd = exe('sh')['-c', cmd].err2out()
cmd = exe(the_exe)[the_args].err2out()
else:
# This block is for all the Non-MET commands
# Some commands still need to be run in a shell in order to work.
#
# There are currently 3 cases of non met commnds that need to be handled.
# case 1. Redirecting to a file in cmd string, which must be through a shell
# ie. cmd = ncdump ...series_F006/min.nc > .../min.txt
# cmd = exe('sh')['-c', cmd]
# case 2. Running the executable directly, w/ arguments, NO redirection
# ie. ncap2, cmd = exe(the_exe)[the_args]
# case 3. Runnng the command and logging the output to
# log_dest
if run_inshell:
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd = exe('sh')['-c', cmd].err2out() >> log_dest
else:
cmd = exe('sh')['-c', cmd]
else:
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd = exe(the_exe)[the_args].err2out() >> log_dest
else:
cmd = exe(the_exe)[the_args]
self.logger.info("RUNNING: %s" % cmd.to_shell())
self.logger.debug("RUNNING %s" % repr(cmd))
ret = run(cmd, **kwargs)
return (ret, cmd)
clustername = produtil.cluster.name()
if clustername in ('tide', 'gyre'):
WHERE_AM_I = 'wcoss'
elif clustername in ('luna', 'surge'):
WHERE_AM_I = 'wcoss_cray'
elif clustername in ('mars', 'venus'):
WHERE_AM_I = 'wcoss_dell_p3'
else:
WHERE_AM_I = clustername
# '--login', '-c', '. %s/machine-setup.sh ; which ruby ; which rocotorun ; rocotorun --verbose=5 -d %s -w %s'
# %( shbackslash(USHhafs), shbackslash(outdb),
cmd = exe('sh') [
'--login', '-c', '. %s/hafs_pre_job.sh.inc; which ruby ; which rocotorun ; rocotorun --verbose=5 -d %s -w %s'
%( shbackslash(USHhafs), shbackslash(outdb),
shbackslash(outxml) ) ] .env(QUIET_PRE_JOB='YES',
HOMEhafs=HOMEhafs,
WHERE_AM_I=WHERE_AM_I) \
< '/dev/null'
result = run(cmd, logger=logger)
if result:
sys.exit(result)
produtil.jlogger.critical('rocotorun failed')
produtil.log.postmsg('Successfully ran rocotorun for %s.' % (outbase, ))
bakdb = outdb + '.bak'
logger.info('Making a backup copy of .db file here: %s' % (bakdb, ))
produtil.fileop.deliver_file(outdb, bakdb)
logger.info('Success. Rejoice: hurrah!')
##@endcond
with open(outxml, 'wt') as outf:
outf.write(rocotoxml.getvalue())
########################################################################
# Run rocotorun
clustername = produtil.cluster.name()
if clustername in ('tide', 'gyre'):
WHERE_AM_I = 'wcoss2'
else:
WHERE_AM_I = clustername
cmd = exe('sh') [
'--login', '-c', '. %s/hwrf_pre_job.sh.inc ; rocotorun --verbose=5 -d %s -w %s'
%( shbackslash(USHhwrf), shbackslash(outdb),
shbackslash(outxml) ) ] .env(QUIET_PRE_JOB='YES',
HOMEhwrf=HOMEhwrf,
WHERE_AM_I=WHERE_AM_I) \
< '/dev/null'
result = run(cmd, logger=logger)
if result:
sys.exit(result)
produtil.jlogger.critical('rocotorun failed')
produtil.log.postmsg('Successfully ran rocotorun for %s.' % (outbase, ))
bakdb = outdb + '.bak'
logger.info('Making a backup copy of .db file here: %s' % (bakdb, ))
produtil.fileop.deliver_file(outdb, bakdb)
logger.info('Success. Rejoice: hurrah!')