def main():
environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs')
conf = hafs.launcher.HAFSLauncher().read(environ_CONFhafs)
if not conf.getstr('archive', 'fv3out', ''):
jlogger.info(
'No fv3out option in [archive] section. Will not make fv3out archive.'
)
sys.exit(0)
logger = conf.log()
files = list()
WORKhafs = conf.getdir('WORKhafs', '/can/not/find/WORKhafs/dir')
forecastdir = os.path.join(WORKhafs, 'forecast')
with produtil.cd.NamedDir(forecastdir):
files.append(glob.glob('*.nc'))
thearchive = conf.timestrinterp('archive', '{fv3out}', 0)
if thearchive[0:5] != 'hpss:':
logger.error('The fv3out archive path must begin with "hpss:": ' +
thearchive)
sys.exit(1)
thearchive = thearchive[5:]
adir = os.path.dirname(thearchive)
mkdir = batchexe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mkdir, logger=logger)
cmd = batchexe(conf.getexe('htar'))['-vcpf', thearchive][files]
checkrun(cmd, logger=logger)
def launchself():
# Instruct further processes not to re-launch scripts via mpirun:
os.environ['LAUNCH_SELF'] = 'no'
# Launch four copies of myself. We must use mpiserial for this
# because we need the SCR_COMM_RANK variable:
checkrun(
mpirun(mpi(hwrf_expt.conf.getexe(
'mpiserial', 'mpiserial'))[os.path.realpath(__file__)],
allranks=True))
def main_tape():
postmsg('hwrf_archive tape step starting')
conf = hwrf_expt.conf
logger = conf.log('archive.disk')
archive = conf.getloc('archive', 'NONE')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hwrf_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
logger.info('Disk archiving enabled.')
logger.info('Nothing to do in the HPSS archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hwrf_archive tape step does nothing when using disk '
'archives.')
return
elif archive[0:5] != 'hpss:' and archive[0:5] != 'hpsz:':
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
if makethedir:
adir = os.path.dirname(archive[5:])
logger.info('%s: make this HPSS directory, even if it exists' %
(adir, ))
mcmd = exe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mcmd, logger=logger)
if archive[0:5] == 'hpss:':
path = archive[5:]
flags = '-cvp'
cmd = exe(conf.getexe('htar'))[flags + 'f', path, '-L',
'-'] << files
elif archive[0:5] == 'hpsz:':
topath = archive[5:]
frompath = conf.strinterp('config',
'{WORKhwrf}/stage-archive.tar.gz')
cmd = exe(conf.getexe('hsi'))['put', frompath, ':', topath]
checkrun(cmd, logger=logger)
postmsg('hwrf_archive tape step completed')
def build_tc_stat(self, series_output_dir, cur_init, tile_dir,
filter_opts):
"""! Create the call to MET tool TC-STAT to subset tc-pairs output
based on the criteria specified in the parameter/config file.
Args:
@param series_output_dir: The output directory where filtered
results are saved.
@param cur_init: The initialization time
@param tile_dir: The input data directory (tc pair data to be
filtered)
@param filter_opts: The list of filter options to apply
Returns:
None: if no error, then invoke MET tool TC-STAT and
subsets tc-pairs data, creating a filter.tcst file.
Raises CalledProcessError
"""
# pylint:disable=protected-access
# Need to call sys.__getframe() to get the filename and method/func
# for logging information.
# Useful for logging
# Logging output: TIME UTC |TYPE (DEBUG, INFO, WARNING, etc.) |
# [File : function]| Message
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
util.mkdir_p(series_output_dir)
filter_filename = "filter_" + cur_init + ".tcst"
filter_name = os.path.join(series_output_dir, cur_init,
filter_filename)
filter_path = os.path.join(series_output_dir, cur_init)
util.mkdir_p(filter_path)
tc_cmd_list = [
self.tc_exe, " -job filter ", " -lookin ", tile_dir,
" -match_points true ", " -init_inc ", cur_init, " -dump_row ",
filter_name, " ", filter_opts
]
tc_cmd_str = ''.join(tc_cmd_list)
# Make call to tc_stat, capturing any stderr and stdout to the MET
# Plus log.
try:
tc_cmd = batchexe('sh')['-c', tc_cmd_str].err2out()
checkrun(tc_cmd)
except produtil.run.ExitStatusException as ese:
msg = ("ERROR| " + cur_filename + ":" + cur_function +
" from calling MET TC-STAT with command:" +
tc_cmd.to_shell())
self.logger.error(msg)
self.logger.error(ese)
def main_disk():
postmsg('hwrf_archive disk step starting')
conf = hwrf_expt.conf
logger = conf.log('archive.disk')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
archive = conf.getloc('archive', 'NONE')
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hwrf_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
path = archive[5:]
if makethedir:
adir = os.path.dirname(path)
if not os.path.exists(adir):
produtil.fileop.makedirs(adir, logger=logger)
flags = '-cvp'
if path[-3:] == '.gz' or path[-4:] == '.tgz':
flags += 'z'
cmd = exe(conf.getexe('tar'))[flags + 'f', path, '-T',
'-'] << files
elif archive[0:5] == 'hpss:':
logger.info('HPSS archiving enabled.')
logger.info('Nothing to do in the disk archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hwrf_archive disk step does nothing when using htar '
'archives.')
return
elif archive[0:5] == 'hpsz:':
path = conf.strinterp('config', '{WORKhwrf}/stage-archive.tar.gz')
cmd = exe(conf.getexe('tar'))['-cvpf', path, '-T', '-'] << files
else:
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
checkrun(cmd, logger=logger)
postmsg('hwrf_archive disk step completed')
def main():
hwrf_expt.init_module()
conf = hwrf_expt.conf
if not conf.has_option('archive', 'wrfout'):
jlogger.info(
'No wrfout option in [archive] section. Will not make wrfout archive.'
)
sys.exit(0)
logger = conf.log()
files = list()
dt = hwrf.numerics.to_timedelta('6:00:00')
t0 = conf.cycle
wrf = hwrf_expt.runwrf.wrf()
with produtil.cd.NamedDir(hwrf_expt.runwrf.location):
for i in xrange(22):
for dom in wrf:
t = t0 + dt * i
out = dom.get_output('auxhist3', t)
if out is None:
out = dom.get_output('history', t)
if out is None:
out = dom.get_output('auxhist2', t)
if out is None:
logger.error('%s: could not determine wrfout for '
'domain %s' %
(t.strftime('%Y%m%d%H'), str(dom)))
if not os.path.exists(out.path()):
logger.error('%s: does not exist' % (out.path(), ))
if not produtil.fileop.isnonempty(out.path(), ):
logger.error('%s: is empty' % (out.path(), ))
files.append(out.path())
thearchive = conf.timestrinterp('archive', '{wrfout}', 0)
if thearchive[0:5] != 'hpss:':
logger.error('The wrfout archive path must begin with "hpss:": ' +
thearchive)
sys.exit(1)
thearchive = thearchive[5:]
adir = os.path.dirname(thearchive)
mkdir = exe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mkdir, logger=logger)
cmd = exe(conf.getexe('htar'))['-cpf', thearchive][files]
checkrun(cmd, logger=logger)
def run_all_times(self):
"""! Builds the command for invoking tcmpr.R plot script.
Args:
Returns:
"""
base_cmds_list = [' Rscript ', self.tcmpr_script, ' -lookin ']
base_cmds = ''.join(base_cmds_list)
self.logger.debug("base_cmds " + base_cmds)
cmds_list = []
self.logger.debug("DEBUG: TCMPR input " + self.input_data)
self.logger.debug("DEBUG: TCMPR config file " + self.plot_config_file)
self.logger.debug("DEBUG: output " + self.output_base_dir)
# Create a list of all the "optional" options and flags.
optionals_list = self.retrieve_optionals()
# Create the output base directory
util.mkdir_p(self.output_base_dir)
# If input data is a file, create a single command and invoke R script.
if os.path.isfile(self.input_data):
self.logger.debug("Currently plotting " + self.input_data)
cmds_list.append(base_cmds)
cmds_list.append(self.input_data)
# Special treatment of the "optional" output_base_dir option
# because we are supporting the plotting of multiple tcst files
# in a directory.
if self.output_base_dir:
# dated_output_dir = self.create_output_subdir(self.input_data)
optionals_list.append(' -outdir ')
# optionals_list.append(dated_output_dir)
optionals_list.append(self.output_base_dir)
optionals = ''.join(optionals_list)
if optionals:
cmds_list.append(optionals)
# Due to the way cmds_list was created, join it all in to
# one string and than split that in to a list, so element [0]
# is 'Rscript', instead of 'Rscript self.tcmpr_script -lookin'
cmds_list = ''.join(cmds_list).split()
# cmd = batchexe('sh')['-c',''.join(cmds_list)] > '/dev/null'
cmd = batchexe(cmds_list[0])[cmds_list[1:]] > '/dev/null'
self.logger.debug("DEBUG: Command run " + cmd.to_shell())
self.logger.info("INFO: Generating requested plots for " +
self.input_data)
# pylint:disable=unnecessary-pass
# If a tc file is empty, continue to the next, thus the pass
# isn't unnecessary.
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
self.logger.warn("WARN: plot_tcmpr.R returned non-zero"
" exit status, "
"tcst file may be missing data, "
"continuing: " + ese)
# Remove the empty directory
if not os.listdir(self.output_base_dir):
os.rmdir(self.output_base_dir)
pass
# If the input data is a directory, create a list of all the
# files in the directory and invoke the R script for this list
# of files.
if os.path.isdir(self.input_data):
self.logger.debug("plot all files in directory " + self.input_data)
cmds_list = []
all_tcst_files_list = util.get_files(self.input_data, ".*.tcst",
self.logger)
all_tcst_files = ' '.join(all_tcst_files_list)
self.logger.debug("num of files " + str(len(all_tcst_files)))
# Append the mandatory -lookin option to the base command.
cmds_list.append(base_cmds)
cmds_list.append(all_tcst_files)
# dated_output_dir = self.create_output_subdir(self.output_plot)
dated_output_dir = self.output_base_dir
if self.output_base_dir:
cmds_list.append(' -outdir ')
util.mkdir_p(self.output_base_dir)
cmds_list.append(self.output_base_dir)
self.logger.debug("DEBUG: Creating dated output dir " +
dated_output_dir)
if optionals_list:
remaining_options = ''.join(optionals_list)
cmds_list.append(remaining_options)
# Due to the way cmds_list was created, join it all in to
# one string and than split that in to a list, so element [0]
# is 'Rscript', instead of 'Rscript self.tcmpr_script -lookin'
cmds_list = ''.join(cmds_list).split()
cmd = batchexe(cmds_list[0])[cmds_list[1:]] > '/dev/null'
self.logger.debug("DEBUG: Command run " + cmd.to_shell())
# pylint:disable=unnecessary-pass
# If a tc file is empty, continue to the next, thus the pass
# isn't unnecessary.
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
# If the tcst file is empty (with the exception of the
# header), or there is some other problem, then
# plot_tcmpr.R will return with a non-zero exit status of 1
self.logger.warn("WARN: plot_tcmpr.R returned non-zero"
" exit status, tcst file may be missing"
" data... continuing: " + str(ese))
# Remove the empty directory
# Remove the empty directory
if not os.listdir(dated_output_dir):
os.rmdir(dated_output_dir)
pass
# Reset empty cmds_list to prepare for next tcst file.
cmds_list = []
self.logger.info("INFO: Plotting complete")
def run_all_times(self):
"""! Builds the command for invoking tcmpr.R plot script.
Args:
Returns:
"""
self.logger.debug("TCMPR input " + self.input_data)
self.logger.debug("TCMPR config file " + self.plot_config_file)
self.logger.debug("output " + self.output_base_dir)
# Create a dictionary of all the "optional" options and flags.
cmds_dict = self.retrieve_optionals()
# Create the TCMPR output base directory, where the final plots
# will be saved.
util.mkdir_p(self.output_base_dir)
# If input data is a file, create a single command and invoke R script.
if os.path.isfile(self.input_data):
self.logger.debug("Currently plotting " + self.input_data)
cmds_dict[' -lookin '] = self.input_data
# Special treatment of the "optional" output_base_dir option
# because we are supporting the plotting of multiple tcst files
# in a directory.
if self.output_base_dir:
# dated_output_dir = self.create_output_subdir(self.input_data)
cmds_dict[' -outdir '] = self.output_base_dir
# Generate the list, where the -args are separated by their
# values.
full_cmd_list = ['Rscript' + self.tcmpr_script]
for key, value in cmds_dict.items():
full_cmd_list.append(key)
full_cmd_list.append(value)
# Separate the 'Rscript' portion from the args, to conform to
# produtil's exe syntax.
cmd = exe(full_cmd_list[0])[full_cmd_list[1:]] > '/dev/null'
self.logger.debug("Command run " + cmd.to_shell())
self.logger.info("Generating requested plots for " +
self.input_data)
# pylint:disable=unnecessary-pass
# If a tc file is empty, continue to the next, thus the pass
# isn't unnecessary.
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
self.logger.warn("plot_tcmpr.R returned non-zero"
" exit status, "
"tcst file may be missing data, "
"continuing: " + repr(ese))
# If the input data is a directory, create a list of all the
# files in the directory and invoke the R script for this list
# of files.
elif os.path.isdir(self.input_data):
self.logger.debug("plot all files in directory " + self.input_data)
cmds_dict = self.retrieve_optionals()
all_tcst_files_list = util.get_files(self.input_data, ".*.tcst",
self.logger)
all_tcst_files = ' '.join(all_tcst_files_list)
self.logger.debug("num of files " + str(len(all_tcst_files)))
# Append the mandatory -lookin option to the base command.
cmds_dict['-lookin'] = all_tcst_files
if self.output_base_dir:
cmds_dict['-outdir'] = self.output_base_dir
self.logger.debug("Creating dated output dir " +
self.output_base_dir)
# Create the full_cmd_list from the keys and values of the
# cmds_dict and then form one command list.
full_cmd_list = list()
full_cmd_list.append("Rscript")
full_cmd_list.append(self.tcmpr_script)
for key, value in cmds_dict.items():
full_cmd_list.append(key)
if key == '-lookin':
# treat the list of dirs in -lookin differently,
# append each individual directory to replicate original
# implementation's behavior of splitting the commands
# by whitespace and assigning each command to an item
# in a list.
for tcst_file in all_tcst_files_list:
full_cmd_list.append(tcst_file)
elif key == '-plot':
# plot types list is also appended as a single string,
# delimited by ','.
full_cmd_list.append(','.join(value))
elif key == '-dep':
# dependant variables list items are appended
# as one string. Convert list into a string delimited
# by ','.
full_cmd_list.append(','.join(value))
else:
full_cmd_list.append(value)
# Separate the 'Rscript' portion from the args, to conform to
# produtil's exe syntax.
cmd = exe(full_cmd_list[0])[full_cmd_list[1:]] > '/dev/null'
# This can be a very long command if the user has
# indicated a directory. Only log this if necessary.
# self.logger.debug("DEBUG: Command run " + cmd.to_shell())
# cmd_str = ' '.join(full_cmd_list)
# cmd_list = 'Rscript ' + cmd_str
# self.logger.debug('TCMPR Command run: ' + cmd_str)
# Now run the command via produtil
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
# If the tcst file is empty (with the exception of the
# header), or there is some other problem, then
# plot_tcmpr.R will return with a non-zero exit status of 1
self.logger.error("plot_tcmpr.R returned non-zero"
" exit status, tcst file may be missing"
" data... continuing: " + str(ese))
sys.exit(1)
else:
self.logger.error("Expected input is neither a file nor directory,"
"exiting...")
sys.exit(1)
self.logger.info("Plotting complete")
logger.info('send a file to cat')
with open('hello_world.txt', 'wt') as f:
f.write('hello world\n')
run(exe('cat') < 'hello_world.txt')
logger.info('capture a string and print it')
hello_world = runstr(exe('echo')['hello world'])
logger.info('printed => %s' % (hello_world.strip(), ))
logger.info('true/false exit status examples')
# "checkrun" example. This will raise an exception if the program fails
for progname in ['/bin/true', '/bin/false']:
try:
checkrun(exe(progname))
logger.info('%s: Success! Rejoice! Exit status 0!' % (progname, ))
except ExitStatusException as ese:
returncode = ese.returncode
logger.error('OHNO! OHNO! %s returncode is %d' %
(progname, returncode))
logger.info('logging examples with serial programs')
logger = logging.getLogger('example/domain')
run(exe('echo')['hello world'], logger=logger)
# prints => hello world
# but logs the execution of the program and the exit status
## Now we add logging. Let's run a program that exits with status 3
run(exe("sh")['-c', 'echo will exit 3 ; exit 3'], logger=logger)
def main_disk():
"""!Main program for disk archiving.
Creates an on-disk archiving for one of two cases:
* disk:/path/to/archive.tar.gz --- generates an on-disk *.tar.gz
archive in a long-term storage disk area
* hpsz:/path/to/tape-archive.tar.gz --- generates an on-disk
*.tar.gz archive in a temporary location so it can be copied to
tape in a later step."""
postmsg('hafs_archive disk step starting')
environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs')
#conf=hafs.launcher.HAFSLauncher().read(environ_CONFhafs)
conf = hafs.launcher.load(environ_CONFhafs)
logger = conf.log('archive.disk')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
archive = conf.getloc('archive', 'NONE')
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hafs_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
path = archive[5:]
if makethedir:
adir = os.path.dirname(path)
if not os.path.exists(adir):
produtil.fileop.makedirs(adir, logger=logger)
flags = '-cvp'
if path[-3:] == '.gz' or path[-4:] == '.tgz':
flags += 'z'
cmd = batchexe(conf.getexe('tar'))[flags + 'f', path, '-T',
'-'] << files
elif archive[0:5] == 'hpss:':
logger.info('HPSS archiving enabled.')
logger.info('Nothing to do in the disk archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hafs_archive disk step does nothing when using htar '
'archives.')
return
elif archive[0:5] == 'hpsz:':
path = conf.strinterp('config', '{WORKhafs}/stage-archive.tar.gz')
flags = '-cvp'
if path[-3:] == '.gz' or path[-4:] == '.tgz':
flags += 'z'
cmd = batchexe(conf.getexe('tar'))[flags + 'f', path, '-T',
'-'] << files
else:
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
checkrun(cmd, logger=logger)
donefile = path + '.done'
with open(donefile, 'wt') as f:
f.write('hafs_archive disk step completed\n')
postmsg('hafs_archive disk step completed')
def main_tape():
"""!Main program for tape archiving.
Does one of two things:
* hpss:/path/to/archive.tar --- will use the htar command to
archive COM directory outputs
* hpsz:/path/to/archive.tar.gz --- will copy a tar.gz file from a
temporary area, made by the disk archiving step, to a tape
destination using the "hsi put" command."""
postmsg('hafs_archive tape step starting')
environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs')
#conf=hafs.launcher.HAFSLauncher().read(environ_CONFhafs)
conf = hafs.launcher.load(environ_CONFhafs)
logger = conf.log('archive.tape')
archive = conf.getloc('archive', 'NONE')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hafs_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
logger.info('Disk archiving enabled.')
logger.info('Nothing to do in the HPSS archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hafs_archive tape step does nothing when using disk '
'archives.')
return
elif archive[0:5] != 'hpss:' and archive[0:5] != 'hpsz:':
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
if makethedir:
adir = os.path.dirname(archive[5:])
logger.info('%s: make this HPSS directory, even if it exists' %
(adir, ))
mcmd = batchexe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mcmd, logger=logger)
if archive[0:5] == 'hpss:':
path = archive[5:]
flags = '-cvp'
cmd = batchexe(conf.getexe('htar'))[flags + 'f', path, '-L',
'-'] << files
elif archive[0:5] == 'hpsz:':
topath = archive[5:]
frompath = conf.strinterp('config',
'{WORKhafs}/stage-archive.tar.gz')
cmd = batchexe(conf.getexe('hsi'))['put', frompath, ':', topath]
checkrun(cmd, logger=logger)
postmsg('hafs_archive tape step completed')