def main():
environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs')
conf = hafs.launcher.HAFSLauncher().read(environ_CONFhafs)
if not conf.getstr('archive', 'fv3out', ''):
jlogger.info(
'No fv3out option in [archive] section. Will not make fv3out archive.'
)
sys.exit(0)
logger = conf.log()
files = list()
WORKhafs = conf.getdir('WORKhafs', '/can/not/find/WORKhafs/dir')
forecastdir = os.path.join(WORKhafs, 'forecast')
with produtil.cd.NamedDir(forecastdir):
files.append(glob.glob('*.nc'))
thearchive = conf.timestrinterp('archive', '{fv3out}', 0)
if thearchive[0:5] != 'hpss:':
logger.error('The fv3out archive path must begin with "hpss:": ' +
thearchive)
sys.exit(1)
thearchive = thearchive[5:]
adir = os.path.dirname(thearchive)
mkdir = batchexe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mkdir, logger=logger)
cmd = batchexe(conf.getexe('htar'))['-vcpf', thearchive][files]
checkrun(cmd, logger=logger)
def find_dbn_alert():
"""!Locates the dbn_alert executable based on environment
variables, and returns it as a produtil.prog.Runner object."""
global no_DBNROOT_warn
ENV = os.environ
if ENV.get('DBNROOT', '') != '':
return alias(batchexe(os.path.join(ENV['DBNROOT'], 'bin/dbn_alert')))
else:
if not no_DBNROOT_warn:
log.warning("$DBNROOT is not set. Will search for dbn_alert "
"in your $PATH.")
no_DBNROOT_warn = True
return alias(batchexe('dbn_alert'))
def build_tc_stat(self, series_output_dir, cur_init, tile_dir,
filter_opts):
"""! Create the call to MET tool TC-STAT to subset tc-pairs output
based on the criteria specified in the parameter/config file.
Args:
@param series_output_dir: The output directory where filtered
results are saved.
@param cur_init: The initialization time
@param tile_dir: The input data directory (tc pair data to be
filtered)
@param filter_opts: The list of filter options to apply
Returns:
None: if no error, then invoke MET tool TC-STAT and
subsets tc-pairs data, creating a filter.tcst file.
Raises CalledProcessError
"""
# pylint:disable=protected-access
# Need to call sys.__getframe() to get the filename and method/func
# for logging information.
# Useful for logging
# Logging output: TIME UTC |TYPE (DEBUG, INFO, WARNING, etc.) |
# [File : function]| Message
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
util.mkdir_p(series_output_dir)
filter_filename = "filter_" + cur_init + ".tcst"
filter_name = os.path.join(series_output_dir, cur_init,
filter_filename)
filter_path = os.path.join(series_output_dir, cur_init)
util.mkdir_p(filter_path)
tc_cmd_list = [
self.tc_exe, " -job filter ", " -lookin ", tile_dir,
" -match_points true ", " -init_inc ", cur_init, " -dump_row ",
filter_name, " ", filter_opts
]
tc_cmd_str = ''.join(tc_cmd_list)
# Make call to tc_stat, capturing any stderr and stdout to the MET
# Plus log.
try:
tc_cmd = batchexe('sh')['-c', tc_cmd_str].err2out()
checkrun(tc_cmd)
except produtil.run.ExitStatusException as ese:
msg = ("ERROR| " + cur_filename + ":" + cur_function +
" from calling MET TC-STAT with command:" +
tc_cmd.to_shell())
self.logger.error(msg)
self.logger.error(ese)
def __call__(self, **kwargs):
"""!Expands strings specified in the constructor and calls
dbn_alert with the results. If dbn alerts are disabled, then
the fact that a dbn alert would be run is logged, but
dbn_alert is NOT called.
@param kwargs string formatting variables for the dbn alert arguments"""
assert (job is not None)
kwargs['job'] = str(job)
alert_args = [s.format(**kwargs) for s in self.alert_args]
if send_dbn_alerts:
if isinstance(self.alert_exe, str):
cmd = batchexe(self.alert_exe)[alert_args]
else:
cmd = self.alert_exe[alert_args]
log.log(logging.INFO, 'DBN Alert: ' + repr(cmd))
ret = run(cmd)
exit_log_level = (logging.INFO if ret == 0 else logging.ERROR)
log.log(exit_log_level,
'Exit status %s from dbn_alert.' % (repr(ret), ))
else:
log.log(self.loglevel, 'dbn_alert is disabled')
log.log(self.loglevel,
'would run: dbn_alert ' + (" ".join(alert_args)))
def run_all_times(self):
"""! Builds the command for invoking tcmpr.R plot script.
Args:
Returns:
"""
base_cmds_list = [' Rscript ', self.tcmpr_script, ' -lookin ']
base_cmds = ''.join(base_cmds_list)
self.logger.debug("base_cmds " + base_cmds)
cmds_list = []
self.logger.debug("DEBUG: TCMPR input " + self.input_data)
self.logger.debug("DEBUG: TCMPR config file " + self.plot_config_file)
self.logger.debug("DEBUG: output " + self.output_base_dir)
# Create a list of all the "optional" options and flags.
optionals_list = self.retrieve_optionals()
# Create the output base directory
util.mkdir_p(self.output_base_dir)
# If input data is a file, create a single command and invoke R script.
if os.path.isfile(self.input_data):
self.logger.debug("Currently plotting " + self.input_data)
cmds_list.append(base_cmds)
cmds_list.append(self.input_data)
# Special treatment of the "optional" output_base_dir option
# because we are supporting the plotting of multiple tcst files
# in a directory.
if self.output_base_dir:
# dated_output_dir = self.create_output_subdir(self.input_data)
optionals_list.append(' -outdir ')
# optionals_list.append(dated_output_dir)
optionals_list.append(self.output_base_dir)
optionals = ''.join(optionals_list)
if optionals:
cmds_list.append(optionals)
# Due to the way cmds_list was created, join it all in to
# one string and than split that in to a list, so element [0]
# is 'Rscript', instead of 'Rscript self.tcmpr_script -lookin'
cmds_list = ''.join(cmds_list).split()
# cmd = batchexe('sh')['-c',''.join(cmds_list)] > '/dev/null'
cmd = batchexe(cmds_list[0])[cmds_list[1:]] > '/dev/null'
self.logger.debug("DEBUG: Command run " + cmd.to_shell())
self.logger.info("INFO: Generating requested plots for " +
self.input_data)
# pylint:disable=unnecessary-pass
# If a tc file is empty, continue to the next, thus the pass
# isn't unnecessary.
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
self.logger.warn("WARN: plot_tcmpr.R returned non-zero"
" exit status, "
"tcst file may be missing data, "
"continuing: " + ese)
# Remove the empty directory
if not os.listdir(self.output_base_dir):
os.rmdir(self.output_base_dir)
pass
# If the input data is a directory, create a list of all the
# files in the directory and invoke the R script for this list
# of files.
if os.path.isdir(self.input_data):
self.logger.debug("plot all files in directory " + self.input_data)
cmds_list = []
all_tcst_files_list = util.get_files(self.input_data, ".*.tcst",
self.logger)
all_tcst_files = ' '.join(all_tcst_files_list)
self.logger.debug("num of files " + str(len(all_tcst_files)))
# Append the mandatory -lookin option to the base command.
cmds_list.append(base_cmds)
cmds_list.append(all_tcst_files)
# dated_output_dir = self.create_output_subdir(self.output_plot)
dated_output_dir = self.output_base_dir
if self.output_base_dir:
cmds_list.append(' -outdir ')
util.mkdir_p(self.output_base_dir)
cmds_list.append(self.output_base_dir)
self.logger.debug("DEBUG: Creating dated output dir " +
dated_output_dir)
if optionals_list:
remaining_options = ''.join(optionals_list)
cmds_list.append(remaining_options)
# Due to the way cmds_list was created, join it all in to
# one string and than split that in to a list, so element [0]
# is 'Rscript', instead of 'Rscript self.tcmpr_script -lookin'
cmds_list = ''.join(cmds_list).split()
cmd = batchexe(cmds_list[0])[cmds_list[1:]] > '/dev/null'
self.logger.debug("DEBUG: Command run " + cmd.to_shell())
# pylint:disable=unnecessary-pass
# If a tc file is empty, continue to the next, thus the pass
# isn't unnecessary.
try:
checkrun(cmd)
except produtil.run.ExitStatusException as ese:
# If the tcst file is empty (with the exception of the
# header), or there is some other problem, then
# plot_tcmpr.R will return with a non-zero exit status of 1
self.logger.warn("WARN: plot_tcmpr.R returned non-zero"
" exit status, tcst file may be missing"
" data... continuing: " + str(ese))
# Remove the empty directory
# Remove the empty directory
if not os.listdir(dated_output_dir):
os.rmdir(dated_output_dir)
pass
# Reset empty cmds_list to prepare for next tcst file.
cmds_list = []
self.logger.info("INFO: Plotting complete")
def retrieve_and_regrid(tmp_filename, cur_init, cur_storm, out_dir, logger,
config):
"""! Retrieves the data from the MODEL_DATA_DIR (defined in metplus.conf)
that corresponds to the storms defined in the tmp_filename:
1) create the analysis tile and forecast file names from the
tmp_filename file.
2) perform regridding via MET tool (regrid_data_plane) and store
results (netCDF files) in the out_dir or via
Regridding via regrid_data_plane on the forecast and analysis
files via a latlon string with the following format:
latlon Nx Ny lat_ll lon_ll delta_lat delta_lon
NOTE: these values are defined in the extract_tiles_parm
parameter/config file as NLAT, NLON.
Args:
@param tmp_filename: Filename of the temporary filter file in
the /tmp directory. Contains rows
of data corresponding to a storm id of varying
times.
@param cur_init: The current init time
@param cur_storm: The current storm
@param out_dir: The directory where regridded netCDF or grib2 output
is saved depending on which regridding methodology is
requested. If the MET tool regrid_data_plane is
requested, then netCDF data is produced. If wgrib2
is requested, then grib2 data is produced.
@param logger: The name of the logger used in logging.
@param config: config instance
Returns:
None
"""
# pylint: disable=protected-access
# Need to call sys._getframe() to get current function and file for
# logging information.
# pylint: disable=too-many-arguments
# all input is needed to perform task
# For logging
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
# Get variables, etc. from param/config file.
model_data_dir = config.getdir('MODEL_DATA_DIR')
metplus_base = config.getdir('MET_BUILD_BASE')
regrid_data_plane_exe = os.path.join(metplus_base, 'bin/regrid_data_plane')
# regrid_data_plane_exe = config.getexe('REGRID_DATA_PLANE_EXE')
wgrib2_exe = config.getexe('WGRIB2')
egrep_exe = config.getexe('EGREP_EXE')
regrid_with_met_tool = config.getbool('config', 'REGRID_USING_MET_TOOL')
overwrite_flag = config.getbool('config', 'OVERWRITE_TRACK')
# Extract the columns of interest: init time, lead time,
# valid time lat and lon of both tropical cyclone tracks, etc.
# Then calculate the forecast hour and other things.
with open(tmp_filename, "r") as tf:
# read header
header = tf.readline().split()
# get column number for columns on interest
# print('header{}:'.format(header))
header_colnum_init, header_colnum_lead, header_colnum_valid = \
header.index('INIT'), header.index('LEAD'), header.index('VALID')
header_colnum_alat, header_colnum_alon =\
header.index('ALAT'), header.index('ALON')
header_colnum_blat, header_colnum_blon = \
header.index('BLAT'), header.index('BLON')
for line in tf:
col = line.split()
init, lead, valid, alat, alon, blat, blon = \
col[header_colnum_init], col[header_colnum_lead], \
col[header_colnum_valid], col[header_colnum_alat], \
col[header_colnum_alon], col[header_colnum_blat], \
col[header_colnum_blon]
# integer division for both Python 2 and 3
lead_time = int(lead)
fcst_hr = lead_time // 10000
init_ymd_match = re.match(r'[0-9]{8}', init)
if init_ymd_match:
init_ymd = init_ymd_match.group(0)
else:
logger.WARN("RuntimeError raised")
raise RuntimeError('init time has unexpected format for YMD')
init_ymdh_match = re.match(r'[0-9|_]{11}', init)
if init_ymdh_match:
init_ymdh = init_ymdh_match.group(0)
else:
logger.WARN("RuntimeError raised")
valid_ymd_match = re.match(r'[0-9]{8}', valid)
if valid_ymd_match:
valid_ymd = valid_ymd_match.group(0)
else:
logger.WARN("RuntimeError raised")
valid_ymdh_match = re.match(r'[0-9|_]{11}', valid)
if valid_ymdh_match:
valid_ymdh = valid_ymdh_match.group(0)
else:
logger.WARN("RuntimeError raised")
lead_str = str(fcst_hr).zfill(3)
fcst_dir = os.path.join(model_data_dir, init_ymd)
init_ymdh_split = init_ymdh.split("_")
init_yyyymmddhh = "".join(init_ymdh_split)
anly_dir = os.path.join(model_data_dir, valid_ymd)
valid_ymdh_split = valid_ymdh.split("_")
valid_yyyymmddhh = "".join(valid_ymdh_split)
# Create output filenames for regridding
# wgrib2 used to regrid.
# Create the filename for the regridded file, which is a
# grib2 file.
fcst_sts = \
StringSub(logger, config.getraw('filename_templates',
'GFS_FCST_FILE_TMPL'),
init=init_yyyymmddhh, lead=lead_str)
anly_sts = \
StringSub(logger, config.getraw('filename_templates',
'GFS_ANLY_FILE_TMPL'),
valid=valid_yyyymmddhh, lead=lead_str)
fcst_file = fcst_sts.doStringSub()
fcst_filename = os.path.join(fcst_dir, fcst_file)
anly_file = anly_sts.doStringSub()
anly_filename = os.path.join(anly_dir, anly_file)
# Check if the forecast input file exists. If it doesn't
# exist, just log it
if file_exists(fcst_filename):
msg = ("INFO| [" + cur_filename + ":" + cur_function +
" ] | Forecast file: " + fcst_filename)
logger.debug(msg)
else:
msg = ("WARNING| [" + cur_filename + ":" + cur_function +
" ] | " +
"Can't find forecast file, continuing anyway: " +
fcst_filename)
logger.debug(msg)
continue
# Check if the analysis input file exists. If it doesn't
# exist, just log it.
if file_exists(anly_filename):
msg = ("INFO| [" + cur_filename + ":" + cur_function +
" ] | Analysis file: " + anly_filename)
logger.debug(msg)
else:
msg = ("WARNING| [" + cur_filename + ":" + cur_function +
" ] | " +
"Can't find analysis file, continuing anyway: " +
anly_filename)
logger.debug(msg)
continue
# Create the arguments used to perform regridding.
# NOTE: the base name
# is the same for both the fcst and anly filenames,
# so use either one to derive the base name that will be used to
# create the fcst_regridded_filename and anly_regridded_filename.
fcst_anly_base = os.path.basename(fcst_filename)
fcst_grid_spec = create_grid_specification_string(
alat, alon, logger, config)
anly_grid_spec = create_grid_specification_string(
blat, blon, logger, config)
if regrid_with_met_tool:
nc_fcst_anly_base = re.sub("grb2", "nc", fcst_anly_base)
fcst_anly_base = nc_fcst_anly_base
tile_dir = os.path.join(out_dir, cur_init, cur_storm)
fcst_hr_str = str(fcst_hr).zfill(3)
fcst_regridded_filename = \
config.getstr('regex_pattern', 'FCST_TILE_PREFIX') + \
fcst_hr_str + "_" + fcst_anly_base
fcst_regridded_file = os.path.join(tile_dir,
fcst_regridded_filename)
anly_regridded_filename = \
config.getstr('regex_pattern', 'ANLY_TILE_PREFIX') +\
fcst_hr_str + "_" + fcst_anly_base
anly_regridded_file = os.path.join(tile_dir,
anly_regridded_filename)
# Regrid the fcst file only if a fcst tile
# file does NOT already exist or if the overwrite flag is True.
# Create new gridded file for fcst tile
if file_exists(fcst_regridded_file) and not overwrite_flag:
msg = ("INFO| [" + cur_filename + ":" + cur_function +
" ] | Forecast tile file " + fcst_regridded_file +
" exists, skip regridding")
logger.debug(msg)
else:
# Perform fcst regridding on the records of interest
var_level_string = retrieve_var_info(config, logger)
if regrid_with_met_tool:
# Perform regridding using MET Tool regrid_data_plane
fcst_cmd_list = [
regrid_data_plane_exe, ' ', fcst_filename, ' ',
fcst_grid_spec, ' ', fcst_regridded_file, ' ',
var_level_string, ' -method NEAREST '
]
regrid_cmd_fcst = ''.join(fcst_cmd_list)
regrid_cmd_fcst = \
batchexe('sh')['-c', regrid_cmd_fcst].err2out()
msg = ("INFO|[regrid]| regrid_data_plane regrid command:" +
regrid_cmd_fcst.to_shell())
logger.debug(msg)
run(regrid_cmd_fcst)
else:
# Perform regridding via wgrib2
requested_records = retrieve_var_info(config, logger)
fcst_cmd_list = [
wgrib2_exe, ' ', fcst_filename, ' | ', egrep_exe, ' ',
requested_records, '|', wgrib2_exe, ' -i ',
fcst_filename, ' -new_grid ', fcst_grid_spec, ' ',
fcst_regridded_file
]
wgrb_cmd_fcst = ''.join(fcst_cmd_list)
wgrb_cmd_fcst = \
batchexe('sh')['-c', wgrb_cmd_fcst].err2out()
msg = ("INFO|[wgrib2]| wgrib2 regrid command:" +
wgrb_cmd_fcst.to_shell())
logger.debug(msg)
run(wgrb_cmd_fcst)
# Create new gridded file for anly tile
if file_exists(anly_regridded_file) and not overwrite_flag:
logger.debug("INFO| [" + cur_filename + ":" + cur_function +
" ] |" + " Analysis tile file: " +
anly_regridded_file + " exists, skip regridding")
else:
# Perform anly regridding on the records of interest
var_level_string = retrieve_var_info(config, logger)
if regrid_with_met_tool:
anly_cmd_list = [
regrid_data_plane_exe, ' ', anly_filename, ' ',
anly_grid_spec, ' ', anly_regridded_file, ' ',
var_level_string, ' ', ' -method NEAREST '
]
regrid_cmd_anly = ''.join(anly_cmd_list)
regrid_cmd_anly = \
batchexe('sh')['-c', regrid_cmd_anly].err2out()
run(regrid_cmd_anly)
msg = ("INFO|[regrid]| on anly file:" +
anly_regridded_file)
logger.debug(msg)
else:
# Regridding via wgrib2.
requested_records = retrieve_var_info(config, logger)
anly_cmd_list = [
wgrib2_exe, ' ', anly_filename, ' | ', egrep_exe, ' ',
requested_records, '|', wgrib2_exe, ' -i ',
anly_filename, ' -new_grid ', anly_grid_spec, ' ',
anly_regridded_file
]
wgrb_cmd_anly = ''.join(anly_cmd_list)
wgrb_cmd_anly = \
batchexe('sh')['-c', wgrb_cmd_anly].err2out()
msg = ("INFO|[wgrib2]| Regridding via wgrib2:" +
wgrb_cmd_anly.to_shell())
run(wgrb_cmd_anly)
logger.debug(msg)
def run_all_times(self):
"""! Build up the command to invoke the MET tool tc_pairs.
"""
# pylint:disable=protected-access
# sys._getframe is a legitimate way to access the current
# filename and method.
# Used for logging information
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
missing_values = \
(self.config.getstr('config', 'MISSING_VAL_TO_REPLACE'),
self.config.getstr('config', 'MISSING_VAL'))
# Get the desired YYYYMMDD_HH init increment list
# convert the increment INIT_INC from seconds to hours
init_list = util.gen_init_list(
self.config.getstr('config', 'INIT_BEG'),
self.config.getstr('config', 'INIT_END'),
int(self.config.getint('config', 'INIT_INC') / 3600),
self.config.getstr('config', 'INIT_HOUR_END'))
# get a list of YYYYMM values
year_month_list = []
for init in init_list:
if init[0:6] not in year_month_list:
year_month_list.append(init[0:6])
# Set up the environment variable to be used in the TCPairs Config
# file (TC_PAIRS_CONFIG_FILE)
# Used to set init_inc in "TC_PAIRS_CONFIG_FILE"
# Need to do some pre-processing so that Python will use " and not '
# because currently MET
# doesn't support single-quotes
tmp_init_string = str(init_list)
tmp_init_string = tmp_init_string.replace("\'", "\"")
os.environ['INIT_INC'] = tmp_init_string
# Get a directory path listing of the dated subdirectories
# (YYYYMM format) in the track_data directory
dir_list = []
# Get a list of all the year_month directories in the
# track data directory specified in the config file.
for year_month in os.listdir(self.config.getdir('TRACK_DATA_DIR')):
# if the full directory path isn't an empty directory,
# check if the current year_month is the requested time.
if os.path.isdir(os.path.join(self.config.getdir('TRACK_DATA_DIR'),
year_month)) \
and year_month in year_month_list:
dir_list.append(
os.path.join(self.config.getdir('TRACK_DATA_DIR'),
year_month))
if not dir_list:
self.logger.warning("ERROR | [" + cur_filename + ":" +
cur_function + "] | There are no dated"
"sub-directories (YYYYMM) " +
"with input data as expected in: " +
self.config.getdir('TRACK_DATA_DIR'))
exit(0)
# Get a list of files in the dated subdirectories
for mydir in dir_list:
myfiles = os.listdir(mydir)
# Need to do extra processing if track_type is extra_tropical
# cyclone
if self.config.getstr('config',
'TRACK_TYPE') == "extra_tropical_cyclone":
# Create an atcf output directory for writing the modified
# files
adeck_mod = os.path.join(
self.config.getdir('TRACK_DATA_SUBDIR_MOD'),
os.path.basename(mydir))
bdeck_mod = os.path.join(
self.config.getdir('TRACK_DATA_SUBDIR_MOD'),
os.path.basename(mydir))
produtil.fileop.makedirs(adeck_mod, logger=self.logger)
# Iterate over the files, modifying them and writing new output
# files if necessary ("extra_tropical_cyclone" track type), and
# run tc_pairs
for myfile in myfiles:
# Check to see if the files have the ADeck prefix
if myfile.startswith(
self.config.getstr('config', 'ADECK_FILE_PREFIX')):
# Create the output directory for the pairs, if
# it doesn't already exist
pairs_out_dir = \
os.path.join(self.config.getdir('TC_PAIRS_DIR'),
os.path.basename(mydir))
produtil.fileop.makedirs(pairs_out_dir, logger=self.logger)
# Need to do extra processing if track_type is
# extra_tropical_cyclone
if self.config.getstr('config',
'TRACK_TYPE') == \
"extra_tropical_cyclone":
# Form the adeck and bdeck input filename paths
adeck_in_file_path = os.path.join(mydir, myfile)
bdeck_in_file_path = re.sub(
self.config.getstr('config', 'ADECK_FILE_PREFIX'),
self.config.getstr('config', 'BDECK_FILE_PREFIX'),
adeck_in_file_path)
adeck_file_path = os.path.join(adeck_mod, myfile)
bdeck_file_path = os.path.join(
bdeck_mod,
re.sub(
self.config.getstr('config',
'ADECK_FILE_PREFIX'),
self.config.getstr('config',
'BDECK_FILE_PREFIX'),
myfile))
# Get the storm number e.g. 0004 in
# amlq2012033118.gfso.0004
# split_basename = myfile.split(".")
# Get the YYYYMM e.g 201203 in amlq2012033118.gfso.0004
year_month = myfile[4:10]
# Get the MM from the YYYYMM
storm_month = year_month[-2:]
# Set up the adeck and bdeck track file paths for the
# extra tropical cyclone data.
self.setup_tropical_track_dirs(adeck_in_file_path,
adeck_file_path,
storm_month,
missing_values)
# Read in the bdeck file, modify it,
# and write a new bdeck file
# Check for existence of data and overwrite if desired
self.setup_tropical_track_dirs(bdeck_in_file_path,
bdeck_file_path,
storm_month,
missing_values)
else:
# Set up the adeck and bdeck file paths
adeck_file_path = os.path.join(mydir, myfile)
bdeck_file_path = re.sub(
self.config.getstr('config', 'ADECK_FILE_PREFIX'),
self.config.getstr('config', 'BDECK_FILE_PREFIX'),
adeck_file_path)
# Run tc_pairs
cmd = self.build_tc_pairs(pairs_out_dir, myfile,
adeck_file_path, bdeck_file_path)
cmd = batchexe('sh')['-c', cmd].err2out()
ret = run(cmd, sleeptime=.00001)
if ret != 0:
self.logger.error("ERROR | [" + cur_filename + ":" +
cur_function + "] | " +
"Problem executing: " +
cmd.to_shell())
exit(0)
def main_disk():
"""!Main program for disk archiving.
Creates an on-disk archiving for one of two cases:
* disk:/path/to/archive.tar.gz --- generates an on-disk *.tar.gz
archive in a long-term storage disk area
* hpsz:/path/to/tape-archive.tar.gz --- generates an on-disk
*.tar.gz archive in a temporary location so it can be copied to
tape in a later step."""
postmsg('hafs_archive disk step starting')
environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs')
#conf=hafs.launcher.HAFSLauncher().read(environ_CONFhafs)
conf = hafs.launcher.load(environ_CONFhafs)
logger = conf.log('archive.disk')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
archive = conf.getloc('archive', 'NONE')
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hafs_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
path = archive[5:]
if makethedir:
adir = os.path.dirname(path)
if not os.path.exists(adir):
produtil.fileop.makedirs(adir, logger=logger)
flags = '-cvp'
if path[-3:] == '.gz' or path[-4:] == '.tgz':
flags += 'z'
cmd = batchexe(conf.getexe('tar'))[flags + 'f', path, '-T',
'-'] << files
elif archive[0:5] == 'hpss:':
logger.info('HPSS archiving enabled.')
logger.info('Nothing to do in the disk archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hafs_archive disk step does nothing when using htar '
'archives.')
return
elif archive[0:5] == 'hpsz:':
path = conf.strinterp('config', '{WORKhafs}/stage-archive.tar.gz')
flags = '-cvp'
if path[-3:] == '.gz' or path[-4:] == '.tgz':
flags += 'z'
cmd = batchexe(conf.getexe('tar'))[flags + 'f', path, '-T',
'-'] << files
else:
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
checkrun(cmd, logger=logger)
donefile = path + '.done'
with open(donefile, 'wt') as f:
f.write('hafs_archive disk step completed\n')
postmsg('hafs_archive disk step completed')
def main_tape():
"""!Main program for tape archiving.
Does one of two things:
* hpss:/path/to/archive.tar --- will use the htar command to
archive COM directory outputs
* hpsz:/path/to/archive.tar.gz --- will copy a tar.gz file from a
temporary area, made by the disk archiving step, to a tape
destination using the "hsi put" command."""
postmsg('hafs_archive tape step starting')
environ_CONFhafs = os.environ.get('CONFhafs', 'NO_CONFhafs')
#conf=hafs.launcher.HAFSLauncher().read(environ_CONFhafs)
conf = hafs.launcher.load(environ_CONFhafs)
logger = conf.log('archive.tape')
archive = conf.getloc('archive', 'NONE')
if conf.has_section('archive'):
makethedir = conf.getbool('archive', 'mkdir', False)
else:
makethedir = False
if archive == 'NONE':
logger.info('No archive location specified. Exiting.')
postmsg('hafs_archive disk step has nothing to do when archiving is '
'disabled.')
return
with NamedDir(conf.getdir('com')):
flist = [filename + '\n' for filename in glob.glob('*')]
flist.sort()
files = ''.join(flist)
assert (len(files) > 0)
if archive.lower() == 'none':
postmsg('Archiving is disabled: archive=none')
return
elif archive[0:5] == 'disk:':
logger.info('Disk archiving enabled.')
logger.info('Nothing to do in the HPSS archiving step.')
logger.info('Returning successfully after doing nothing.')
postmsg('hafs_archive tape step does nothing when using disk '
'archives.')
return
elif archive[0:5] != 'hpss:' and archive[0:5] != 'hpsz:':
jlogger.error('Ignoring invalid archive method %s in %s' %
(archive[0:4], archive))
return
if makethedir:
adir = os.path.dirname(archive[5:])
logger.info('%s: make this HPSS directory, even if it exists' %
(adir, ))
mcmd = batchexe(conf.getexe('hsi'))['-P', 'mkdir', '-p', adir]
run(mcmd, logger=logger)
if archive[0:5] == 'hpss:':
path = archive[5:]
flags = '-cvp'
cmd = batchexe(conf.getexe('htar'))[flags + 'f', path, '-L',
'-'] << files
elif archive[0:5] == 'hpsz:':
topath = archive[5:]
frompath = conf.strinterp('config',
'{WORKhafs}/stage-archive.tar.gz')
cmd = batchexe(conf.getexe('hsi'))['put', frompath, ':', topath]
checkrun(cmd, logger=logger)
postmsg('hafs_archive tape step completed')
def generate_plots(self, sorted_filter_init, tile_dir):
"""! Generate the plots from the series_analysis output.
Args:
@param sorted_filter_init: A list of the sorted directories
corresponding to the init times (that
are the result of filtering). If
filtering produced no results, this
is the list of files created from
running extract_tiles.
@param tile_dir: The directory where input data resides.
Returns:
"""
convert_exe = self.p.getexe('CONVERT_EXE')
background_map = self.p.getbool('config', 'BACKGROUND_MAP')
plot_data_plane_exe = os.path.join(self.p.getdir('MET_BUILD_BASE'),
'bin/plot_data_plane')
for cur_var in self.var_list:
name, level = util.get_name_level(cur_var, self.logger)
for cur_init in sorted_filter_init:
storm_list = self.get_storms_for_init(cur_init, tile_dir)
for cur_storm in storm_list:
# create the output directory where the finished
# plots will reside
output_dir = os.path.join(self.series_out_dir, cur_init,
cur_storm)
util.mkdir_p(output_dir)
# Now we need to invoke the MET tool
# plot_data_plane to generate plots that are
# recognized by the MET viewer.
# Get the number of forecast tile files,
# the name of the first and last in the list
# to be used by the -title option.
if tile_dir == self.extract_tiles_dir:
# Since filtering was not requested, or
# the additional filtering doesn't yield results,
# search the series_out_dir
num, beg, end = \
self.get_fcst_file_info(self.series_out_dir,
cur_init, cur_storm)
else:
# Search the series_filtered_out_dir for
# the filtered files.
num, beg, end = self.get_fcst_file_info(
self.series_filtered_out_dir, cur_init, cur_storm)
# Assemble the input file, output file, field string,
# and title
plot_data_plane_input_fname = self.sbi_plotting_out_dir
for cur_stat in self.stat_list:
plot_data_plane_output = [
output_dir, '/series_', name, '_', level, '_',
cur_stat, '.ps'
]
plot_data_plane_output_fname = ''.join(
plot_data_plane_output)
os.environ['CUR_STAT'] = cur_stat
self.add_env_var('CUR_STAT', cur_stat)
# Create versions of the arg based on
# whether the background map is requested
# in param file.
map_data = ' map_data={ source=[];}'
if background_map:
# Flag set to True, draw background map.
field_string_parts = [
"'name=", '"series_cnt_', cur_stat, '";',
'level="', level, '";', "'"
]
else:
field_string_parts = [
"'name=", '"series_cnt_', cur_stat, '";',
'level="', level, '";', map_data, "'"
]
field_string = ''.join(field_string_parts)
title_parts = [
' -title "GFS Init ', cur_init, ' Storm ',
cur_storm, ' ',
str(num), ' Forecasts (',
str(beg), ' to ',
str(end), '),', cur_stat, ' for ', cur_var, '"'
]
title = ''.join(title_parts)
# Now assemble the entire plot data plane command
data_plane_command_parts = \
[plot_data_plane_exe, ' ',
plot_data_plane_input_fname, ' ',
plot_data_plane_output_fname, ' ',
field_string, ' ', title]
data_plane_command = ''.join(data_plane_command_parts)
data_plane_command = \
batchexe('sh')[
'-c', data_plane_command].err2out()
run(data_plane_command)
# Now assemble the command to convert the
# postscript file to png
png_fname = plot_data_plane_output_fname.replace(
'.ps', '.png')
convert_parts = [
convert_exe, ' -rotate 90',
' -background white -flatten ',
plot_data_plane_output_fname, ' ', png_fname
]
convert_command = ''.join(convert_parts)
convert_command = \
batchexe('sh')['-c', convert_command].err2out()
run(convert_command)