def run_all_times(self):
"""! Build up the command to invoke the MET tool tc_pairs.
"""
# pylint:disable=protected-access
# sys._getframe is a legitimate way to access the current
# filename and method.
# Used for logging information
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
missing_values = \
(self.config.getstr('config', 'MISSING_VAL_TO_REPLACE'),
self.config.getstr('config', 'MISSING_VAL'))
# Get the desired YYYYMMDD_HH init increment list
# convert the increment INIT_INC from seconds to hours
init_list = util.gen_init_list(
self.config.getstr('config', 'INIT_BEG'),
self.config.getstr('config', 'INIT_END'),
int(self.config.getint('config', 'INIT_INC') / 3600),
self.config.getstr('config', 'INIT_HOUR_END'))
# get a list of YYYYMM values
year_month_list = []
for init in init_list:
if init[0:6] not in year_month_list:
year_month_list.append(init[0:6])
# Set up the environment variable to be used in the TCPairs Config
# file (TC_PAIRS_CONFIG_FILE)
# Used to set init_inc in "TC_PAIRS_CONFIG_FILE"
# Need to do some pre-processing so that Python will use " and not '
# because currently MET
# doesn't support single-quotes
tmp_init_string = str(init_list)
tmp_init_string = tmp_init_string.replace("\'", "\"")
os.environ['INIT_INC'] = tmp_init_string
# Get a directory path listing of the dated subdirectories
# (YYYYMM format) in the track_data directory
dir_list = []
# Get a list of all the year_month directories in the
# track data directory specified in the config file.
for year_month in os.listdir(self.config.getdir('TRACK_DATA_DIR')):
# if the full directory path isn't an empty directory,
# check if the current year_month is the requested time.
if os.path.isdir(os.path.join(self.config.getdir('TRACK_DATA_DIR'),
year_month)) \
and year_month in year_month_list:
dir_list.append(
os.path.join(self.config.getdir('TRACK_DATA_DIR'),
year_month))
if not dir_list:
self.logger.warning("ERROR | [" + cur_filename + ":" +
cur_function + "] | There are no dated"
"sub-directories (YYYYMM) " +
"with input data as expected in: " +
self.config.getdir('TRACK_DATA_DIR'))
exit(0)
# Get a list of files in the dated subdirectories
for mydir in dir_list:
myfiles = os.listdir(mydir)
# Need to do extra processing if track_type is extra_tropical
# cyclone
if self.config.getstr('config',
'TRACK_TYPE') == "extra_tropical_cyclone":
# Create an atcf output directory for writing the modified
# files
adeck_mod = os.path.join(
self.config.getdir('TRACK_DATA_SUBDIR_MOD'),
os.path.basename(mydir))
bdeck_mod = os.path.join(
self.config.getdir('TRACK_DATA_SUBDIR_MOD'),
os.path.basename(mydir))
produtil.fileop.makedirs(adeck_mod, logger=self.logger)
# Iterate over the files, modifying them and writing new output
# files if necessary ("extra_tropical_cyclone" track type), and
# run tc_pairs
for myfile in myfiles:
# Check to see if the files have the ADeck prefix
if myfile.startswith(
self.config.getstr('config', 'ADECK_FILE_PREFIX')):
# Create the output directory for the pairs, if
# it doesn't already exist
pairs_out_dir = \
os.path.join(self.config.getdir('TC_PAIRS_DIR'),
os.path.basename(mydir))
produtil.fileop.makedirs(pairs_out_dir, logger=self.logger)
# Need to do extra processing if track_type is
# extra_tropical_cyclone
if self.config.getstr('config',
'TRACK_TYPE') == \
"extra_tropical_cyclone":
# Form the adeck and bdeck input filename paths
adeck_in_file_path = os.path.join(mydir, myfile)
bdeck_in_file_path = re.sub(
self.config.getstr('config', 'ADECK_FILE_PREFIX'),
self.config.getstr('config', 'BDECK_FILE_PREFIX'),
adeck_in_file_path)
adeck_file_path = os.path.join(adeck_mod, myfile)
bdeck_file_path = os.path.join(
bdeck_mod,
re.sub(
self.config.getstr('config',
'ADECK_FILE_PREFIX'),
self.config.getstr('config',
'BDECK_FILE_PREFIX'),
myfile))
# Get the storm number e.g. 0004 in
# amlq2012033118.gfso.0004
# split_basename = myfile.split(".")
# Get the YYYYMM e.g 201203 in amlq2012033118.gfso.0004
year_month = myfile[4:10]
# Get the MM from the YYYYMM
storm_month = year_month[-2:]
# Set up the adeck and bdeck track file paths for the
# extra tropical cyclone data.
self.setup_tropical_track_dirs(adeck_in_file_path,
adeck_file_path,
storm_month,
missing_values)
# Read in the bdeck file, modify it,
# and write a new bdeck file
# Check for existence of data and overwrite if desired
self.setup_tropical_track_dirs(bdeck_in_file_path,
bdeck_file_path,
storm_month,
missing_values)
else:
# Set up the adeck and bdeck file paths
adeck_file_path = os.path.join(mydir, myfile)
bdeck_file_path = re.sub(
self.config.getstr('config', 'ADECK_FILE_PREFIX'),
self.config.getstr('config', 'BDECK_FILE_PREFIX'),
adeck_file_path)
# Run tc_pairs
cmd = self.build_tc_pairs(pairs_out_dir, myfile,
adeck_file_path, bdeck_file_path)
cmd = batchexe('sh')['-c', cmd].err2out()
ret = run(cmd, sleeptime=.00001)
if ret != 0:
self.logger.error("ERROR | [" + cur_filename + ":" +
cur_function + "] | " +
"Problem executing: " +
cmd.to_shell())
exit(0)
def retrieve_and_regrid(tmp_filename, cur_init, cur_storm, out_dir, logger,
config):
"""! Retrieves the data from the MODEL_DATA_DIR (defined in metplus.conf)
that corresponds to the storms defined in the tmp_filename:
1) create the analysis tile and forecast file names from the
tmp_filename file.
2) perform regridding via MET tool (regrid_data_plane) and store
results (netCDF files) in the out_dir or via
Regridding via regrid_data_plane on the forecast and analysis
files via a latlon string with the following format:
latlon Nx Ny lat_ll lon_ll delta_lat delta_lon
NOTE: these values are defined in the extract_tiles_parm
parameter/config file as NLAT, NLON.
Args:
@param tmp_filename: Filename of the temporary filter file in
the /tmp directory. Contains rows
of data corresponding to a storm id of varying
times.
@param cur_init: The current init time
@param cur_storm: The current storm
@param out_dir: The directory where regridded netCDF or grib2 output
is saved depending on which regridding methodology is
requested. If the MET tool regrid_data_plane is
requested, then netCDF data is produced. If wgrib2
is requested, then grib2 data is produced.
@param logger: The name of the logger used in logging.
@param config: config instance
Returns:
None
"""
# pylint: disable=protected-access
# Need to call sys._getframe() to get current function and file for
# logging information.
# pylint: disable=too-many-arguments
# all input is needed to perform task
# For logging
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
# Get variables, etc. from param/config file.
model_data_dir = config.getdir('MODEL_DATA_DIR')
metplus_base = config.getdir('MET_BUILD_BASE')
regrid_data_plane_exe = os.path.join(metplus_base, 'bin/regrid_data_plane')
# regrid_data_plane_exe = config.getexe('REGRID_DATA_PLANE_EXE')
wgrib2_exe = config.getexe('WGRIB2')
egrep_exe = config.getexe('EGREP_EXE')
regrid_with_met_tool = config.getbool('config', 'REGRID_USING_MET_TOOL')
overwrite_flag = config.getbool('config', 'OVERWRITE_TRACK')
# Extract the columns of interest: init time, lead time,
# valid time lat and lon of both tropical cyclone tracks, etc.
# Then calculate the forecast hour and other things.
with open(tmp_filename, "r") as tf:
# read header
header = tf.readline().split()
# get column number for columns on interest
# print('header{}:'.format(header))
header_colnum_init, header_colnum_lead, header_colnum_valid = \
header.index('INIT'), header.index('LEAD'), header.index('VALID')
header_colnum_alat, header_colnum_alon =\
header.index('ALAT'), header.index('ALON')
header_colnum_blat, header_colnum_blon = \
header.index('BLAT'), header.index('BLON')
for line in tf:
col = line.split()
init, lead, valid, alat, alon, blat, blon = \
col[header_colnum_init], col[header_colnum_lead], \
col[header_colnum_valid], col[header_colnum_alat], \
col[header_colnum_alon], col[header_colnum_blat], \
col[header_colnum_blon]
# integer division for both Python 2 and 3
lead_time = int(lead)
fcst_hr = lead_time // 10000
init_ymd_match = re.match(r'[0-9]{8}', init)
if init_ymd_match:
init_ymd = init_ymd_match.group(0)
else:
logger.WARN("RuntimeError raised")
raise RuntimeError('init time has unexpected format for YMD')
init_ymdh_match = re.match(r'[0-9|_]{11}', init)
if init_ymdh_match:
init_ymdh = init_ymdh_match.group(0)
else:
logger.WARN("RuntimeError raised")
valid_ymd_match = re.match(r'[0-9]{8}', valid)
if valid_ymd_match:
valid_ymd = valid_ymd_match.group(0)
else:
logger.WARN("RuntimeError raised")
valid_ymdh_match = re.match(r'[0-9|_]{11}', valid)
if valid_ymdh_match:
valid_ymdh = valid_ymdh_match.group(0)
else:
logger.WARN("RuntimeError raised")
lead_str = str(fcst_hr).zfill(3)
fcst_dir = os.path.join(model_data_dir, init_ymd)
init_ymdh_split = init_ymdh.split("_")
init_yyyymmddhh = "".join(init_ymdh_split)
anly_dir = os.path.join(model_data_dir, valid_ymd)
valid_ymdh_split = valid_ymdh.split("_")
valid_yyyymmddhh = "".join(valid_ymdh_split)
# Create output filenames for regridding
# wgrib2 used to regrid.
# Create the filename for the regridded file, which is a
# grib2 file.
fcst_sts = \
StringSub(logger, config.getraw('filename_templates',
'GFS_FCST_FILE_TMPL'),
init=init_yyyymmddhh, lead=lead_str)
anly_sts = \
StringSub(logger, config.getraw('filename_templates',
'GFS_ANLY_FILE_TMPL'),
valid=valid_yyyymmddhh, lead=lead_str)
fcst_file = fcst_sts.doStringSub()
fcst_filename = os.path.join(fcst_dir, fcst_file)
anly_file = anly_sts.doStringSub()
anly_filename = os.path.join(anly_dir, anly_file)
# Check if the forecast input file exists. If it doesn't
# exist, just log it
if file_exists(fcst_filename):
msg = ("INFO| [" + cur_filename + ":" + cur_function +
" ] | Forecast file: " + fcst_filename)
logger.debug(msg)
else:
msg = ("WARNING| [" + cur_filename + ":" + cur_function +
" ] | " +
"Can't find forecast file, continuing anyway: " +
fcst_filename)
logger.debug(msg)
continue
# Check if the analysis input file exists. If it doesn't
# exist, just log it.
if file_exists(anly_filename):
msg = ("INFO| [" + cur_filename + ":" + cur_function +
" ] | Analysis file: " + anly_filename)
logger.debug(msg)
else:
msg = ("WARNING| [" + cur_filename + ":" + cur_function +
" ] | " +
"Can't find analysis file, continuing anyway: " +
anly_filename)
logger.debug(msg)
continue
# Create the arguments used to perform regridding.
# NOTE: the base name
# is the same for both the fcst and anly filenames,
# so use either one to derive the base name that will be used to
# create the fcst_regridded_filename and anly_regridded_filename.
fcst_anly_base = os.path.basename(fcst_filename)
fcst_grid_spec = create_grid_specification_string(
alat, alon, logger, config)
anly_grid_spec = create_grid_specification_string(
blat, blon, logger, config)
if regrid_with_met_tool:
nc_fcst_anly_base = re.sub("grb2", "nc", fcst_anly_base)
fcst_anly_base = nc_fcst_anly_base
tile_dir = os.path.join(out_dir, cur_init, cur_storm)
fcst_hr_str = str(fcst_hr).zfill(3)
fcst_regridded_filename = \
config.getstr('regex_pattern', 'FCST_TILE_PREFIX') + \
fcst_hr_str + "_" + fcst_anly_base
fcst_regridded_file = os.path.join(tile_dir,
fcst_regridded_filename)
anly_regridded_filename = \
config.getstr('regex_pattern', 'ANLY_TILE_PREFIX') +\
fcst_hr_str + "_" + fcst_anly_base
anly_regridded_file = os.path.join(tile_dir,
anly_regridded_filename)
# Regrid the fcst file only if a fcst tile
# file does NOT already exist or if the overwrite flag is True.
# Create new gridded file for fcst tile
if file_exists(fcst_regridded_file) and not overwrite_flag:
msg = ("INFO| [" + cur_filename + ":" + cur_function +
" ] | Forecast tile file " + fcst_regridded_file +
" exists, skip regridding")
logger.debug(msg)
else:
# Perform fcst regridding on the records of interest
var_level_string = retrieve_var_info(config, logger)
if regrid_with_met_tool:
# Perform regridding using MET Tool regrid_data_plane
fcst_cmd_list = [
regrid_data_plane_exe, ' ', fcst_filename, ' ',
fcst_grid_spec, ' ', fcst_regridded_file, ' ',
var_level_string, ' -method NEAREST '
]
regrid_cmd_fcst = ''.join(fcst_cmd_list)
regrid_cmd_fcst = \
batchexe('sh')['-c', regrid_cmd_fcst].err2out()
msg = ("INFO|[regrid]| regrid_data_plane regrid command:" +
regrid_cmd_fcst.to_shell())
logger.debug(msg)
run(regrid_cmd_fcst)
else:
# Perform regridding via wgrib2
requested_records = retrieve_var_info(config, logger)
fcst_cmd_list = [
wgrib2_exe, ' ', fcst_filename, ' | ', egrep_exe, ' ',
requested_records, '|', wgrib2_exe, ' -i ',
fcst_filename, ' -new_grid ', fcst_grid_spec, ' ',
fcst_regridded_file
]
wgrb_cmd_fcst = ''.join(fcst_cmd_list)
wgrb_cmd_fcst = \
batchexe('sh')['-c', wgrb_cmd_fcst].err2out()
msg = ("INFO|[wgrib2]| wgrib2 regrid command:" +
wgrb_cmd_fcst.to_shell())
logger.debug(msg)
run(wgrb_cmd_fcst)
# Create new gridded file for anly tile
if file_exists(anly_regridded_file) and not overwrite_flag:
logger.debug("INFO| [" + cur_filename + ":" + cur_function +
" ] |" + " Analysis tile file: " +
anly_regridded_file + " exists, skip regridding")
else:
# Perform anly regridding on the records of interest
var_level_string = retrieve_var_info(config, logger)
if regrid_with_met_tool:
anly_cmd_list = [
regrid_data_plane_exe, ' ', anly_filename, ' ',
anly_grid_spec, ' ', anly_regridded_file, ' ',
var_level_string, ' ', ' -method NEAREST '
]
regrid_cmd_anly = ''.join(anly_cmd_list)
regrid_cmd_anly = \
batchexe('sh')['-c', regrid_cmd_anly].err2out()
run(regrid_cmd_anly)
msg = ("INFO|[regrid]| on anly file:" +
anly_regridded_file)
logger.debug(msg)
else:
# Regridding via wgrib2.
requested_records = retrieve_var_info(config, logger)
anly_cmd_list = [
wgrib2_exe, ' ', anly_filename, ' | ', egrep_exe, ' ',
requested_records, '|', wgrib2_exe, ' -i ',
anly_filename, ' -new_grid ', anly_grid_spec, ' ',
anly_regridded_file
]
wgrb_cmd_anly = ''.join(anly_cmd_list)
wgrb_cmd_anly = \
batchexe('sh')['-c', wgrb_cmd_anly].err2out()
msg = ("INFO|[wgrib2]| Regridding via wgrib2:" +
wgrb_cmd_anly.to_shell())
run(wgrb_cmd_anly)
logger.debug(msg)
def run_cmd(self,
cmd,
env=None,
log_name=None,
copyable_env=None,
**kwargs):
"""!The command cmd is a string which is converted to a produtil
exe Runner object and than run. Output of the command may also
be redirected to either METplus log, MET log, or TTY.
Some subclasses of CommandBuilder ie. series_by_init_wrapper, run
non MET commands ie. convert, in addition to MET binary commands,
ie. regrid_data_plane.
Args:
@param cmd: A string, Command used in the produtil exe Runner object.
@param env: Default None, environment for run to pass in, uses
os.environ if not set.
@param log_name: Used only when ismetcmd=True, The name of the exectable
being run.
@param kwargs Other options sent to the produtil Run constructor
"""
if cmd is None:
return cmd
# if env not set, use os.environ
if env is None:
env = os.environ
self.logger.info("COMMAND: %s" % cmd)
# don't run app if DO_NOT_RUN_EXE is set to True
if self.skip_run:
self.logger.info("Not running command (DO_NOT_RUN_EXE = True)")
return 0, cmd
# self.log_name MUST be defined in the subclass' constructor,
# this code block is a safety net in case that was not done.
# self.log_name is used to generate the MET log output name,
# if output is directed there, based on the conf settings.
#
# cmd.split()[0] 'should' be the /path/to/<some_met_binary>
if not log_name:
log_name = os.path.basename(cmd.split()[0])
self.logger.warning('MISSING self.log_name, '
'setting name to: %s' % repr(log_name))
self.logger.warning('Fix the code and edit the following objects'
' contructor: %s, ' % repr(self))
# Determine where to send the output from the MET command.
log_dest = self.cmdlog_destination(cmdlog=log_name + '.log')
# determine if command must be run in a shell
run_inshell = False
if '*' in cmd or ';' in cmd or '<' in cmd or '>' in cmd:
run_inshell = True
# KEEP This comment as a reference note.
# Run the executable in a new process instead of through a shell.
# FYI. We were originally running the command through a shell
# which also works just fine. IF we go back to running through
# a shell, The string ,cmd, is formatted exactly as is needed.
# By formatted, it is as it would be when typed at the shell prompt.
# This includes, for example, quoting or backslash escaping filenames
# with spaces in them.
# Run the executable and pass the arguments as a sequence.
# Split the command in to a sequence using shell syntax.
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_dest:
self.logger.debug("log_name is: %s, output sent to: %s" %
(log_name, log_dest))
self.log_header_info(log_dest, copyable_env, cmd)
if run_inshell:
cmd_exe = exe('sh')['-c', cmd].env(**env).err2out() >> log_dest
else:
cmd_exe = exe(the_exe)[the_args].env(
**env).err2out() >> log_dest
else:
if run_inshell:
cmd_exe = exe('sh')['-c', cmd].env(**env)
else:
cmd_exe = exe(the_exe)[the_args].env(**env).err2out()
# get current time to calculate total time to run command
start_cmd_time = datetime.now()
# run command
try:
ret = run(cmd_exe, **kwargs)
except:
ret = -1
else:
# calculate time to run
end_cmd_time = datetime.now()
total_cmd_time = end_cmd_time - start_cmd_time
self.logger.debug(f'Finished running {the_exe} '
f'in {total_cmd_time}')
return ret, cmd
def main():
"""!Main program.
Master MET+ script that invokes the necessary Python scripts
to perform various activities, such as series analysis."""
# Job Logger
produtil.log.jlogger.info('Top of master_metplus')
# Setup Task logger, Until Conf object is created, Task logger is
# only logging to tty, not a file.
logger = logging.getLogger('master_metplus')
logger.info('logger Top of master_metplus.')
# Used for logging and usage statment
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
short_opts = "c:r:h"
long_opts = ["config=", "help", "runtime="]
# All command line input, get options and arguments
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, long_opts)
except getopt.GetoptError as err:
print(str(err))
usage('SCRIPT IS EXITING DUE TO UNRECOGNIZED COMMAND LINE OPTION')
for k, v in opts:
if k in ('-c', '--config'):
# adds the conf file to the list of arguments.
print("ADDED CONF FILE: " + v)
args.append(config_launcher.set_conf_file_path(v))
elif k in ('-h', '--help'):
usage()
exit()
elif k in ('-r', '--runtime'):
start_time = v
end_time = v
else:
assert False, "UNHANDLED OPTION"
if not args:
args = None
(parm, infiles,
moreopt) = config_launcher.parse_launch_args(args, usage, None, logger)
p = config_launcher.launch(infiles, moreopt)
# NOW I have a conf object p, I can now setup the handler
# to write to the LOG_FILENAME.
logger = util.get_logger(p)
# This is available in each subprocess from os.system BUT
# we also set it in each process since they may be called stand alone.
os.environ['MET_BASE'] = p.getdir('MET_BASE')
# Use config object to get the list of processes to call
process_list = util.getlist(p.getstr('config', 'PROCESS_LIST'))
# Keep this comment.
# When running commands in the process_list, reprocess the
# original command line using (item))[sys.argv[1:]].
#
# You could call each task (ie. run_tc_pairs.py) without any args since
# the final METPLUS_CONF file was just created from config_metplus.setup,
# and each task, also calls setup, which use an existing final conf
# file over command line args.
#
# both work ...
# Note: Using (item))sys.argv[1:], is preferable since
# it doesn't depend on the conf file existing.
processes = []
for item in process_list:
try:
command_builder = getattr(sys.modules[__name__],
item + "Wrapper")(p, logger)
except AttributeError:
raise NameError("Process %s doesn't exist" % item)
exit()
processes.append(command_builder)
if p.getstr('config', 'LOOP_METHOD') == "processes":
for process in processes:
process.run_all_times()
elif p.getstr('config', 'LOOP_METHOD') == "times":
time_format = p.getstr('config', 'INIT_TIME_FMT')
start_t = p.getstr('config', 'INIT_BEG')
end_t = p.getstr('config', 'INIT_END')
time_interval = p.getint('config', 'INIT_INC')
if time_interval < 60:
print(
"ERROR: time_interval parameter must be greater than 60 seconds"
)
exit(1)
init_time = calendar.timegm(time.strptime(start_t, time_format))
end_time = calendar.timegm(time.strptime(end_t, time_format))
while init_time <= end_time:
run_time = time.strftime("%Y%m%d%H%M", time.gmtime(init_time))
print("")
print("****************************************")
print("* RUNNING MET+")
print("* at init time: " + run_time)
print("****************************************")
logger.info("****************************************")
logger.info("* RUNNING MET+")
logger.info("* at init time: " + run_time)
logger.info("****************************************")
for process in processes:
process.run_at_time(run_time)
process.clear()
init_time += time_interval
else:
print("ERROR: Invalid LOOP_METHOD defined. " + \
"Options are processes, times")
exit()
exit()
for item in process_list:
cmd_shell = cmd.to_shell()
logger.info("INFO | [" + cur_filename + ":" + cur_function + "] | " +
"Running: " + cmd_shell)
ret = run(cmd)
if ret != 0:
logger.error("ERROR | [" + cur_filename + ":" + cur_function +
"] | " + "Problem executing: " + cmd_shell)
exit(0)
def run_cmd(self,
cmd,
env=None,
ismetcmd=True,
app_name=None,
run_inshell=False,
log_theoutput=False,
copyable_env=None,
**kwargs):
"""!The command cmd is a string which is converted to a produtil
exe Runner object and than run. Output of the command may also
be redirected to either METplus log, MET log, or TTY.
Some subclasses of CommandBuilder ie. series_by_init_wrapper, run
non MET commands ie. convert, in addition to MET binary commands,
ie. regrid_data_plane.
Args:
@param cmd: A string, Command used in the produtil exe Runner object.
@param env: Default None, environment for run to pass in, uses
os.environ if not set.
@param ismetcmd: Default True, Will direct output to METplus log,
Metlog , or TTY. Set to False and use the other keywords as needed.
@param app_name: Used only when ismetcmd=True, The name of the exectable
being run.
@param run_inshell: Used only when ismetcmd=False, will Create a
runner object with the cmd being run through a shell, exe('sh')['-c', cmd]
This is required by commands, such as ncdump that are redirecting
output to a file, and other commands such as the convert command
when creating animated gifs.
@param log_theoutput: Used only when ismetcmd=False, will redirect
the stderr and stdout to a the METplus log file or tty.
DO Not set to True if the command is redirecting output to a file.
@param kwargs Other options sent to the produtil Run constructor
"""
if cmd is None:
return cmd
# if env not set, use os.environ
if env is None:
env = os.environ
self.logger.info("COMMAND: %s" % cmd)
if ismetcmd:
# self.app_name MUST be defined in the subclass' constructor,
# this code block is a safety net in case that was not done.
# self.app_name is used to generate the MET log output name,
# if output is directed there, based on the conf settings.
#
# cmd.split()[0] 'should' be the /path/to/<some_met_binary>
if not app_name:
app_name = os.path.basename(cmd.split()[0])
self.logger.warning('MISSING self.app_name, '
'setting name to: %s' % repr(app_name))
self.logger.warning(
'Fix the code and edit the following objects'
' contructor: %s, ' % repr(self))
# Determine where to send the output from the MET command.
log_dest = self.cmdlog_destination(cmdlog=app_name + '.log')
# KEEP This comment as a reference note.
# Run the executable in a new process instead of through a shell.
# FYI. We were originally running the command through a shell
# which also works just fine. IF we go back to running through
# a shell, The string ,cmd, is formatted exactly as is needed.
# By formatted, it is as it would be when typed at the shell prompt.
# This includes, for example, quoting or backslash escaping filenames
# with spaces in them.
# Run the executable and pass the arguments as a sequence.
# Split the command in to a sequence using shell syntax.
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_dest:
self.logger.debug("app_name is: %s, output sent to: %s" %
(app_name, log_dest))
with open(log_dest, 'a+') as log_file_handle:
# if logging MET command to its own log file, add command that was run to that log
if self.log_command_to_met_log:
# if environment variables were set and available, write them to MET tool log
if copyable_env:
log_file_handle.write(
"\nCOPYABLE ENVIRONMENT FOR NEXT COMMAND:\n")
log_file_handle.write(f"{copyable_env}\n\n")
else:
log_file_handle.write('\n')
log_file_handle.write(f"COMMAND:\n{cmd}\n\n")
# write line to designate where MET tool output starts
log_file_handle.write("MET OUTPUT:\n")
cmd_exe = exe(the_exe)[the_args].env(
**env).err2out() >> log_dest
else:
cmd_exe = exe(the_exe)[the_args].env(**env).err2out()
else:
# This block is for all the Non-MET commands
# Some commands still need to be run in a shell in order to work.
#
# There are currently 3 cases of non met commnds that need to be handled.
# case 1. Redirecting to a file in cmd string, which must be through a shell
# ie. cmd = ncdump ...series_F006/min.nc > .../min.txt
# cmd = exe('sh')['-c', cmd]
# case 2. Running the executable directly, w/ arguments, NO redirection
# ie. ncap2, cmd = exe(the_exe)[the_args]
# case 3. Runnng the command and logging the output to
# log_dest
if run_inshell:
# set the_exe to log command has finished running
the_exe = shlex.split(cmd)[0]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd_exe = exe('sh')['-c',
cmd].env(**env).err2out() >> log_dest
else:
cmd_exe = exe('sh')['-c', cmd].env(**env)
else:
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd_exe = exe(the_exe)[the_args].env(
**env).err2out() >> log_dest
else:
cmd_exe = exe(the_exe)[the_args].env(**env)
ret = 0
# run app unless DO_NOT_RUN_EXE is set to True
if not self.config.getbool('config', 'DO_NOT_RUN_EXE', False):
# get current time to calculate total time to run command
start_cmd_time = datetime.now()
# run command
try:
ret = run(cmd_exe, **kwargs)
except:
ret = -1
else:
# calculate time to run
end_cmd_time = datetime.now()
total_cmd_time = end_cmd_time - start_cmd_time
self.logger.debug(
f'Finished running {the_exe} in {total_cmd_time}')
return (ret, cmd)
def generate_plots(self, sorted_filter_init, tile_dir):
"""! Generate the plots from the series_analysis output.
Args:
@param sorted_filter_init: A list of the sorted directories
corresponding to the init times (that
are the result of filtering). If
filtering produced no results, this
is the list of files created from
running extract_tiles.
@param tile_dir: The directory where input data resides.
Returns:
"""
convert_exe = self.p.getexe('CONVERT_EXE')
background_map = self.p.getbool('config', 'BACKGROUND_MAP')
plot_data_plane_exe = os.path.join(self.p.getdir('MET_BUILD_BASE'),
'bin/plot_data_plane')
for cur_var in self.var_list:
name, level = util.get_name_level(cur_var, self.logger)
for cur_init in sorted_filter_init:
storm_list = self.get_storms_for_init(cur_init, tile_dir)
for cur_storm in storm_list:
# create the output directory where the finished
# plots will reside
output_dir = os.path.join(self.series_out_dir, cur_init,
cur_storm)
util.mkdir_p(output_dir)
# Now we need to invoke the MET tool
# plot_data_plane to generate plots that are
# recognized by the MET viewer.
# Get the number of forecast tile files,
# the name of the first and last in the list
# to be used by the -title option.
if tile_dir == self.extract_tiles_dir:
# Since filtering was not requested, or
# the additional filtering doesn't yield results,
# search the series_out_dir
num, beg, end = \
self.get_fcst_file_info(self.series_out_dir,
cur_init, cur_storm)
else:
# Search the series_filtered_out_dir for
# the filtered files.
num, beg, end = self.get_fcst_file_info(
self.series_filtered_out_dir, cur_init, cur_storm)
# Assemble the input file, output file, field string,
# and title
plot_data_plane_input_fname = self.sbi_plotting_out_dir
for cur_stat in self.stat_list:
plot_data_plane_output = [
output_dir, '/series_', name, '_', level, '_',
cur_stat, '.ps'
]
plot_data_plane_output_fname = ''.join(
plot_data_plane_output)
os.environ['CUR_STAT'] = cur_stat
self.add_env_var('CUR_STAT', cur_stat)
# Create versions of the arg based on
# whether the background map is requested
# in param file.
map_data = ' map_data={ source=[];}'
if background_map:
# Flag set to True, draw background map.
field_string_parts = [
"'name=", '"series_cnt_', cur_stat, '";',
'level="', level, '";', "'"
]
else:
field_string_parts = [
"'name=", '"series_cnt_', cur_stat, '";',
'level="', level, '";', map_data, "'"
]
field_string = ''.join(field_string_parts)
title_parts = [
' -title "GFS Init ', cur_init, ' Storm ',
cur_storm, ' ',
str(num), ' Forecasts (',
str(beg), ' to ',
str(end), '),', cur_stat, ' for ', cur_var, '"'
]
title = ''.join(title_parts)
# Now assemble the entire plot data plane command
data_plane_command_parts = \
[plot_data_plane_exe, ' ',
plot_data_plane_input_fname, ' ',
plot_data_plane_output_fname, ' ',
field_string, ' ', title]
data_plane_command = ''.join(data_plane_command_parts)
data_plane_command = \
batchexe('sh')[
'-c', data_plane_command].err2out()
run(data_plane_command)
# Now assemble the command to convert the
# postscript file to png
png_fname = plot_data_plane_output_fname.replace(
'.ps', '.png')
convert_parts = [
convert_exe, ' -rotate 90',
' -background white -flatten ',
plot_data_plane_output_fname, ' ', png_fname
]
convert_command = ''.join(convert_parts)
convert_command = \
batchexe('sh')['-c', convert_command].err2out()
run(convert_command)
def run_cmd(self, cmd, ismetcmd = True, app_name=None, run_inshell=False,
log_theoutput=False, **kwargs):
"""!The command cmd is a string which is converted to a produtil
exe Runner object and than run. Output of the command may also
be redirected to either METplus log, MET log, or TTY.
Some subclasses of CommandBuilder ie. series_by_init_wrapper, run
non MET commands ie. convert, in addition to MET binary commands,
ie. regrid_data_plane.
Args:
@param cmd: A string, Command used in the produtil exe Runner object.
@param ismetcmd: Default True, Will direct output to METplus log,
Metlog , or TTY. Set to False and use the other keywords as needed.
@param app_name: Used only when ismetcmd=True, The name of the exectable
being run.
@param run_inshell: Used only when ismetcmd=False, will Create a
runner object with the cmd being run through a shell, exe('sh')['-c', cmd]
This is required by commands, such as ncdump that are redirecting
output to a file, and other commands such as the convert command
when creating animated gifs.
@param log_theoutput: Used only when ismetcmd=False, will redirect
the stderr and stdout to a the METplus log file or tty.
DO Not set to True if the command is redirecting output to a file.
@param kwargs Other options sent to the produtil Run constructor
"""
if cmd is None:
return cmd
if ismetcmd:
# self.app_name MUST be defined in the subclass' constructor,
# this code block is a safety net in case that was not done.
# self.app_name is used to generate the MET log output name,
# if output is directed there, based on the conf settings.
#
# cmd.split()[0] 'should' be the /path/to/<some_met_binary>
if not app_name:
app_name = os.path.basename(cmd.split()[0])
self.logger.warning('MISSING self.app_name, '
'setting name to: %s' % repr(app_name))
self.logger.warning('Fix the code and edit the following objects'
' contructor: %s, ' % repr(self))
# Determine where to send the output from the MET command.
log_dest = self.cmdlog_destination(cmdlog=app_name+'.log')
# KEEP This comment as a reference note.
# Run the executable in a new process instead of through a shell.
# FYI. We were originally running the command through a shell
# which also works just fine. IF we go back to running through
# a shell, The string ,cmd, is formatted exactly as is needed.
# By formatted, it is as it would be when typed at the shell prompt.
# This includes, for example, quoting or backslash escaping filenames
# with spaces in them.
# Run the executable and pass the arguments as a sequence.
# Split the command in to a sequence using shell syntax.
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_dest:
self.logger.info("app_name is: %s, output sent to: %s" % (app_name, log_dest))
#cmd = exe('sh')['-c', cmd].err2out() >> log_dest
cmd = exe(the_exe)[the_args].err2out() >> log_dest
else:
#cmd = exe('sh')['-c', cmd].err2out()
cmd = exe(the_exe)[the_args].err2out()
else:
# This block is for all the Non-MET commands
# Some commands still need to be run in a shell in order to work.
#
# There are currently 3 cases of non met commnds that need to be handled.
# case 1. Redirecting to a file in cmd string, which must be through a shell
# ie. cmd = ncdump ...series_F006/min.nc > .../min.txt
# cmd = exe('sh')['-c', cmd]
# case 2. Running the executable directly, w/ arguments, NO redirection
# ie. ncap2, cmd = exe(the_exe)[the_args]
# case 3. Runnng the command and logging the output to
# log_dest
if run_inshell:
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd = exe('sh')['-c', cmd].err2out() >> log_dest
else:
cmd = exe('sh')['-c', cmd]
else:
the_exe = shlex.split(cmd)[0]
the_args = shlex.split(cmd)[1:]
if log_theoutput:
log_dest = self.cmdlog_destination()
cmd = exe(the_exe)[the_args].err2out() >> log_dest
else:
cmd = exe(the_exe)[the_args]
self.logger.info("RUNNING: %s" % cmd.to_shell())
self.logger.debug("RUNNING %s" % repr(cmd))
ret = run(cmd, **kwargs)
return (ret, cmd)