def tc_stat_wrapper():
"""! Returns a default TCStatWrapper with /path/to entries in the
metplus_system.conf and metplus_runtime.conf configuration
files. Subsequent tests can customize the final METplus configuration
to over-ride these /path/to values."""
# Default, empty TcStatWrapper with some configuration values set
# to /path/to:
conf = metplus_config()
return TcStatWrapper(conf, None)
def run_at_time(self, cur_init):
"""!Get TC-paris data then regrid tiles centered on the storm.
Get TC-pairs track data and GFS model data, do any necessary
processing then regrid the forecast and analysis files to a
30 x 30 degree tile centered on the storm.
Args:
Returns:
None: invokes regrid_data_plane to create a netCDF file from two
extratropical storm track files.
"""
# pylint:disable=protected-access
# Need to call sys.__getframe() to get the filename and method/func
# for logging information.
# Used in logging
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
# get the process id to be used to identify the output
# amongst different users and runs.
cur_pid = str(os.getpid())
tmp_dir = os.path.join(self.config.getdir('TMP_DIR'), cur_pid)
msg = ("INFO|[" + cur_filename + ":" + cur_function + "]"
"|Begin extract tiles")
self.logger.info(msg)
# Check that there are tc_pairs data which are used as input
if util.is_dir_empty(self.tc_pairs_dir):
msg = ("ERROR|[" + cur_filename + ":" + cur_function + "]"
"|No tc pairs data found at " + self.tc_pairs_dir +
"Exiting...")
self.logger.error(msg)
sys.exit(1)
# Logging output: TIME UTC |TYPE (DEBUG, INFO, WARNING, etc.) |
# [File : function]| Message logger.info("INFO | [" +
# cur_filename + ":" + "cur_function] |" + "BEGIN extract_tiles")
# Process TC pairs by initialization time
# Begin processing for initialization time, cur_init
year_month = util.extract_year_month(cur_init, self.logger)
# Create the name of the filter file we need to find. If
# the file doesn't exist, then run TC_STAT
filter_filename = "filter_" + cur_init + ".tcst"
filter_name = os.path.join(self.filtered_out_dir, cur_init,
filter_filename)
if util.file_exists(filter_name) and not self.overwrite_flag:
msg = ("DEBUG| [" + cur_filename + ":" + cur_function +
" ] | Filter file exists, using Track data file: " +
filter_name)
self.logger.debug(msg)
else:
# Create the storm track by applying the
# filter options defined in the config/param file.
tile_dir_parts = [self.tc_pairs_dir, "/", year_month]
tile_dir = ''.join(tile_dir_parts)
# Use TcStatWrapper to build up the tc_stat command and invoke
# the MET tool tc_stat to perform the filtering.
tcs = TcStatWrapper(self.config)
tcs.build_tc_stat(self.filtered_out_dir, cur_init,
tile_dir, self.addl_filter_opts)
# Remove any empty files and directories that can occur
# from filtering.
util.prune_empty(filter_name, self.logger)
# Now get unique storm ids from the filter file,
# filter_yyyymmdd_hh.tcst
sorted_storm_ids = util.get_storm_ids(filter_name, self.logger)
# Check for empty sorted_storm_ids, if empty,
# continue to the next time.
if not sorted_storm_ids:
# No storms found for init time, cur_init
msg = ("DEBUG|[" + cur_filename + ":" + cur_function + " ]|" +
"No storms were found for " + cur_init +
"...continue to next in list")
self.logger.debug(msg)
return
# Process each storm in the sorted_storm_ids list
# Iterate over each filter file in the output directory and
# search for the presence of the storm id. Store this
# corresponding row of data into a temporary file in the
# /tmp/<pid> directory.
for cur_storm in sorted_storm_ids:
storm_output_dir = os.path.join(self.filtered_out_dir,
cur_init, cur_storm)
header = open(filter_name, "r").readline()
util.mkdir_p(storm_output_dir)
util.mkdir_p(tmp_dir)
tmp_filename = "filter_" + cur_init + "_" + cur_storm
full_tmp_filename = os.path.join(tmp_dir, tmp_filename)
storm_match_list = util.grep(cur_storm, filter_name)
with open(full_tmp_filename, "a+") as tmp_file:
# copy over header information
tmp_file.write(header)
for storm_match in storm_match_list:
tmp_file.write(storm_match)
# Perform regridding of the forecast and analysis files
# to an n X n degree tile centered on the storm (dimensions
# are indicated in the config/param file).
util.retrieve_and_regrid(full_tmp_filename, cur_init,
cur_storm, self.filtered_out_dir,
self.logger, self.config)
# end of for cur_storm
# Remove any empty files and directories in the extract_tiles output
# directory
util.prune_empty(self.filtered_out_dir, self.logger)
# Clean up the tmp directory if it exists
if os.path.isdir(tmp_dir):
util.rmtree(tmp_dir)
msg = ("INFO|[" + cur_function + ":" + cur_filename + "]"
"| Finished extract tiles")
self.logger.info(msg)
def run_at_time(self, input_dict):
"""!Get TC-paris data then regrid tiles centered on the storm.
Get TC-pairs track data and GFS model data, do any necessary
processing then regrid the forecast and analysis files to a
30 x 30 degree tile centered on the storm.
Args:
input_dict: Time dictionary
Returns:
None: invokes regrid_data_plane to create a netCDF file from two
extratropical storm track files.
"""
time_info = time_util.ti_calculate(input_dict)
init_time = time_info['init_fmt']
# get the process id to be used to identify the output
# amongst different users and runs.
cur_pid = str(os.getpid())
tmp_dir = os.path.join(self.config.getdir('TMP_DIR'), cur_pid)
self.logger.info("Begin extract tiles")
cur_init = init_time[0:8] + "_" + init_time[8:10]
# Check that there are tc_pairs data which are used as input
if util.is_dir_empty(self.tc_pairs_dir):
self.logger.error("No tc pairs data found at {}"\
.format(self.tc_pairs_dir))
sys.exit(1)
# Create the name of the filter file we need to find. If
# the file doesn't exist, then run TC_STAT
filter_filename = "filter_" + cur_init + ".tcst"
filter_name = os.path.join(self.filtered_out_dir, cur_init,
filter_filename)
if util.file_exists(filter_name) and not self.overwrite_flag:
self.logger.debug("Filter file exists, using Track data file: {}"\
.format(filter_name))
else:
# Create the storm track by applying the
# filter options defined in the config/param file.
# Use TcStatWrapper to build up the tc_stat command and invoke
# the MET tool tc_stat to perform the filtering.
tiles_list = util.get_files(self.tc_pairs_dir, ".*tcst",
self.logger)
tiles_list_str = ' '.join(tiles_list)
tcs = TcStatWrapper(self.config, self.logger)
tcs.build_tc_stat(self.filtered_out_dir, cur_init, tiles_list_str,
self.addl_filter_opts)
# Remove any empty files and directories that can occur
# from filtering.
util.prune_empty(filter_name, self.logger)
# Now get unique storm ids from the filter file,
# filter_yyyymmdd_hh.tcst
sorted_storm_ids = util.get_storm_ids(filter_name, self.logger)
# Check for empty sorted_storm_ids, if empty,
# continue to the next time.
if not sorted_storm_ids:
# No storms found for init time, cur_init
msg = "No storms were found for {} ...continue to next in list"\
.format(cur_init)
self.logger.debug(msg)
return
# Process each storm in the sorted_storm_ids list
# Iterate over each filter file in the output directory and
# search for the presence of the storm id. Store this
# corresponding row of data into a temporary file in the
# /tmp/<pid> directory.
for cur_storm in sorted_storm_ids:
storm_output_dir = os.path.join(self.filtered_out_dir, cur_init,
cur_storm)
header = open(filter_name, "r").readline()
util.mkdir_p(storm_output_dir)
util.mkdir_p(tmp_dir)
tmp_filename = "filter_" + cur_init + "_" + cur_storm
full_tmp_filename = os.path.join(tmp_dir, tmp_filename)
storm_match_list = util.grep(cur_storm, filter_name)
with open(full_tmp_filename, "a+") as tmp_file:
# copy over header information
tmp_file.write(header)
for storm_match in storm_match_list:
tmp_file.write(storm_match)
# Perform regridding of the forecast and analysis files
# to an n X n degree tile centered on the storm (dimensions
# are indicated in the config/param file).
feature_util.retrieve_and_regrid(full_tmp_filename, cur_init,
cur_storm, self.filtered_out_dir,
self.config)
# end of for cur_storm
# Remove any empty files and directories in the extract_tiles output
# directory
util.prune_empty(self.filtered_out_dir, self.logger)
# Clean up the tmp directory if it exists
if os.path.isdir(tmp_dir):
util.rmtree(tmp_dir)
def apply_series_filters(tile_dir, init_times, series_output_dir, filter_opts,
temporary_dir, logger, config):
"""! Apply filter options, as specified in the
param/config file.
Args:
@param tile_dir: Directory where input data files reside.
e.g. data which we will be applying our filter
criteria.
@param init_times: List of init times that define the input data.
@param series_output_dir: The directory where the filter results
will be stored.
@param filter_opts: The filter options to apply
@param temporary_dir: The temporary directory where intermediate
files are saved.
@param logger: The logger to which all logging is directed.
@param config: The config/param instance
Returns:
None
"""
# pylint: disable=too-many-arguments
# Seven input arguments are needed to perform filtering.
# pylint:disable=protected-access
# Need to call sys.__getframe() to get the filename and method/func
# for logging information.
# Useful for logging
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
# Create temporary directory where intermediate files are saved.
cur_pid = str(os.getpid())
tmp_dir = os.path.join(temporary_dir, cur_pid)
logger.debug("DEBUG|" + cur_filename + "|" + cur_function +
" creating tmp dir: " + tmp_dir)
for cur_init in init_times:
# Call the tc_stat wrapper to build up the command and invoke
# the MET tool tc_stat.
filter_file = "filter_" + cur_init + ".tcst"
filter_filename = os.path.join(series_output_dir, cur_init,
filter_file)
tcs = TcStatWrapper(config)
tcs.build_tc_stat(series_output_dir, cur_init, tile_dir, filter_opts)
# Check that the filter.tcst file isn't empty. If
# it is, then use the files from extract_tiles as
# input (tile_dir = extract_out_dir)
if not file_exists(filter_filename):
msg = ("WARN| " + cur_filename + ":" + cur_function +
"]| Non-existent filter file, filter " +
" Never created by MET Tool tc_stat.")
logger.debug(msg)
continue
elif os.stat(filter_filename).st_size == 0:
msg = ("WARN| " + cur_filename + ":" + cur_function +
"]| Empty filter file, filter " + " options yield nothing.")
logger.debug(msg)
continue
else:
# Now retrieve the files corresponding to these
# storm ids that resulted from filtering.
sorted_storm_ids = get_storm_ids(filter_filename, logger)
# Retrieve the header from filter_filename to be used in
# creating the temporary files.
with open(filter_filename, 'r') as filter_file:
header = filter_file.readline()
for cur_storm in sorted_storm_ids:
msg = ("INFO| [" + cur_filename + ":" + cur_function +
" ] | Processing storm: " + cur_storm + " for file: " +
filter_filename)
logger.debug(msg)
storm_output_dir = os.path.join(series_output_dir, cur_init,
cur_storm)
mkdir_p(storm_output_dir)
mkdir_p(tmp_dir)
tmp_file = "filter_" + cur_init + "_" + cur_storm
tmp_filename = os.path.join(tmp_dir, tmp_file)
storm_match_list = grep(cur_storm, filter_filename)
with open(tmp_filename, "a+") as tmp_file:
tmp_file.write(header)
for storm_match in storm_match_list:
tmp_file.write(storm_match)
# Create the analysis and forecast files based
# on the storms (defined in the tmp_filename created above)
# Store the analysis and forecast files in the
# series_output_dir.
retrieve_and_regrid(tmp_filename, cur_init, cur_storm,
series_output_dir, logger, config)
# Check for any empty files and directories and remove them to avoid
# any errors or performance degradation when performing
# series analysis.
prune_empty(series_output_dir, logger)
# Clean up the tmp dir
rmtree(tmp_dir)