def tc_stat_wrapper(): """! Returns a default TCStatWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" # Default, empty TcStatWrapper with some configuration values set # to /path/to: conf = metplus_config() return TcStatWrapper(conf, None)
def run_at_time(self, cur_init): """!Get TC-paris data then regrid tiles centered on the storm. Get TC-pairs track data and GFS model data, do any necessary processing then regrid the forecast and analysis files to a 30 x 30 degree tile centered on the storm. Args: Returns: None: invokes regrid_data_plane to create a netCDF file from two extratropical storm track files. """ # pylint:disable=protected-access # Need to call sys.__getframe() to get the filename and method/func # for logging information. # Used in logging cur_filename = sys._getframe().f_code.co_filename cur_function = sys._getframe().f_code.co_name # get the process id to be used to identify the output # amongst different users and runs. cur_pid = str(os.getpid()) tmp_dir = os.path.join(self.config.getdir('TMP_DIR'), cur_pid) msg = ("INFO|[" + cur_filename + ":" + cur_function + "]" "|Begin extract tiles") self.logger.info(msg) # Check that there are tc_pairs data which are used as input if util.is_dir_empty(self.tc_pairs_dir): msg = ("ERROR|[" + cur_filename + ":" + cur_function + "]" "|No tc pairs data found at " + self.tc_pairs_dir + "Exiting...") self.logger.error(msg) sys.exit(1) # Logging output: TIME UTC |TYPE (DEBUG, INFO, WARNING, etc.) | # [File : function]| Message logger.info("INFO | [" + # cur_filename + ":" + "cur_function] |" + "BEGIN extract_tiles") # Process TC pairs by initialization time # Begin processing for initialization time, cur_init year_month = util.extract_year_month(cur_init, self.logger) # Create the name of the filter file we need to find. If # the file doesn't exist, then run TC_STAT filter_filename = "filter_" + cur_init + ".tcst" filter_name = os.path.join(self.filtered_out_dir, cur_init, filter_filename) if util.file_exists(filter_name) and not self.overwrite_flag: msg = ("DEBUG| [" + cur_filename + ":" + cur_function + " ] | Filter file exists, using Track data file: " + filter_name) self.logger.debug(msg) else: # Create the storm track by applying the # filter options defined in the config/param file. tile_dir_parts = [self.tc_pairs_dir, "/", year_month] tile_dir = ''.join(tile_dir_parts) # Use TcStatWrapper to build up the tc_stat command and invoke # the MET tool tc_stat to perform the filtering. tcs = TcStatWrapper(self.config) tcs.build_tc_stat(self.filtered_out_dir, cur_init, tile_dir, self.addl_filter_opts) # Remove any empty files and directories that can occur # from filtering. util.prune_empty(filter_name, self.logger) # Now get unique storm ids from the filter file, # filter_yyyymmdd_hh.tcst sorted_storm_ids = util.get_storm_ids(filter_name, self.logger) # Check for empty sorted_storm_ids, if empty, # continue to the next time. if not sorted_storm_ids: # No storms found for init time, cur_init msg = ("DEBUG|[" + cur_filename + ":" + cur_function + " ]|" + "No storms were found for " + cur_init + "...continue to next in list") self.logger.debug(msg) return # Process each storm in the sorted_storm_ids list # Iterate over each filter file in the output directory and # search for the presence of the storm id. Store this # corresponding row of data into a temporary file in the # /tmp/<pid> directory. for cur_storm in sorted_storm_ids: storm_output_dir = os.path.join(self.filtered_out_dir, cur_init, cur_storm) header = open(filter_name, "r").readline() util.mkdir_p(storm_output_dir) util.mkdir_p(tmp_dir) tmp_filename = "filter_" + cur_init + "_" + cur_storm full_tmp_filename = os.path.join(tmp_dir, tmp_filename) storm_match_list = util.grep(cur_storm, filter_name) with open(full_tmp_filename, "a+") as tmp_file: # copy over header information tmp_file.write(header) for storm_match in storm_match_list: tmp_file.write(storm_match) # Perform regridding of the forecast and analysis files # to an n X n degree tile centered on the storm (dimensions # are indicated in the config/param file). util.retrieve_and_regrid(full_tmp_filename, cur_init, cur_storm, self.filtered_out_dir, self.logger, self.config) # end of for cur_storm # Remove any empty files and directories in the extract_tiles output # directory util.prune_empty(self.filtered_out_dir, self.logger) # Clean up the tmp directory if it exists if os.path.isdir(tmp_dir): util.rmtree(tmp_dir) msg = ("INFO|[" + cur_function + ":" + cur_filename + "]" "| Finished extract tiles") self.logger.info(msg)
def run_at_time(self, input_dict): """!Get TC-paris data then regrid tiles centered on the storm. Get TC-pairs track data and GFS model data, do any necessary processing then regrid the forecast and analysis files to a 30 x 30 degree tile centered on the storm. Args: input_dict: Time dictionary Returns: None: invokes regrid_data_plane to create a netCDF file from two extratropical storm track files. """ time_info = time_util.ti_calculate(input_dict) init_time = time_info['init_fmt'] # get the process id to be used to identify the output # amongst different users and runs. cur_pid = str(os.getpid()) tmp_dir = os.path.join(self.config.getdir('TMP_DIR'), cur_pid) self.logger.info("Begin extract tiles") cur_init = init_time[0:8] + "_" + init_time[8:10] # Check that there are tc_pairs data which are used as input if util.is_dir_empty(self.tc_pairs_dir): self.logger.error("No tc pairs data found at {}"\ .format(self.tc_pairs_dir)) sys.exit(1) # Create the name of the filter file we need to find. If # the file doesn't exist, then run TC_STAT filter_filename = "filter_" + cur_init + ".tcst" filter_name = os.path.join(self.filtered_out_dir, cur_init, filter_filename) if util.file_exists(filter_name) and not self.overwrite_flag: self.logger.debug("Filter file exists, using Track data file: {}"\ .format(filter_name)) else: # Create the storm track by applying the # filter options defined in the config/param file. # Use TcStatWrapper to build up the tc_stat command and invoke # the MET tool tc_stat to perform the filtering. tiles_list = util.get_files(self.tc_pairs_dir, ".*tcst", self.logger) tiles_list_str = ' '.join(tiles_list) tcs = TcStatWrapper(self.config, self.logger) tcs.build_tc_stat(self.filtered_out_dir, cur_init, tiles_list_str, self.addl_filter_opts) # Remove any empty files and directories that can occur # from filtering. util.prune_empty(filter_name, self.logger) # Now get unique storm ids from the filter file, # filter_yyyymmdd_hh.tcst sorted_storm_ids = util.get_storm_ids(filter_name, self.logger) # Check for empty sorted_storm_ids, if empty, # continue to the next time. if not sorted_storm_ids: # No storms found for init time, cur_init msg = "No storms were found for {} ...continue to next in list"\ .format(cur_init) self.logger.debug(msg) return # Process each storm in the sorted_storm_ids list # Iterate over each filter file in the output directory and # search for the presence of the storm id. Store this # corresponding row of data into a temporary file in the # /tmp/<pid> directory. for cur_storm in sorted_storm_ids: storm_output_dir = os.path.join(self.filtered_out_dir, cur_init, cur_storm) header = open(filter_name, "r").readline() util.mkdir_p(storm_output_dir) util.mkdir_p(tmp_dir) tmp_filename = "filter_" + cur_init + "_" + cur_storm full_tmp_filename = os.path.join(tmp_dir, tmp_filename) storm_match_list = util.grep(cur_storm, filter_name) with open(full_tmp_filename, "a+") as tmp_file: # copy over header information tmp_file.write(header) for storm_match in storm_match_list: tmp_file.write(storm_match) # Perform regridding of the forecast and analysis files # to an n X n degree tile centered on the storm (dimensions # are indicated in the config/param file). feature_util.retrieve_and_regrid(full_tmp_filename, cur_init, cur_storm, self.filtered_out_dir, self.config) # end of for cur_storm # Remove any empty files and directories in the extract_tiles output # directory util.prune_empty(self.filtered_out_dir, self.logger) # Clean up the tmp directory if it exists if os.path.isdir(tmp_dir): util.rmtree(tmp_dir)
def apply_series_filters(tile_dir, init_times, series_output_dir, filter_opts, temporary_dir, logger, config): """! Apply filter options, as specified in the param/config file. Args: @param tile_dir: Directory where input data files reside. e.g. data which we will be applying our filter criteria. @param init_times: List of init times that define the input data. @param series_output_dir: The directory where the filter results will be stored. @param filter_opts: The filter options to apply @param temporary_dir: The temporary directory where intermediate files are saved. @param logger: The logger to which all logging is directed. @param config: The config/param instance Returns: None """ # pylint: disable=too-many-arguments # Seven input arguments are needed to perform filtering. # pylint:disable=protected-access # Need to call sys.__getframe() to get the filename and method/func # for logging information. # Useful for logging cur_filename = sys._getframe().f_code.co_filename cur_function = sys._getframe().f_code.co_name # Create temporary directory where intermediate files are saved. cur_pid = str(os.getpid()) tmp_dir = os.path.join(temporary_dir, cur_pid) logger.debug("DEBUG|" + cur_filename + "|" + cur_function + " creating tmp dir: " + tmp_dir) for cur_init in init_times: # Call the tc_stat wrapper to build up the command and invoke # the MET tool tc_stat. filter_file = "filter_" + cur_init + ".tcst" filter_filename = os.path.join(series_output_dir, cur_init, filter_file) tcs = TcStatWrapper(config) tcs.build_tc_stat(series_output_dir, cur_init, tile_dir, filter_opts) # Check that the filter.tcst file isn't empty. If # it is, then use the files from extract_tiles as # input (tile_dir = extract_out_dir) if not file_exists(filter_filename): msg = ("WARN| " + cur_filename + ":" + cur_function + "]| Non-existent filter file, filter " + " Never created by MET Tool tc_stat.") logger.debug(msg) continue elif os.stat(filter_filename).st_size == 0: msg = ("WARN| " + cur_filename + ":" + cur_function + "]| Empty filter file, filter " + " options yield nothing.") logger.debug(msg) continue else: # Now retrieve the files corresponding to these # storm ids that resulted from filtering. sorted_storm_ids = get_storm_ids(filter_filename, logger) # Retrieve the header from filter_filename to be used in # creating the temporary files. with open(filter_filename, 'r') as filter_file: header = filter_file.readline() for cur_storm in sorted_storm_ids: msg = ("INFO| [" + cur_filename + ":" + cur_function + " ] | Processing storm: " + cur_storm + " for file: " + filter_filename) logger.debug(msg) storm_output_dir = os.path.join(series_output_dir, cur_init, cur_storm) mkdir_p(storm_output_dir) mkdir_p(tmp_dir) tmp_file = "filter_" + cur_init + "_" + cur_storm tmp_filename = os.path.join(tmp_dir, tmp_file) storm_match_list = grep(cur_storm, filter_filename) with open(tmp_filename, "a+") as tmp_file: tmp_file.write(header) for storm_match in storm_match_list: tmp_file.write(storm_match) # Create the analysis and forecast files based # on the storms (defined in the tmp_filename created above) # Store the analysis and forecast files in the # series_output_dir. retrieve_and_regrid(tmp_filename, cur_init, cur_storm, series_output_dir, logger, config) # Check for any empty files and directories and remove them to avoid # any errors or performance degradation when performing # series analysis. prune_empty(series_output_dir, logger) # Clean up the tmp dir rmtree(tmp_dir)