def test_create_grid2obs_regex_all(): pytest.skip('deprecated function') # Test that the regex created from the template that has valid # cycle, lead and offset is correct (expected). logger = logging.getLogger("test") templ = \ '/path/to/nam.{init?fmt=%Y%m%d}/rap.t{cycle?fmt=%HH}z.awphys{lead?fmt=%H}.' \ 'tm{offset?fmt=%H}.grib2' # variables to set for StringSub init_str = "20180815" cycle_str = '00' lead_str = '60' offset_str = '00' ss = StringSub(logger, templ, init=init_str, cycle=cycle_str, lead=lead_str, offset=offset_str) actual_regex = ss.create_grid2obs_regex() expected_regex = '/path/to/nam.([0-9]{8})/rap.t([0-9]{2,3})z.' \ 'awphys([0-9]{1,3}).tm([0-9]{2,3}).grib2$' assert actual_regex == expected_regex
def test_offset_hour(): logger = logging.getLogger("dummy") expected_hour = "03" offset = 10800 templ = "{offset?fmt=%2H}" ss = StringSub(logger, templ, offset=offset) offset_hour = ss.doStringSub() assert (offset_hour == expected_hour)
def test_h_lead_pad_2_digit_sub(): logger = logging.getLogger("test") file_template = "{init?fmt=%Y%m%d%H}_A{lead?fmt=%.3H}h" init_time = datetime.datetime.strptime("1987020103", '%Y%m%d%H') lead_time = int("3") * 3600 fSts = StringSub(logger, file_template, init=init_time, lead=lead_time) out_string = fSts.doStringSub() assert (out_string == "1987020103_A003h")
def test_shift_time_negative(): init_string = datetime.datetime.strptime("2017060400", '%Y%m%d%H') logger = logging.getLogger("testing") templ = "{init?fmt=%Y%m%d%H?shift=-86400}" expected_filename = "2017060300" ss = StringSub(logger, templ, init=init_string) filename = ss.doStringSub() assert (filename == expected_filename)
def test_shift_time_lead_negative(): init_string = datetime.datetime.strptime("2019020700", '%Y%m%d%H') lead_string = int("60") * 3600 logger = logging.getLogger("testing") templ = "dwd_{init?fmt=%Y%m%d%H}_{lead?fmt=%.3H?shift=-86400}_{lead?fmt=%.3H}" expected_filename = "dwd_2019020700_036_060" ss = StringSub(logger, templ, init=init_string, lead=lead_string) filename = ss.doStringSub() assert (filename == expected_filename)
def test_multiple_valid_substitution_valid(): valid_string = datetime.datetime.strptime("2018020112", '%Y%m%d%H') lead_string = int("123") * 3600 logger = logging.getLogger("testing") templ = "{valid?fmt=%Y%m%d%H}/gfs.t{valid?fmt=%H}.pgrb2.0p25.{lead?fmt=%HHH}" expected_filename = "2018020112/gfs.t12.pgrb2.0p25.123" ss = StringSub(logger, templ, valid=valid_string, lead=lead_string) filename = ss.doStringSub() assert (filename == expected_filename)
def test_multiple_valid_substitution_init(): init_string = datetime.datetime.strptime("2017060400", '%Y%m%d%H') lead_string = 0 logger = logging.getLogger("testing") templ = "{init?fmt=%Y%m%d%H}/gfs.t{init?fmt=%H}z.pgrb2.0p25.f{lead?fmt=%.2H}" expected_filename = "2017060400/gfs.t00z.pgrb2.0p25.f00" ss = StringSub(logger, templ, init=init_string, lead=lead_string) filename = ss.doStringSub() assert (filename == expected_filename)
def test_multiple_valid_substitution_init_complex(): init_string = datetime.datetime.strptime("2016061018", '%Y%m%d%H') lead_string = int("6") * 3600 logger = logging.getLogger("testing") templ = "ncar.ral.CoSPA.HRRR.{init?fmt=%Y-%m-%dT%H:%M:%S}.PT{lead?fmt=%.2H}:00.nc" expected_filename = "ncar.ral.CoSPA.HRRR.2016-06-10T18:00:00.PT06:00.nc" ss = StringSub(logger, templ, init=init_string, lead=lead_string) filename = ss.doStringSub() assert (filename == expected_filename)
def test_cycle_hour(): cycle_string = 0 valid_string = datetime.datetime.strptime("20180103", '%Y%m%d') logger = logging.getLogger("dummy") templ = "prefix.{valid?fmt=%Y%m%d}.tm{cycle?fmt=%2H}" expected_filename = "prefix.20180103.tm00" ss = StringSub(logger, templ, valid=valid_string, cycle=cycle_string) filename = ss.doStringSub() assert (filename == expected_filename)
def test_crow_variable_hour(): # Test that StringSub's doStringSub() correctly creates the valid hour # without any zero-padding when given the following as input: # pgbf{lead?fmt=%H}.gfs.{valid?fmt=%Y%M%D%H} # pgbf([0-9]{1,3}).gfs.(2[0-9]{9}) logger = logging.getLogger("crow_data") # crow input files with 3, 2, and 1-digit lead times: crow_input_file_3 = 'pgbf219.gfs.2017060418' crow_input_file_2 = 'pgbf18.gfs.2017062000' crow_input_file_1 = 'pgbf3.gfs.2017060418' lead_1 = int('3') * 3600 lead_2 = int('18') * 3600 lead_3 = int('219') * 3600 valid_2 = datetime.datetime.strptime('2017062000', '%Y%m%d%H') valid_1 = valid_3 = datetime.datetime.strptime('2017060418', '%Y%m%d%H') templ = 'pgbf{lead?fmt=%H}.gfs.{valid?fmt=%Y%m%d%H}' ss_1 = StringSub(logger, templ, valid=valid_1, lead=lead_1) ss_2 = StringSub(logger, templ, valid=valid_2, lead=lead_2) ss_3 = StringSub(logger, templ, valid=valid_3, lead=lead_3) crow_1_output = ss_1.doStringSub() crow_2_output = ss_2.doStringSub() crow_3_output = ss_3.doStringSub() # print("crow_1 output: ", crow_1_output) # print("crow_2 output: ", crow_2_output) # print("crow_3 output: ", crow_3_output) assert (crow_1_output == crow_input_file_1 and crow_2_output == crow_input_file_2 and crow_3_output == crow_input_file_3)
def test_ym_date_dir(): # Test that the ym directory can be read in and does substitution correctly logger = logging.getLogger("test") # e.g. /d1/METplus_TC/adeck_orig/201708/atcfunix.gfs.2017080100 date_str = '201708' templ = '/d1/METplus_TC/adeck_orig/{date?fmt=%s}/' \ 'atcfunix.gfs.2017080100.dat' ss = StringSub(logger, templ, date=date_str) filename = ss.doStringSub() expected_filename = '/d1/METplus_TC/adeck_orig/201708/' \ 'atcfunix.gfs.2017080100.dat' assert filename == expected_filename
def test_gdas_substitution(): # Test that the string template substitution works correctly for GDAS # prepbufr files, which do not make use of the cycle hour or the offset # to generate the valid time. valid_string = "2018010411" valid_obj = datetime.datetime.strptime(valid_string, '%Y%m%d%H') logger = logging.getLogger("testing") templ = "prepbufr.gdas.{valid?fmt=%Y%m%d%H}.nc" expected_filename = 'prepbufr.gdas.' + valid_string + '.nc' ss = StringSub(logger, templ, valid=valid_obj) filename = ss.doStringSub() assert (filename == expected_filename)
def generate_output_nc_filename(self, prepbufr_file_info): """! Create the output netCDF filename as specified in the use case/custom configuration file. Args: @param prepbufr_file_info - a list of the full filepaths of prepbufr data of interest. Returns: a tuple: nc_output_filepath - the full filepath nc_output_filename - the filename follows the format specified in the configuration file """ # pylint:disable=protected-access # Need to call sys.__getframe() to get the filename and method/func # for logging information. cur_filename = sys._getframe().f_code.co_filename cur_function = sys._getframe().f_code.co_name self.logger.debug('DEBUG:|' + cur_function + '|' + cur_filename + ' Generating output NetCDF file name...') # Get the output directory pb2nc_output_dir = self.pb_dict['PB2NC_OUTPUT_DIR'] # Get the cycle hour and offset hour from the prepbufr file info named # tuple if prepbufr_file_info.cycle: # Get the cycle hour, offset hour and add the appropriate # prefix, validation ymd and .nc extension cycle = prepbufr_file_info.cycle offset = prepbufr_file_info.offset date = prepbufr_file_info.date string_sub = StringSub(self.logger, self.pb_dict['NC_FILE_TMPL'], init=str(date), cycle=cycle, offset=offset) nc_output_filename = string_sub.doStringSub() nc_output_filepath = os.path.join(pb2nc_output_dir, nc_output_filename) else: # Typically for files that aren't separated into dated # subdirectories, the date is incorporated in the filename. # Append the input file name with .nc extension # extract the filename portion of the full_filepath filename = os.path.basename(prepbufr_file_info.full_filepath) nc_output_filename = filename + ".nc" nc_output_filepath = os.path.join(pb2nc_output_dir, nc_output_filename) return nc_output_filepath
def test_ymd_date_dir(): # Test that the ymd directory can be read in and does substitution correctly logger = logging.getLogger("test") # e.g. /d1/METplus_TC/adeck_orig/20170811/atcfunix.gfs.2017080100 init_str = datetime.datetime.strptime('2017081118', '%Y%m%d%H') date_str = '20170811' templ = '/d1/METplus_TC/adeck_orig/{date?fmt=%s}/atcfunix.gfs.' \ '{init?fmt=%Y%m%d%H}.dat' ss = StringSub(logger, templ, date=date_str, init=init_str) filename = ss.doStringSub() expected_filename = '/d1/METplus_TC/adeck_orig/20170811/' \ 'atcfunix.gfs.2017081118.dat' assert filename == expected_filename
def test_calc_valid_for_prepbufr(key, value): pytest.skip('deprecated function') # Verify that the previous day is correctly calculated when # the negative_offset_hour > cycle_hour cycle_hour = "00" init_string = "2018010306" logger = logging.getLogger("dummy") templ = "prefix.{valid?fmt=%Y%m%d%H}.tm{cycle?fmt=%H}z.tm{" \ "offset?fmt=%H}.nc" ss = StringSub(logger, templ, init=init_string, cycle=cycle_hour, offset=key) valid_time = ss.calc_valid_for_prepbufr() assert (valid_time == value)
def test_create_grid2obs_regex_gfs(): pytest.skip('deprecated function') # Test that the regex created from a template is what is expected logger = logging.getLogger("test") templ = '/path/to/gfs/pgbf{lead?fmt=%H}.gfs.{valid?fmt=%Y%m%d%HH}' # variables to pass into StringSub valid_str = datetime.datetime.strptime('2017081118', '%Y%m%d%H') lead_str = 0 ss = StringSub(logger, templ, valid=valid_str, lead=lead_str) actual_regex = ss.create_grid2obs_regex() expected_regex = '/path/to/gfs/pgbf([0-9]{1,3}).gfs.([0-9]{10})$' assert actual_regex == expected_regex
def test_create_cyclone_regex(): pytest.skip('deprecated function') # Test that the regex created from a template is what is expected logger = logging.getLogger("test") templ = '/d1/METplus_TC/bdeck/{date?fmt=%s}/b{region?fmt=%s}' \ '{cyclone?fmt=%s}{misc?fmt=%s}.dat' date_str = '201708' region_str = 'al' cyclone_str = '05' year_str = '2017' ss = StringSub(logger, templ, date=date_str, region=region_str, cyclone=cyclone_str, misc=year_str) actual_regex = ss.create_cyclone_regex() expected_regex = '/d1/METplus_TC/bdeck/([0-9]{4,10})/b([a-zA-Z]{2})([0-9]' \ '{2,3})([a-zA-Z0-9-_.]+).dat' assert actual_regex == expected_regex
def test_offset_hour(): logger = logging.getLogger("dummy") expected_hour = int(3) offset = "03" templ = "prefix.{valid?fmt=%Y%m%d}.tm{offset?fmt=%H}" ss = StringSub(logger, templ, offset=offset) assert (ss.offset_hour == expected_hour)
def test_create_grid2obs_regex_hrrr(): pytest.skip('deprecated function') # Test that the regex created from a template is what is expected logger = logging.getLogger("test") templ = \ '/path/to/hrrr/hrrr.t{cycle?fmt=%HH}z.wrfprsf{lead?fmt=%HH}.grib2' # variables to set for StringSub cycle_str = '15' lead_str = '12' ss = StringSub(logger, templ, cycle=cycle_str, lead=lead_str) actual_regex = ss.create_grid2obs_regex() expected_regex = \ '/path/to/hrrr/hrrr.t([0-9]{2,3})z.wrfprsf([0-9]{1,3}).grib2$' assert actual_regex == expected_regex
def test_nam_substitution_HH(key, value): pytest.skip('time offsets no longer computed in StringSub') # Test that the substitution works correctly when given an init time, # cycle hour, and negative offset hour. init_string = datetime.datetime.strptime("20180102", '%Y%m%d') cycle_string = key offset_string = 10800 #'03' expected_filename = value logger = logging.getLogger("test") templ = \ 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%HH}z.tm{' \ 'offset?fmt=%HH}.nc' ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, offset=offset_string) filename = ss.do_string_sub() # print('nam filename: ', filename) assert (filename == expected_filename)
def test_create_grid2obs_regex_gdas(): pytest.skip('deprecated function') # Test that the regex created from a template is what is expected logger = logging.getLogger("test") templ = \ '/path/to/gdas/prepbufr.gdas.{valid?fmt=%Y%m%d%HH}' # '/path/to/nam.20170611/nam.t[cycle?fmt=%HH}z.prepbufr.tm{offset?fmt=%HH}' # templ_hrrr ='/path/to/hrrr.t{cycle?fmt=%HH}z.wrfprsf{lead?fmt=%HH}.grib2' # tmpl_gdas = 'prepbufr.gdas.{valid?fmt=%Y%m%d%HH}' # variables to set for StringSub valid_str = '2017063018' ss = StringSub(logger, templ, valid=valid_str) actual_regex = ss.create_grid2obs_regex() expected_regex = '/path/to/gdas/prepbufr.gdas.([0-9]{10})$' assert actual_regex == expected_regex
def test_create_grid2obs_regex_nam(): pytest.skip('deprecated function') # Test that the regex created from a template is what is expected logger = logging.getLogger("test") templ = \ '/path/to/nam.20170811/nam.t{cycle?' \ 'fmt=%HH}z.prepbufr.tm{offset?fmt=%HH}' # variables to set for StringSub cycle_str = '18' offset_str = '03' ss = StringSub(logger, templ, cycle=cycle_str, offset=offset_str) actual_regex = ss.create_grid2obs_regex() expected_regex = \ '/path/to/nam.20170811/nam.t([0-9]{2,3})z.prepbufr.tm([0-9]{2,3})$' assert actual_regex == expected_regex
def test_ymd_region_cyclone(): # Test that we can recreate the full file path with a date, # region, and cyclone logger = logging.getLogger("test") # /d1/METplus_TC/bdeck_orig/20170811/bal052017.dat date_str = '201708' region_str = 'al' cyclone_str = '05' year_str = '2017' # templ = '/d1/METplus_TC/bdeck/{date?fmt=%Y%m}/bal{region?fmt=%s}.dat' templ = '/d1/METplus_TC/bdeck/{date?fmt=%s}/b{region?fmt=%s}' \ '{cyclone?fmt=%s}{misc?fmt=%s}.dat' ss = StringSub(logger, templ, date=date_str, region=region_str, cyclone=cyclone_str, misc=year_str) full_file = ss.do_string_sub() expected_full_file = '/d1/METplus_TC/bdeck/201708/bal052017.dat' assert full_file == expected_full_file
def test_cycle_hour(): cycle_string = "00" valid_string = "20180103" logger = logging.getLogger("dummy") templ = "prefix.{valid?fmt=%Y%m%d}.tm{cycle?fmt=%H}" ss = StringSub(logger, templ, valid=valid_string, cycle=cycle_string) expected_hours = int(0) assert (ss.cycle_time_hours == expected_hours)
def test_nam_substitution_HHH(key, value): # Test that the substitution works correctly when given an init time, # cycle hour, and negative offset hour. init_string = "20180102" cycle_string = key offset_string = '03' expected_filename = value logger = logging.getLogger("test") templ = \ 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%HHH}z.tm{' \ 'offset?fmt=%HH}.nc' ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, offset=offset_string) filename = ss.doStringSub() print('nam filename: ', filename) assert (filename == expected_filename)
def find_deck_files(self, deck, basin, cyclone, model_list, time_info): """!Find ADECK or EDECK files that correspond to the BDECk file found Args: @param deck type of deck (A or E) @param basin region of storm from config @param cyclone ID number of cyclone from config @param model_list list of models that be available @param time_info object containing timing information to process """ deck_list = [] # get matching adeck wildcard expression for first model string_sub = StringSub(self.logger, self.c_dict[deck + 'DECK_TEMPLATE'], basin=basin, cyclone=cyclone, model=model_list[0], **time_info) deck_expr = os.path.join(self.c_dict[deck + 'DECK_DIR'], string_sub.do_string_sub()) # add adeck files if they exist for model in model_list: deck_glob = deck_expr.replace(model_list[0], model) self.logger.debug('Looking for {}DECK file: {}'.format( deck, deck_glob)) deck_files = glob.glob(deck_glob) if not deck_files: continue # there should only be 1 file that matches deck_file = deck_files[0] # if deck exists, add to list if os.path.isfile(deck_file): self.logger.debug('Adding {}DECK: {}'.format(deck, deck_file)) deck_list.append(deck_file) return deck_list
def run_at_time(self, input_dict): """! Do some processing for the current run time (init or valid) Args: @param input_dict dictionary containing time information of current run generally contains 'now' (current) time and 'init' or 'valid' time """ # fill in time info dictionary time_info = time_util.ti_calculate(input_dict) # check if looping by valid or init and log time for run loop_by = time_info['loop_by'] self.logger.info('Running ExampleWrapper at {} time {}'.format(loop_by, time_info[loop_by+'_fmt'])) # read input directory and template from config dictionary input_dir = self.c_dict['INPUT_DIR'] input_template = self.c_dict['INPUT_TEMPLATE'] self.logger.info('Input directory is {}'.format(input_dir)) self.logger.info('Input template is {}'.format(input_template)) # get forecast leads to loop over lead_seq = util.get_lead_sequence(self.config, input_dict) for lead in lead_seq: # set forecast lead time in hours time_info['lead_hours'] = lead # recalculate time info items time_info = time_util.ti_calculate(time_info) # log init, valid, and forecast lead times for current loop iteration self.logger.info('Processing forecast lead {} initialized at {} and valid at {}' .format(lead, time_info['init'].strftime('%Y-%m-%d %HZ'), time_info['valid'].strftime('%Y-%m-%d %HZ'))) # perform string substitution to find filename based on template and current run time # pass in logger, then template, then any items to use to fill in template # pass time info with ** in front to expand each dictionary item to a variable # i.e. time_info['init'] becomes init=init_value filename = StringSub(self.logger, input_template, **time_info).do_string_sub() self.logger.info('Looking in input directory for file: {}'.format(filename)) return True
def test_ccpa_template(): passed = True valid_string = datetime.datetime.strptime("2019022403", '%Y%m%d%H') lead_string = 10800 logger = logging.getLogger("testing") templ = "ccpa.{valid?fmt=%Y%m%d}/06/ccpa.t{valid?fmt=%H}z.{lead?fmt=%.2H}h.hrap.conus.gb2" expected_filename = "ccpa.20190224/06/ccpa.t03z.03h.hrap.conus.gb2" ss = StringSub(logger, templ, valid=valid_string, lead=lead_string) filename = ss.doStringSub() if filename != expected_filename: passed = False valid_string = datetime.datetime.strptime("2019022406", '%Y%m%d%H') lead_string = int("6") * 3600 expected_filename = "ccpa.20190224/06/ccpa.t06z.06h.hrap.conus.gb2" ss = StringSub(logger, templ, valid=valid_string, lead=lead_string) filename = ss.doStringSub() if filename == expected_filename: passed = False return passed
def run_at_time_once(self, input_dict): self.clear() if self.c_dict['OBS_INPUT_DIR'] == '': self.logger.error('Must set PB2NC_INPUT_DIR in config file') exit(1) if self.c_dict['OBS_INPUT_TEMPLATE'] == '': self.logger.error('Must set PB2NC_INPUT_TEMPLATE in config file') exit(1) if self.c_dict['OUTPUT_DIR'] == '': self.logger.error('Must set PB2NC_OUTPUT_DIR in config file') exit(1) if self.c_dict['OUTPUT_TEMPLATE'] == '': self.logger.error('Must set PB2NC_OUTPUT_TEMPLATE in config file') exit(1) input_dir = self.c_dict['OBS_INPUT_DIR'] input_template = self.c_dict['OBS_INPUT_TEMPLATE'] output_dir = self.c_dict['OUTPUT_DIR'] output_template = self.c_dict['OUTPUT_TEMPLATE'] infile = None # loop over offset list and find first file that matches for offset in self.c_dict['OFFSETS']: input_dict['offset'] = offset time_info = time_util.ti_calculate(input_dict) infile = self.find_obs(time_info, None) if infile is not None: if isinstance(infile, list): for f in infile: self.infiles.append(f) else: self.infiles.append(infile) self.logger.debug('Adding input file {}'.format(infile)) break if infile is None: self.logger.error('Could not find input file in {} matching template {}' .format(input_dir, input_template)) return False outSts = StringSub(self.logger, output_template, **time_info) outfile = outSts.do_string_sub() outfile = os.path.join(output_dir, outfile) self.set_output_path(outfile) # if we don't overwrite and the output file exists, warn and continue if os.path.exists(outfile) and \ self.c_dict['SKIP_IF_OUTPUT_EXISTS'] is True: self.logger.debug('Skip writing output file {} because it already ' 'exists. Remove file or change ' 'PB2NC_SKIP_IF_OUTPUT_EXISTS to False to process' .format(outfile)) return True # set config file since command is reset after each run self.param = self.c_dict['CONFIG_FILE'] # list of fields to print to log print_list = ["PB2NC_MESSAGE_TYPE", "PB2NC_STATION_ID", "OBS_WINDOW_BEGIN", "OBS_WINDOW_END", "PB2NC_GRID", "PB2NC_POLY", "OBS_BUFR_VAR_LIST", "TIME_SUMMARY_FLAG", "TIME_SUMMARY_BEG", "TIME_SUMMARY_END", "TIME_SUMMARY_VAR_NAMES", "TIME_SUMMARY_TYPES" ] # set environment variables needed for MET application self.add_env_var("PB2NC_MESSAGE_TYPE", self.c_dict['MESSAGE_TYPE']) self.add_env_var("PB2NC_STATION_ID", self.c_dict['STATION_ID']) self.add_env_var("OBS_WINDOW_BEGIN", str(self.c_dict['OBS_WINDOW_BEGIN'])) self.add_env_var("OBS_WINDOW_END", str(self.c_dict['OBS_WINDOW_END'])) self.add_env_var("PB2NC_GRID", self.c_dict['GRID']) self.add_env_var("PB2NC_POLY", self.c_dict['POLY']) tmp_message_type = str(self.c_dict['BUFR_VAR_LIST']).replace("\'", "\"") bufr_var_list = ''.join(tmp_message_type.split()) self.add_env_var("OBS_BUFR_VAR_LIST", bufr_var_list) self.add_env_var('TIME_SUMMARY_FLAG', str(self.c_dict['TIME_SUMMARY_FLAG'])) self.add_env_var('TIME_SUMMARY_BEG', self.c_dict['TIME_SUMMARY_BEG']) self.add_env_var('TIME_SUMMARY_END', self.c_dict['TIME_SUMMARY_END']) self.add_env_var('TIME_SUMMARY_VAR_NAMES', str(self.c_dict['TIME_SUMMARY_VAR_NAMES'])) self.add_env_var('TIME_SUMMARY_TYPES', str(self.c_dict['TIME_SUMMARY_TYPES'])) # send environment variables to logger self.logger.debug("ENVIRONMENT FOR NEXT COMMAND: ") self.print_user_env_items() for l in print_list: self.print_env_item(l) self.logger.debug("COPYABLE ENVIRONMENT FOR NEXT COMMAND: ") self.print_env_copy(print_list) cmd = self.get_command() if cmd is None: self.logger.error("Could not generate command") return self.build()
def process_data(self, basin, cyclone, model_list, time_info): """!Find requested files and run tc_pairs Args: @param basin region of storm from config @param cyclone ID number of cyclone from config @param model_list list of models that be available @param time_info object containing timing information to process """ # get bdeck file bdeck_files = [] # set regex expressions for basin and cyclone if wildcard is used # cast cyclone value to integer if it is not a wildcard if cyclone != '*': cyclone_regex = cyclone else: cyclone_regex = "([0-9]{2,4})" if basin != '??': basin_regex = basin else: basin_regex = "([a-zA-Z]{2})" # get search expression for bdeck files to pass to glob string_sub = StringSub(self.logger, self.c_dict['BDECK_TEMPLATE'], basin=basin, cyclone=cyclone, **time_info) bdeck_glob = os.path.join(self.c_dict['BDECK_DIR'], string_sub.do_string_sub()) self.logger.debug('Looking for BDECK: {}'.format(bdeck_glob)) # get all files that match expression bdeck_files = sorted(glob.glob(bdeck_glob)) # if no bdeck_files found if len(bdeck_files) == 0: self.logger.warning('No BDECK files found') return False # find corresponding adeck or edeck files for bdeck_file in bdeck_files: self.logger.debug('Found BDECK: {}'.format(bdeck_file)) # set current basin and cyclone from bdeck file # if basin or cyclone are a wildcard, these will be # replaced by the value pulled from the bdeck file current_basin = basin current_cyclone = cyclone # if wildcard was used in bdeck, pull out what was # substituted for * to find corresponding bdeck file matches = [] if '*' in bdeck_glob or '?' in bdeck_glob: # get regex expression to pull out basin and cyclone string_sub = StringSub(self.logger, self.c_dict['BDECK_TEMPLATE'], basin=basin_regex, cyclone=cyclone_regex, **time_info) bdeck_regex = os.path.join(self.c_dict['BDECK_DIR'], string_sub.do_string_sub()) # capture wildcard values in template bdeck_regex = bdeck_regex.replace('*', '(.*)') bdeck_regex = bdeck_regex.replace('?', '(.)') match = re.match(bdeck_regex, bdeck_file) if match: matches = match.groups() tags = get_tags(self.c_dict['BDECK_TEMPLATE']) match_count = 0 for tag in tags: # if wildcard is set for tag found, get value # if wildcard if found in template, increment index if tag == 'basin' and basin == '??': current_basin = matches[match_count] match_count += 1 elif tag == 'cyclone' and cyclone == '*': current_cyclone = matches[match_count] match_count += 1 elif tag == '*' or tag == '?': match_count += 1 # create lists for deck files, put bdeck in list so it can be handled # the same as a and e for reformatting even though it will always be # size 1 bdeck_list = [bdeck_file] adeck_list = [] edeck_list = [] # get adeck files if self.c_dict['GET_ADECK']: adeck_list = self.find_deck_files('A', current_basin, current_cyclone, model_list, time_info) # get edeck files if self.c_dict['GET_EDECK']: edeck_list = self.find_deck_files('E', current_basin, current_cyclone, model_list, time_info) if not adeck_list and not edeck_list: self.logger.debug('Could not find any corresponding ' 'ADECK or EDECK files') continue # reformat extra tropical cyclone files if necessary if self.c_dict['REFORMAT_DECK']: adeck_list = self.reformat_files(adeck_list, 'A', time_info) bdeck_list = self.reformat_files(bdeck_list, 'B', time_info) edeck_list = self.reformat_files(edeck_list, 'E', time_info) self.adeck = adeck_list self.bdeck = bdeck_list self.edeck = edeck_list if self.c_dict['OUTPUT_TEMPLATE']: # get output filename from template string_sub = StringSub(self.logger, self.c_dict['OUTPUT_TEMPLATE'], basin=current_basin, cyclone=current_cyclone, **time_info) output_file = string_sub.do_string_sub() else: output_file = 'tc_pairs' self.outfile = output_file # build command and run tc_pairs cmd = self.get_command() if cmd is None: self.logger.error("Could not generate command") return output_path = self.get_output_path() + '.tcst' if os.path.isfile( output_path) and self.c_dict['SKIP_OUTPUT'] is True: self.logger.debug('Skip running tc_pairs because '+\ 'output file {} already exists'.format(output_path)+\ 'Change TC_PAIRS_SKIP_IF_OUTPUT_EXISTS to False to '+\ 'overwrite file') else: self.build()