def test_crow_variable_hour():
# Test that StringSub's do_string_sub() correctly creates the valid hour
# without any zero-padding when given the following as input:
# pgbf{lead?fmt=%H}.gfs.{valid?fmt=%Y%M%D%H}
# pgbf([0-9]{1,3}).gfs.(2[0-9]{9})
logger = logging.getLogger("crow_data")
# crow input files with 3, 2, and 1-digit lead times:
crow_input_file_3 = 'pgbf219.gfs.2017060418'
crow_input_file_2 = 'pgbf18.gfs.2017062000'
crow_input_file_1 = 'pgbf3.gfs.2017060418'
lead_1 = int('3') * 3600
lead_2 = int('18') * 3600
lead_3 = int('219') * 3600
valid_2 = datetime.datetime.strptime('2017062000', '%Y%m%d%H')
valid_1 = valid_3 = datetime.datetime.strptime('2017060418', '%Y%m%d%H')
templ = 'pgbf{lead?fmt=%H}.gfs.{valid?fmt=%Y%m%d%H}'
ss_1 = StringSub(logger, templ, valid=valid_1, lead=lead_1)
ss_2 = StringSub(logger, templ, valid=valid_2, lead=lead_2)
ss_3 = StringSub(logger, templ, valid=valid_3, lead=lead_3)
crow_1_output = ss_1.do_string_sub()
crow_2_output = ss_2.do_string_sub()
crow_3_output = ss_3.do_string_sub()
# print("crow_1 output: ", crow_1_output)
# print("crow_2 output: ", crow_2_output)
# print("crow_3 output: ", crow_3_output)
assert(crow_1_output == crow_input_file_1 and
crow_2_output == crow_input_file_2 and
crow_3_output == crow_input_file_3)
def test_shift_time_negative():
init_string = datetime.datetime.strptime("2017060400", '%Y%m%d%H')
logger = logging.getLogger("testing")
templ = "{init?fmt=%Y%m%d%H?shift=-86400}"
expected_filename = "2017060300"
ss = StringSub(logger, templ, init=init_string)
filename = ss.do_string_sub()
assert(filename == expected_filename)
def test_offset_hour():
logger = logging.getLogger("dummy")
expected_hour = "03"
offset = 10800
templ = "{offset?fmt=%2H}"
ss = StringSub(logger, templ, offset=offset)
offset_hour = ss.do_string_sub()
assert (offset_hour == expected_hour)
def test_multiple_valid_substitution_valid():
valid_string = datetime.datetime.strptime("2018020112", '%Y%m%d%H')
lead_string = int("123") * 3600
logger = logging.getLogger("testing")
templ = "{valid?fmt=%Y%m%d%H}/gfs.t{valid?fmt=%H}.pgrb2.0p25.{lead?fmt=%HHH}"
expected_filename = "2018020112/gfs.t12.pgrb2.0p25.123"
ss = StringSub(logger, templ, valid=valid_string, lead=lead_string)
filename = ss.do_string_sub()
assert(filename == expected_filename)
def test_multiple_valid_substitution_init():
init_string = datetime.datetime.strptime("2017060400", '%Y%m%d%H')
lead_string = 0
logger = logging.getLogger("testing")
templ = "{init?fmt=%Y%m%d%H}/gfs.t{init?fmt=%H}z.pgrb2.0p25.f{lead?fmt=%.2H}"
expected_filename = "2017060400/gfs.t00z.pgrb2.0p25.f00"
ss = StringSub(logger, templ, init=init_string, lead=lead_string)
filename = ss.do_string_sub()
assert(filename == expected_filename)
def test_shift_time_lead_negative():
init_string = datetime.datetime.strptime("2019020700", '%Y%m%d%H')
lead_string = int("60") * 3600
logger = logging.getLogger("testing")
templ = "dwd_{init?fmt=%Y%m%d%H}_{lead?fmt=%.3H?shift=-86400}_{lead?fmt=%.3H}"
expected_filename = "dwd_2019020700_036_060"
ss = StringSub(logger, templ, init=init_string, lead=lead_string)
filename = ss.do_string_sub()
assert(filename == expected_filename)
def test_multiple_valid_substitution_init_complex():
init_string = datetime.datetime.strptime("2016061018", '%Y%m%d%H')
lead_string = int("6") * 3600
logger = logging.getLogger("testing")
templ = "ncar.ral.CoSPA.HRRR.{init?fmt=%Y-%m-%dT%H:%M:%S}.PT{lead?fmt=%.2H}:00.nc"
expected_filename = "ncar.ral.CoSPA.HRRR.2016-06-10T18:00:00.PT06:00.nc"
ss = StringSub(logger, templ, init=init_string, lead=lead_string)
filename = ss.do_string_sub()
assert(filename == expected_filename)
def test_cycle_hour():
cycle_string = 0
valid_string = datetime.datetime.strptime("20180103", '%Y%m%d')
logger = logging.getLogger("dummy")
templ = "prefix.{valid?fmt=%Y%m%d}.tm{cycle?fmt=%2H}"
expected_filename = "prefix.20180103.tm00"
ss = StringSub(logger, templ, valid=valid_string, cycle=cycle_string)
filename = ss.do_string_sub()
assert(filename == expected_filename)
def test_h_lead_pad_2_digit_sub():
logger = logging.getLogger("test")
file_template = "{init?fmt=%Y%m%d%H}_A{lead?fmt=%.3H}h"
init_time = datetime.datetime.strptime("1987020103", '%Y%m%d%H')
lead_time = int("3") * 3600
fSts = StringSub(logger,
file_template,
init=init_time,
lead=lead_time)
out_string = fSts.do_string_sub()
assert(out_string == "1987020103_A003h")
def test_gdas_substitution():
# Test that the string template substitution works correctly for GDAS
# prepbufr files, which do not make use of the cycle hour or the offset
# to generate the valid time.
valid_string = "2018010411"
valid_obj = datetime.datetime.strptime(valid_string, '%Y%m%d%H')
logger = logging.getLogger("testing")
templ = "prepbufr.gdas.{valid?fmt=%Y%m%d%H}.nc"
expected_filename = 'prepbufr.gdas.' + valid_string + '.nc'
ss = StringSub(logger, templ, valid=valid_obj)
filename = ss.do_string_sub()
assert(filename == expected_filename)
def test_ccpa_template():
passed = True
valid_string = datetime.datetime.strptime("2019022403", '%Y%m%d%H')
lead_string = 10800
logger = logging.getLogger("testing")
templ = "ccpa.{valid?fmt=%Y%m%d}/06/ccpa.t{valid?fmt=%H}z.{lead?fmt=%.2H}h.hrap.conus.gb2"
expected_filename = "ccpa.20190224/06/ccpa.t03z.03h.hrap.conus.gb2"
ss = StringSub(logger, templ, valid=valid_string, lead=lead_string)
filename = ss.do_string_sub()
if filename != expected_filename:
passed = False
valid_string = datetime.datetime.strptime("2019022406", '%Y%m%d%H')
lead_string = int("6") * 3600
expected_filename = "ccpa.20190224/06/ccpa.t06z.06h.hrap.conus.gb2"
ss = StringSub(logger, templ, valid=valid_string, lead=lead_string)
filename = ss.do_string_sub()
if filename == expected_filename:
passed = False
return passed
def test_ym_date_dir():
# Test that the ym directory can be read in and does substitution correctly
logger = logging.getLogger("test")
# e.g. /d1/METplus_TC/adeck_orig/201708/atcfunix.gfs.2017080100
date_str = '201708'
templ = '/d1/METplus_TC/adeck_orig/{date?fmt=%s}/' \
'atcfunix.gfs.2017080100.dat'
ss = StringSub(logger, templ, date=date_str)
filename = ss.do_string_sub()
expected_filename = '/d1/METplus_TC/adeck_orig/201708/' \
'atcfunix.gfs.2017080100.dat'
assert filename == expected_filename
def test_ymd_date_dir():
# Test that the ymd directory can be read in and does substitution correctly
logger = logging.getLogger("test")
# e.g. /d1/METplus_TC/adeck_orig/20170811/atcfunix.gfs.2017080100
init_str = datetime.datetime.strptime('2017081118', '%Y%m%d%H')
date_str = '20170811'
templ = '/d1/METplus_TC/adeck_orig/{date?fmt=%s}/atcfunix.gfs.' \
'{init?fmt=%Y%m%d%H}.dat'
ss = StringSub(logger, templ, date=date_str, init=init_str)
filename = ss.do_string_sub()
expected_filename = '/d1/METplus_TC/adeck_orig/20170811/' \
'atcfunix.gfs.2017081118.dat'
assert filename == expected_filename
def test_nam_substitution_HH(key, value):
pytest.skip('time offsets no longer computed in StringSub')
# Test that the substitution works correctly when given an init time,
# cycle hour, and negative offset hour.
init_string = datetime.datetime.strptime("20180102", '%Y%m%d')
cycle_string = key
offset_string = 10800 #'03'
expected_filename = value
logger = logging.getLogger("test")
templ = \
'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%HH}z.tm{' \
'offset?fmt=%HH}.nc'
ss = StringSub(logger, templ, init=init_string, cycle=cycle_string,
offset=offset_string)
filename = ss.do_string_sub()
# print('nam filename: ', filename)
assert (filename == expected_filename)
def test_ymd_region_cyclone():
# Test that we can recreate the full file path with a date,
# region, and cyclone
logger = logging.getLogger("test")
# /d1/METplus_TC/bdeck_orig/20170811/bal052017.dat
date_str = '201708'
region_str = 'al'
cyclone_str = '05'
year_str = '2017'
# templ = '/d1/METplus_TC/bdeck/{date?fmt=%Y%m}/bal{region?fmt=%s}.dat'
templ = '/d1/METplus_TC/bdeck/{date?fmt=%s}/b{region?fmt=%s}' \
'{cyclone?fmt=%s}{misc?fmt=%s}.dat'
ss = StringSub(logger, templ, date=date_str, region=region_str,
cyclone=cyclone_str, misc=year_str)
full_file = ss.do_string_sub()
expected_full_file = '/d1/METplus_TC/bdeck/201708/bal052017.dat'
assert full_file == expected_full_file
def find_deck_files(self, deck, basin, cyclone, model_list, time_info):
"""!Find ADECK or EDECK files that correspond to the BDECk file found
Args:
@param deck type of deck (A or E)
@param basin region of storm from config
@param cyclone ID number of cyclone from config
@param model_list list of models that be available
@param time_info object containing timing information to process
"""
deck_list = []
# get matching adeck wildcard expression for first model
string_sub = StringSub(self.logger,
self.c_dict[deck + 'DECK_TEMPLATE'],
basin=basin,
cyclone=cyclone,
model=model_list[0],
**time_info)
deck_expr = os.path.join(self.c_dict[deck + 'DECK_DIR'],
string_sub.do_string_sub())
# add adeck files if they exist
for model in model_list:
deck_glob = deck_expr.replace(model_list[0], model)
self.logger.debug('Looking for {}DECK file: {}'.format(
deck, deck_glob))
deck_files = glob.glob(deck_glob)
if not deck_files:
continue
# there should only be 1 file that matches
deck_file = deck_files[0]
# if deck exists, add to list
if os.path.isfile(deck_file):
self.logger.debug('Adding {}DECK: {}'.format(deck, deck_file))
deck_list.append(deck_file)
return deck_list
def process_data(self, basin, cyclone, model_list, time_info):
"""!Find requested files and run tc_pairs
Args:
@param basin region of storm from config
@param cyclone ID number of cyclone from config
@param model_list list of models that be available
@param time_info object containing timing information to process
"""
# get bdeck file
bdeck_files = []
# set regex expressions for basin and cyclone if wildcard is used
# cast cyclone value to integer if it is not a wildcard
if cyclone != '*':
cyclone_regex = cyclone
else:
cyclone_regex = "([0-9]{2,4})"
if basin != '??':
basin_regex = basin
else:
basin_regex = "([a-zA-Z]{2})"
# get search expression for bdeck files to pass to glob
string_sub = StringSub(self.logger,
self.c_dict['BDECK_TEMPLATE'],
basin=basin,
cyclone=cyclone,
**time_info)
bdeck_glob = os.path.join(self.c_dict['BDECK_DIR'],
string_sub.do_string_sub())
self.logger.debug('Looking for BDECK: {}'.format(bdeck_glob))
# get all files that match expression
bdeck_files = sorted(glob.glob(bdeck_glob))
# if no bdeck_files found
if len(bdeck_files) == 0:
self.logger.warning('No BDECK files found')
return False
# find corresponding adeck or edeck files
for bdeck_file in bdeck_files:
self.logger.debug('Found BDECK: {}'.format(bdeck_file))
# set current basin and cyclone from bdeck file
# if basin or cyclone are a wildcard, these will be
# replaced by the value pulled from the bdeck file
current_basin = basin
current_cyclone = cyclone
# if wildcard was used in bdeck, pull out what was
# substituted for * to find corresponding bdeck file
matches = []
if '*' in bdeck_glob or '?' in bdeck_glob:
# get regex expression to pull out basin and cyclone
string_sub = StringSub(self.logger,
self.c_dict['BDECK_TEMPLATE'],
basin=basin_regex,
cyclone=cyclone_regex,
**time_info)
bdeck_regex = os.path.join(self.c_dict['BDECK_DIR'],
string_sub.do_string_sub())
# capture wildcard values in template
bdeck_regex = bdeck_regex.replace('*', '(.*)')
bdeck_regex = bdeck_regex.replace('?', '(.)')
match = re.match(bdeck_regex, bdeck_file)
if match:
matches = match.groups()
tags = get_tags(self.c_dict['BDECK_TEMPLATE'])
match_count = 0
for tag in tags:
# if wildcard is set for tag found, get value
# if wildcard if found in template, increment index
if tag == 'basin' and basin == '??':
current_basin = matches[match_count]
match_count += 1
elif tag == 'cyclone' and cyclone == '*':
current_cyclone = matches[match_count]
match_count += 1
elif tag == '*' or tag == '?':
match_count += 1
# create lists for deck files, put bdeck in list so it can be handled
# the same as a and e for reformatting even though it will always be
# size 1
bdeck_list = [bdeck_file]
adeck_list = []
edeck_list = []
# get adeck files
if self.c_dict['GET_ADECK']:
adeck_list = self.find_deck_files('A', current_basin,
current_cyclone, model_list,
time_info)
# get edeck files
if self.c_dict['GET_EDECK']:
edeck_list = self.find_deck_files('E', current_basin,
current_cyclone, model_list,
time_info)
if not adeck_list and not edeck_list:
self.logger.debug('Could not find any corresponding '
'ADECK or EDECK files')
continue
# reformat extra tropical cyclone files if necessary
if self.c_dict['REFORMAT_DECK']:
adeck_list = self.reformat_files(adeck_list, 'A', time_info)
bdeck_list = self.reformat_files(bdeck_list, 'B', time_info)
edeck_list = self.reformat_files(edeck_list, 'E', time_info)
self.adeck = adeck_list
self.bdeck = bdeck_list
self.edeck = edeck_list
if self.c_dict['OUTPUT_TEMPLATE']:
# get output filename from template
string_sub = StringSub(self.logger,
self.c_dict['OUTPUT_TEMPLATE'],
basin=current_basin,
cyclone=current_cyclone,
**time_info)
output_file = string_sub.do_string_sub()
else:
output_file = 'tc_pairs'
self.outfile = output_file
# build command and run tc_pairs
cmd = self.get_command()
if cmd is None:
self.logger.error("Could not generate command")
return
output_path = self.get_output_path() + '.tcst'
if os.path.isfile(
output_path) and self.c_dict['SKIP_OUTPUT'] is True:
self.logger.debug('Skip running tc_pairs because '+\
'output file {} already exists'.format(output_path)+\
'Change TC_PAIRS_SKIP_IF_OUTPUT_EXISTS to False to '+\
'overwrite file')
else:
self.build()
def run_at_time_once(self, input_dict):
self.clear()
if self.c_dict['OBS_INPUT_DIR'] == '':
self.logger.error('Must set PB2NC_INPUT_DIR in config file')
exit(1)
if self.c_dict['OBS_INPUT_TEMPLATE'] == '':
self.logger.error('Must set PB2NC_INPUT_TEMPLATE in config file')
exit(1)
if self.c_dict['OUTPUT_DIR'] == '':
self.logger.error('Must set PB2NC_OUTPUT_DIR in config file')
exit(1)
if self.c_dict['OUTPUT_TEMPLATE'] == '':
self.logger.error('Must set PB2NC_OUTPUT_TEMPLATE in config file')
exit(1)
input_dir = self.c_dict['OBS_INPUT_DIR']
input_template = self.c_dict['OBS_INPUT_TEMPLATE']
output_dir = self.c_dict['OUTPUT_DIR']
output_template = self.c_dict['OUTPUT_TEMPLATE']
infile = None
# loop over offset list and find first file that matches
for offset in self.c_dict['OFFSETS']:
input_dict['offset'] = offset
time_info = time_util.ti_calculate(input_dict)
infile = self.find_obs(time_info, None)
if infile is not None:
if isinstance(infile, list):
for f in infile:
self.infiles.append(f)
else:
self.infiles.append(infile)
self.logger.debug('Adding input file {}'.format(infile))
break
if infile is None:
self.logger.error('Could not find input file in {} matching template {}'
.format(input_dir, input_template))
return False
outSts = StringSub(self.logger,
output_template,
**time_info)
outfile = outSts.do_string_sub()
outfile = os.path.join(output_dir, outfile)
self.set_output_path(outfile)
# if we don't overwrite and the output file exists, warn and continue
if os.path.exists(outfile) and \
self.c_dict['SKIP_IF_OUTPUT_EXISTS'] is True:
self.logger.debug('Skip writing output file {} because it already '
'exists. Remove file or change '
'PB2NC_SKIP_IF_OUTPUT_EXISTS to False to process'
.format(outfile))
return True
# set config file since command is reset after each run
self.param = self.c_dict['CONFIG_FILE']
# list of fields to print to log
print_list = ["PB2NC_MESSAGE_TYPE", "PB2NC_STATION_ID",
"OBS_WINDOW_BEGIN", "OBS_WINDOW_END",
"PB2NC_GRID", "PB2NC_POLY", "OBS_BUFR_VAR_LIST",
"TIME_SUMMARY_FLAG", "TIME_SUMMARY_BEG",
"TIME_SUMMARY_END", "TIME_SUMMARY_VAR_NAMES",
"TIME_SUMMARY_TYPES" ]
# set environment variables needed for MET application
self.add_env_var("PB2NC_MESSAGE_TYPE", self.c_dict['MESSAGE_TYPE'])
self.add_env_var("PB2NC_STATION_ID", self.c_dict['STATION_ID'])
self.add_env_var("OBS_WINDOW_BEGIN", str(self.c_dict['OBS_WINDOW_BEGIN']))
self.add_env_var("OBS_WINDOW_END", str(self.c_dict['OBS_WINDOW_END']))
self.add_env_var("PB2NC_GRID", self.c_dict['GRID'])
self.add_env_var("PB2NC_POLY", self.c_dict['POLY'])
tmp_message_type = str(self.c_dict['BUFR_VAR_LIST']).replace("\'", "\"")
bufr_var_list = ''.join(tmp_message_type.split())
self.add_env_var("OBS_BUFR_VAR_LIST", bufr_var_list)
self.add_env_var('TIME_SUMMARY_FLAG',
str(self.c_dict['TIME_SUMMARY_FLAG']))
self.add_env_var('TIME_SUMMARY_BEG',
self.c_dict['TIME_SUMMARY_BEG'])
self.add_env_var('TIME_SUMMARY_END',
self.c_dict['TIME_SUMMARY_END'])
self.add_env_var('TIME_SUMMARY_VAR_NAMES',
str(self.c_dict['TIME_SUMMARY_VAR_NAMES']))
self.add_env_var('TIME_SUMMARY_TYPES',
str(self.c_dict['TIME_SUMMARY_TYPES']))
# send environment variables to logger
self.logger.debug("ENVIRONMENT FOR NEXT COMMAND: ")
self.print_user_env_items()
for l in print_list:
self.print_env_item(l)
self.logger.debug("COPYABLE ENVIRONMENT FOR NEXT COMMAND: ")
self.print_env_copy(print_list)
cmd = self.get_command()
if cmd is None:
self.logger.error("Could not generate command")
return
self.build()
def retrieve_and_regrid(tmp_filename, cur_init, cur_storm, out_dir, config):
"""! Retrieves the data from the MODEL_DATA_DIR (defined in metplus.conf)
that corresponds to the storms defined in the tmp_filename:
1) create the analysis tile and forecast file names from the
tmp_filename file.
2) perform regridding via MET tool (regrid_data_plane) and store
results (netCDF files) in the out_dir or via
Regridding via regrid_data_plane on the forecast and analysis
files via a latlon string with the following format:
latlon Nx Ny lat_ll lon_ll delta_lat delta_lon
NOTE: these values are defined in the extract_tiles_parm
parameter/config file as NLAT, NLON.
***NOTE: This is used by both extract_tiles_wrapper.py and
series_by_lead_wrapper.py
Args:
@param tmp_filename: Filename of the temporary filter file in
the /tmp directory. Contains rows
of data corresponding to a storm id of varying
times.
@param cur_init: The current init time
@param cur_storm: The current storm
@param out_dir: The directory where regridded netCDF or grib2 output
is saved depending on which regridding methodology is
requested. If the MET tool regrid_data_plane is
requested, then netCDF data is produced. If wgrib2
is requested, then grib2 data is produced.
@param config: config instance
Returns:
None
"""
# pylint: disable=protected-access
# Need to call sys._getframe() to get current function and file for
# logging information.
# pylint: disable=too-many-arguments
# all input is needed to perform task
# rdp=, was added when logging capability was added to capture
# all MET output to log files. It is a temporary work around
# to get logging up and running as needed.
# It is being used to call the run_cmd method, which runs the cmd
# and redirects logging based on the conf settings.
# Instantiate a RegridDataPlaneWrapper
logger = config.logger
rdp = RegridDataPlaneWrapper(config, logger)
# For logging
cur_filename = sys._getframe().f_code.co_filename
cur_function = sys._getframe().f_code.co_name
# Get variables, etc. from param/config file.
model_data_dir = config.getdir('MODEL_DATA_DIR')
met_install_dir = config.getdir('MET_INSTALL_DIR')
regrid_data_plane_exe = os.path.join(met_install_dir,
'bin/regrid_data_plane')
# regrid_data_plane_exe = config.getexe('REGRID_DATA_PLANE')
wgrib2_exe = config.getexe('WGRIB2')
egrep_exe = config.getexe('EGREP')
regrid_with_met_tool = config.getbool('config', 'REGRID_USING_MET_TOOL')
overwrite_flag = config.getbool('config', 'OVERWRITE_TRACK')
# Extract the columns of interest: init time, lead time,
# valid time lat and lon of both tropical cyclone tracks, etc.
# Then calculate the forecast hour and other things.
with open(tmp_filename, "r") as tf:
# read header
header = tf.readline().split()
# get column number for columns on interest
# print('header{}:'.format(header))
header_colnum_init, header_colnum_lead, header_colnum_valid = \
header.index('INIT'), header.index('LEAD'), header.index(
'VALID')
header_colnum_alat, header_colnum_alon = \
header.index('ALAT'), header.index('ALON')
header_colnum_blat, header_colnum_blon = \
header.index('BLAT'), header.index('BLON')
for line in tf:
col = line.split()
init, lead, valid, alat, alon, blat, blon = \
col[header_colnum_init], col[header_colnum_lead], \
col[header_colnum_valid], col[header_colnum_alat], \
col[header_colnum_alon], col[header_colnum_blat], \
col[header_colnum_blon]
# integer division for both Python 2 and 3
lead_time = int(lead)
fcst_hr = lead_time // 10000
init_ymd_match = re.match(r'[0-9]{8}', init)
if init_ymd_match:
init_ymd = init_ymd_match.group(0)
else:
logger.WARN("RuntimeError raised")
raise RuntimeError('init time has unexpected format for YMD')
init_ymdh_match = re.match(r'[0-9|_]{11}', init)
if init_ymdh_match:
init_ymdh = init_ymdh_match.group(0)
else:
logger.WARN("RuntimeError raised")
valid_ymd_match = re.match(r'[0-9]{8}', valid)
if valid_ymd_match:
valid_ymd = valid_ymd_match.group(0)
else:
logger.WARN("RuntimeError raised")
valid_ymdh_match = re.match(r'[0-9|_]{11}', valid)
if valid_ymdh_match:
valid_ymdh = valid_ymdh_match.group(0)
else:
logger.WARN("RuntimeError raised")
lead_str = str(fcst_hr).zfill(3)
fcst_dir = os.path.join(model_data_dir, init_ymd)
init_ymdh_split = init_ymdh.split("_")
init_yyyymmddhh = "".join(init_ymdh_split)
anly_dir = os.path.join(model_data_dir, valid_ymd)
valid_ymdh_split = valid_ymdh.split("_")
valid_yyyymmddhh = "".join(valid_ymdh_split)
init_dt = datetime.datetime.strptime(init_yyyymmddhh, '%Y%m%d%H')
valid_dt = datetime.datetime.strptime(valid_yyyymmddhh, '%Y%m%d%H')
lead_seconds = int(fcst_hr * 3600)
# Create output filenames for regridding
# wgrib2 used to regrid.
# Create the filename for the regridded file, which is a
# grib2 file.
fcst_sts = \
StringSub(logger, config.getraw('filename_templates',
'GFS_FCST_FILE_TMPL'),
init=init_dt, lead=lead_seconds)
anly_sts = \
StringSub(logger, config.getraw('filename_templates',
'GFS_ANLY_FILE_TMPL'),
valid=valid_dt, lead=lead_seconds)
fcst_file = fcst_sts.do_string_sub()
fcst_filename = os.path.join(fcst_dir, fcst_file)
anly_file = anly_sts.do_string_sub()
anly_filename = os.path.join(anly_dir, anly_file)
# Check if the forecast input file exists. If it doesn't
# exist, just log it
if util.file_exists(fcst_filename):
logger.debug("Forecast file: {}".format(fcst_filename))
else:
logger.warning("Can't find forecast file {}, continuing"\
.format(fcst_filename))
continue
# Check if the analysis input file exists. If it doesn't
# exist, just log it.
if util.file_exists(anly_filename):
logger.debug("Analysis file: {}".format(anly_filename))
else:
logger.warning("Can't find analysis file {}, continuing"\
.format(anly_filename))
continue
# Create the arguments used to perform regridding.
# NOTE: the base name
# is the same for both the fcst and anly filenames,
# so use either one to derive the base name that will
# be used to create the fcst_regridded_filename and
# anly_regridded_filename.
fcst_anly_base = os.path.basename(fcst_filename)
fcst_grid_spec = \
util.create_grid_specification_string(alat, alon,
logger,
config)
anly_grid_spec = \
util.create_grid_specification_string(blat, blon,
logger,
config)
if regrid_with_met_tool:
nc_fcst_anly_base = re.sub("grb2", "nc", fcst_anly_base)
fcst_anly_base = nc_fcst_anly_base
tile_dir = os.path.join(out_dir, cur_init, cur_storm)
fcst_hr_str = str(fcst_hr).zfill(3)
fcst_regridded_filename = \
config.getstr('regex_pattern', 'FCST_TILE_PREFIX') + \
fcst_hr_str + "_" + fcst_anly_base
fcst_regridded_file = os.path.join(tile_dir,
fcst_regridded_filename)
anly_regridded_filename = \
config.getstr('regex_pattern', 'ANLY_TILE_PREFIX') + \
fcst_hr_str + "_" + fcst_anly_base
anly_regridded_file = os.path.join(tile_dir,
anly_regridded_filename)
# Regrid the fcst file only if a fcst tile
# file does NOT already exist or if the overwrite flag is True.
# Create new gridded file for fcst tile
if util.file_exists(fcst_regridded_file) and not overwrite_flag:
msg = "Forecast tile file {} exists, skip regridding"\
.format(fcst_regridded_file)
logger.debug(msg)
else:
# Perform fcst regridding on the records of interest
var_level_string = retrieve_var_info(config)
if regrid_with_met_tool:
# Perform regridding using MET Tool regrid_data_plane
fcst_cmd_list = [
regrid_data_plane_exe, ' ', fcst_filename, ' ',
fcst_grid_spec, ' ', fcst_regridded_file, ' ',
var_level_string, ' -method NEAREST '
]
regrid_cmd_fcst = ''.join(fcst_cmd_list)
# Since not using the CommandBuilder to build the cmd,
# add the met verbosity level to the
# MET cmd created before we run the command.
regrid_cmd_fcst = rdp.cmdrunner.insert_metverbosity_opt(
regrid_cmd_fcst)
(ret, regrid_cmd_fcst) = rdp.cmdrunner.run_cmd(
regrid_cmd_fcst, env=None, app_name=rdp.app_name)
else:
# Perform regridding via wgrib2
requested_records = retrieve_var_info(config)
fcst_cmd_list = [
wgrib2_exe, ' ', fcst_filename, ' | ', egrep_exe, ' ',
requested_records, '|', wgrib2_exe, ' -i ',
fcst_filename, ' -new_grid ', fcst_grid_spec, ' ',
fcst_regridded_file
]
wgrb_cmd_fcst = ''.join(fcst_cmd_list)
(ret,
wgrb_cmd_fcst) = rdp.cmdrunner.run_cmd(wgrb_cmd_fcst,
env=None,
ismetcmd=False)
# Create new gridded file for anly tile
if util.file_exists(anly_regridded_file) and not overwrite_flag:
logger.debug("Analysis tile file: " + anly_regridded_file +
" exists, skip regridding")
else:
# Perform anly regridding on the records of interest
var_level_string = retrieve_var_info(config)
if regrid_with_met_tool:
anly_cmd_list = [
regrid_data_plane_exe, ' ', anly_filename, ' ',
anly_grid_spec, ' ', anly_regridded_file, ' ',
var_level_string, ' ', ' -method NEAREST '
]
regrid_cmd_anly = ''.join(anly_cmd_list)
# Since not using the CommandBuilder to build the cmd,
# add the met verbosity level to the MET cmd
# created before we run the command.
regrid_cmd_anly = rdp.cmdrunner.insert_metverbosity_opt(
regrid_cmd_anly)
(ret, regrid_cmd_anly) = rdp.cmdrunner.run_cmd(
regrid_cmd_anly, env=None, app_name=rdp.app_name)
msg = ("on anly file:" + anly_regridded_file)
logger.debug(msg)
else:
# Regridding via wgrib2.
requested_records = util.retrieve_var_info(config)
anly_cmd_list = [
wgrib2_exe, ' ', anly_filename, ' | ', egrep_exe, ' ',
requested_records, '|', wgrib2_exe, ' -i ',
anly_filename, ' -new_grid ', anly_grid_spec, ' ',
anly_regridded_file
]
wgrb_cmd_anly = ''.join(anly_cmd_list)
(ret,
wgrb_cmd_anly) = rdp.cmdrunner.run_cmd(wgrb_cmd_anly,
env=None,
ismetcmd=False)