def parmRead(fname):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "LongRangeRegridDriver"
WhfLog.init(parser, forcing_config_label, 'Long', 'Regrid', 'CFS')
cfsDir = parser.get('data_dir', 'CFS_data')
cfsNumEnsemble = int(parser.get('data_dir', 'CFS_num_ensemble'))
maxFcstHourCfs = int(parser.get('fcsthr_max', 'CFS_fcsthr_max'))
hoursBackCfs = int(parser.get('triggering', 'CFS_hours_back'))
stateFile = parser.get('triggering', 'long_range_regrid_state_file')
parms = Parms(cfsDir, cfsNumEnsemble, maxFcstHourCfs, hoursBackCfs,
stateFile)
return parms
def parmRead(fname, fileType, realtime):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
fileType: str
'RAP', 'HRRR', 'MRMS', 'GFS'
realtime: boolean
True if realtime, False for archive mode
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
label = 'Regrid' + fileType
if (realtime):
WhfLog.init(parser, label, False)
else:
WhfLog.set(label)
dataDir = parser.get('data_dir', fileType + '_data')
maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max'))
hoursBack = int(parser.get('triggering', fileType + '_hours_back'))
stateFile = parser.get('triggering', fileType + '_regrid_state_file')
parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile)
return parms
def parmRead(fname, realtime):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
if (realtime):
WhfLog.init(parser, 'LongRegrid', False)
else:
WhfLog.set('LongRegrid')
cfsDir = parser.get('data_dir', 'CFS_data')
cfsNumEnsemble = int(parser.get('data_dir', 'CFS_num_ensemble'))
maxFcstHourCfs = int(parser.get('fcsthr_max', 'CFS_fcsthr_max'))
hoursBackCfs = int(parser.get('triggering', 'CFS_hours_back'))
stateFile = parser.get('triggering', 'long_range_regrid_state_file')
parms = Parms(cfsDir, cfsNumEnsemble, maxFcstHourCfs, hoursBackCfs,
stateFile)
return parms
def parmRead(fname, fileType, realtime):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
fileType: str
'RAP', 'HRRR', 'MRMS', 'GFS'
realtime: boolean
True if realtime, False for archive mode
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
label = 'Regrid' + fileType
if (realtime):
WhfLog.init(parser, label, False)
else:
WhfLog.set(label)
dataDir = parser.get('data_dir', fileType + '_data')
maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max'))
hoursBack = int(parser.get('triggering', fileType + '_hours_back'))
stateFile = parser.get('triggering', fileType + '_regrid_state_file')
parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile)
return parms
def process(configFile, parm):
'''Moves all the files from the source directory to the destination directory.
Args:
fileType (string) :
source_dir (string): Full path to the source directory
destination_dir (string): Full path to the destination directory
extension (string): File extension of files to be moved
Returns:
None
'''
# initialize the logging first
parser = SafeConfigParser()
parser.read(configFile)
WhfLog.init(parser, 'RegridMRMS', True) # just use any valid 2nd arg
for day in parm._days:
processDay(configFile, day, parm)
def process(configFile, parm):
'''Moves all the files from the source directory to the destination directory.
Args:
fileType (string) :
source_dir (string): Full path to the source directory
destination_dir (string): Full path to the destination directory
extension (string): File extension of files to be moved
Returns:
None
'''
# initialize the logging first
parser = SafeConfigParser()
parser.read(configFile)
WhfLog.init(parser, 'RegridMRMS', True) # just use any valid 2nd arg
for day in parm._days:
processDay(configFile, day, parm)
def parmRead(fname):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "AnalysisAssimLayeringDriver"
WhfLog.init(parser, forcing_config_label, 'AA', 'Layer', 'RAP/HRRR/MRMS')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
hrrr0hrDir = parser.get('downscaling',
'HRRR_finished_output_dir_0hr') # placeholder
mrmsDir = parser.get('regridding',
'MRMS_finished_output_dir') # maybe incorrect
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
rap0hrDir = parser.get('downscaling',
'RAP_finished_output_dir_0hr') # placeholder
layerDir = parser.get('layering', 'analysis_assimilation_output')
maxFcstHour = 3
hoursBack = 5 # 3 hours back at 0, -1, and -2 (-2 -3 = -5)
maxWaitMinutes = 15 #int(parser.get('triggering',
# 'short_range_fcst_max_wait_minutes'))
veryLateMinutes = 20 #int(parser.get('triggering',
# 'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'analysis_assim_layering_state_file')
parms = Parms(hrrrDir, hrrr0hrDir, rapDir, rap0hrDir, mrmsDir, layerDir,
maxFcstHour, hoursBack, maxWaitMinutes, veryLateMinutes,
stateFile)
return parms
def parmRead(fname, fileType):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
fileType: str
'RAP', 'HRRR', 'MRMS', 'GFS'
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "Regrid" + fileType
if (fileType == 'HRRR'):
configType = 'Short'
elif (fileType == 'RAP'):
configType = 'Short'
elif (fileType == 'GFS'):
configType = 'Medium'
elif (fileType == 'MRMS'):
configType = 'AA'
else:
configType = '???'
WhfLog.init(parser, forcing_config_label, configType, 'Regrid', fileType)
dataDir = parser.get('data_dir', fileType + '_data')
maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max'))
hoursBack = int(parser.get('triggering', fileType + '_hours_back'))
stateFile = parser.get('triggering', fileType + '_regrid_state_file')
parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile)
return parms
def parmRead(fname, fileType):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
fileType: str
'RAP', 'HRRR', 'MRMS', 'GFS'
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "Regrid" + fileType
if (fileType == 'HRRR'):
configType = 'Short'
elif (fileType == 'RAP'):
configType = 'Short'
elif (fileType == 'GFS'):
configType = 'Medium'
elif (fileType == 'MRMS'):
configType = 'AA'
else:
configType = '???'
WhfLog.init(parser, forcing_config_label, configType, 'Regrid', fileType)
dataDir = parser.get('data_dir', fileType + '_data')
maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max'))
hoursBack = int(parser.get('triggering', fileType + '_hours_back'))
stateFile = parser.get('triggering', fileType + '_regrid_state_file')
parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile)
return parms
def parmRead(fname):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "AnalysisAssimLayeringDriver"
WhfLog.init(parser, forcing_config_label, 'AA', 'Layer', 'RAP/HRRR/MRMS')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
hrrr0hrDir = parser.get('downscaling', 'HRRR_finished_output_dir_0hr') # placeholder
mrmsDir = parser.get('regridding', 'MRMS_finished_output_dir') # maybe incorrect
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
rap0hrDir = parser.get('downscaling', 'RAP_finished_output_dir_0hr') # placeholder
layerDir = parser.get('layering', 'analysis_assimilation_output')
maxFcstHour = 3
hoursBack = 5 # 3 hours back at 0, -1, and -2 (-2 -3 = -5)
maxWaitMinutes=15 #int(parser.get('triggering',
# 'short_range_fcst_max_wait_minutes'))
veryLateMinutes=20 #int(parser.get('triggering',
# 'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'analysis_assim_layering_state_file')
parms = Parms(hrrrDir, hrrr0hrDir, rapDir, rap0hrDir, mrmsDir, layerDir,
maxFcstHour, hoursBack, maxWaitMinutes, veryLateMinutes, stateFile)
return parms
def parmRead(fname, realtime):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
realtime: boolean
True if realtime, False for archive mode
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
if (realtime):
WhfLog.init(parser, 'ShortLayer', False)
else:
WhfLog.set('ShortLayer')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
layerDir = parser.get('layering', 'short_range_output')
maxFcstHour = int(parser.get('fcsthr_max', 'HRRR_fcsthr_max'))
# go with HRRR
hoursBack = int(parser.get('triggering', 'HRRR_hours_back'))
maxWaitMinutes = int(
parser.get('triggering', 'short_range_fcst_max_wait_minutes'))
veryLateMinutes = int(
parser.get('triggering', 'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'short_range_layering_state_file')
parms = Parms(hrrrDir, rapDir, layerDir, maxFcstHour, hoursBack,
maxWaitMinutes, veryLateMinutes, stateFile)
return parms
def parmRead(fname, realtime):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
realtime: boolean
True if realtime, False for archive mode
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
if (realtime):
WhfLog.init(parser, 'ShortLayer', False)
else:
WhfLog.set('ShortLayer')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
layerDir = parser.get('layering', 'short_range_output')
maxFcstHour = int(parser.get('fcsthr_max', 'HRRR_fcsthr_max'))
# go with HRRR
hoursBack = int(parser.get('triggering', 'HRRR_hours_back'))
maxWaitMinutes=int(parser.get('triggering',
'short_range_fcst_max_wait_minutes'))
veryLateMinutes=int(parser.get('triggering',
'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'short_range_layering_state_file')
parms = Parms(hrrrDir, rapDir, layerDir, maxFcstHour, hoursBack,
maxWaitMinutes, veryLateMinutes, stateFile)
return parms
def parmRead(fname):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "ShortRangeLayeringDriver"
WhfLog.init(parser, forcing_config_label, 'Short', 'Layer', 'RAP/HRRR')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
#mrmsDir = parser.get('data_dir', 'MRMS_data')
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
layerDir = parser.get('layering', 'short_range_output')
maxFcstHour = int(parser.get('fcsthr_max', 'HRRR_fcsthr_max'))
# go with HRRR
hoursBack = int(parser.get('triggering', 'HRRR_hours_back'))
maxWaitMinutes=int(parser.get('triggering',
'short_range_fcst_max_wait_minutes'))
veryLateMinutes=int(parser.get('triggering',
'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'short_range_layering_state_file')
parms = Parms(hrrrDir, rapDir, layerDir, maxFcstHour, hoursBack,
maxWaitMinutes, veryLateMinutes, stateFile)
return parms
def main():
"""Tests the regridding and downscaling of RAP and HRRR
data for the Short Range Forcing Configuration.
"""
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# CHANGE THIS TO REFLECT WHICH RUN ENVIRONMENT:
# YELLOWSTONE OR HYDRO-C!
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Set flag for testing host
#is_yellowstone = True
is_yellowstone = False
parser = SafeConfigParser()
config_file = "../../parm/wrf_hydro_forcing.parm"
parser.read(config_file)
# Set up logger
wlog.init(parser, "testAA", "AA","Regrid","MRMS")
# Start and end dates
if is_yellowstone:
start_dt = datetime.datetime.strptime("20151004","%Y%m%d")
end_dt = datetime.datetime.strptime("20151005","%Y%m%d")
else:
start_dt = datetime.datetime.strptime("20160202","%Y%m%d")
end_dt = datetime.datetime.strptime("20160204","%Y%m%d")
# Set the directory where the input data resides.
# For running on yellowstone:
# RAP_dir_base = "/glade/scratch/lpan/IOC/data/RAP"
# HRRR_dir_base = "/glade/scratch/lpan/IOC/data/HRRR"
# MRMS_dir_base = "/glade/scratch/lpan/IOC/data/MRMS"
# For running on hydro-c1:
# RAP_downscale_dir =
# "/glade/scratch/gochis/IOC_evaluation_datasets/
# Forcing_Engine/workspace/downscaled/RAP"
# HRRR_downscale_dir = "/glade/scratch/gochis/
# IOC_evaluation_datasets/Forcing_Engine/workspace/downscaled/HRRR"
RAP_dir_base = parser.get('data_dir','RAP_data')
HRRR_dir_base = parser.get('data_dir', 'HRRR_data')
MRMS_dir_base = parser.get('data_dir', 'MRMS_data')
RAP_downscale_dir = parser.get('downscaling', 'RAP_finished_output_dir')
RAP_0hr_downscale_dir = parser.get('downscaling', 'RAP_finished_output_dir_0hr')
HRRR_downscale_dir = parser.get('downscaling', 'HRRR_finished_output_dir')
HRRR_0hr_downscale_dir = parser.get('downscaling', 'HRRR_finished_output_dir_0hr')
MRMS_downscale_dir = parser.get('regridding','MRMS_finished_output_dir')
all_RAP_files_with_path = whf.get_filepaths(RAP_dir_base)
all_HRRR_files_with_path = whf.get_filepaths(HRRR_dir_base)
all_MRMS_files_with_path = whf.get_filepaths(MRMS_dir_base)
# We are only interested in the MRMS, RAP and HRRR files that are
# within the start and end forecast times, since the /glade/scratch/lpan/IOC/data
# directory is continually adding more dates.
RAP_files_with_path = [x for x in all_RAP_files_with_path if is_within_time_range(start_dt,end_dt,x,"RAP",is_yellowstone)]
HRRR_files_with_path = [x for x in all_HRRR_files_with_path if is_within_time_range(start_dt,end_dt,x,"HRRR",is_yellowstone)]
#
MRMS_files_with_path = [x for x in all_MRMS_files_with_path if is_within_time_range(start_dt,end_dt,x,"MRMS",is_yellowstone)]
#do_regrid(config_file, RAP_dir_base,'RAP', RAP_files_with_path, is_yellowstone)
#do_regrid(config_file, HRRR_dir_base, 'HRRR', HRRR_files_with_path, is_yellowstone)
do_regrid(config_file,MRMS_dir_base, 'MRMS', MRMS_files_with_path, is_yellowstone)
def main():
"""Tests the regridding and downscaling of RAP and HRRR
data for the Short Range Forcing Configuration.
"""
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# CHANGE THIS TO REFLECT WHICH RUN ENVIRONMENT:
# YELLOWSTONE OR HYDRO-C!
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Set flag for testing host
#is_yellowstone = True
is_yellowstone = False
parser = SafeConfigParser()
config_file = "../../parm/b_wrf_hydro_forcing.parm"
try:
parser.read(config_file)
except:
print "d'oh!"
# Set up logger
#wlog.init(parser, "testShort", "Short","Regrid","HRRR")
# Start and end dates
if is_yellowstone:
start_dt = datetime.datetime.strptime("20150930","%Y%m%d")
end_dt = datetime.datetime.strptime("20151001","%Y%m%d")
else:
start_dt = datetime.datetime.strptime("20160215","%Y%m%d")
end_dt = datetime.datetime.strptime("20160216","%Y%m%d")
# Set the directory where the input data resides.
# For running on yellowstone:
# RAP_dir_base = "/glade/scratch/lpan/IOC/data/RAP"
# HRRR_dir_base = "/glade/scratch/lpan/IOC/data/HRRR"
# For running on hydro-c1:
# RAP_downscale_dir =
# "/glade/scratch/gochis/IOC_evaluation_datasets/
# Forcing_Engine/workspace/downscaled/RAP"
# HRRR_downscale_dir = "/glade/scratch/gochis/
# IOC_evaluation_datasets/Forcing_Engine/workspace/downscaled/HRRR"
RAP_dir_base = parser.get('data_dir','RAP_data')
HRRR_dir_base = parser.get('data_dir', 'HRRR_data')
RAP_downscale_dir = parser.get('downscaling', 'RAP_downscale_output_dir')
HRRR_downscale_dir = parser.get('downscaling', 'HRRR_downscale_output_dir')
all_RAP_files_with_path = whf.get_filepaths(RAP_dir_base)
all_HRRR_files_with_path = whf.get_filepaths(HRRR_dir_base)
# We are only interested in the RAP and HRRR files that are
# within the start and end forecast times.
HRRR_files_with_path = [x for x in all_HRRR_files_with_path if is_within_time_range(start_dt,end_dt,x,"HRRR",is_yellowstone)]
RAP_files_with_path = [x for x in all_RAP_files_with_path if is_within_time_range(start_dt,end_dt,x,"RAP",is_yellowstone)]
#for hrrr in HRRR_files_with_path:
# print ("process %s")%(hrrr)
# do the processing on only the input grib files
wlog.init(parser, "testShort", "Short","Regrid","RAP")
do_regrid(config_file,RAP_dir_base,'RAP', RAP_files_with_path, is_yellowstone)
wlog.init(parser, "testShort", "Short","Regrid","HRRR")
do_regrid(HRRR_dir_base, 'HRRR', HRRR_files_with_path, is_yellowstone)
