def unzip_file(GzFileIn, FileOut, ConfigOptions, MpiConfig):
"""
Generic I/O function to unzip a .gz file to a new location.
:param GzFileIn:
:param FileOut:
:param ConfigOptions:
:param MpiConfig:
:return:
"""
# Ensure all processors are synced up before outputting.
#MpiConfig.comm.barrier()
if MpiConfig.rank == 0:
# Unzip the file in place.
try:
ConfigOptions.statusMsg = "Unzipping file: {}".format(GzFileIn)
err_handler.log_msg(ConfigOptions, MpiConfig)
with gzip.open(GzFileIn, 'rb') as fTmpGz:
with open(FileOut, 'wb') as fTmp:
shutil.copyfileobj(fTmpGz, fTmp)
except:
ConfigOptions.errMsg = "Unable to unzip: " + GzFileIn
err_handler.log_critical(ConfigOptions, MpiConfig)
return
if not os.path.isfile(FileOut):
ConfigOptions.errMsg = "Unable to locate expected unzipped file: " + \
FileOut
err_handler.log_critical(ConfigOptions, MpiConfig)
return
else:
return
def ncar_topo_adj(input_forcings, ConfigOptions, GeoMetaWrfHydro, MpiConfig):
"""
Topographic adjustment of incoming shortwave radiation fluxes,
given input parameters.
:param input_forcings:
:param ConfigOptions:
:return:
"""
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Performing topographic adjustment to incoming " \
"shortwave radiation flux."
err_handler.log_msg(ConfigOptions, MpiConfig)
# Establish where we have missing values.
try:
indNdv = np.where(
input_forcings.final_forcings == ConfigOptions.globalNdv)
except:
ConfigOptions.errMsg = "Unable to perform NDV search on input forcings"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
# By the time this function has been called, necessary input static grids (height, slope, etc),
# should have been calculated for each local slab of data.
DEGRAD = math.pi / 180.0
DPD = 360.0 / 365.0
try:
DECLIN, SOLCON = radconst(ConfigOptions)
except:
ConfigOptions.errMsg = "Unable to calculate solar constants based on datetime information."
err_handler.log_critical(ConfigOptions, MpiConfig)
return
try:
coszen_loc, hrang_loc = calc_coszen(ConfigOptions, DECLIN,
GeoMetaWrfHydro)
except:
ConfigOptions.errMsg = "Unable to calculate COSZEN or HRANG variables for topographic adjustment " \
"of incoming shortwave radiation"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
try:
TOPO_RAD_ADJ_DRVR(GeoMetaWrfHydro, input_forcings, coszen_loc, DECLIN,
SOLCON, hrang_loc)
except:
ConfigOptions.errMsg = "Unable to perform final topographic adjustment of incoming " \
"shortwave radiation fluxes."
err_handler.log_critical(ConfigOptions, MpiConfig)
return
# Assign missing values based on our mask.
input_forcings.final_forcings[indNdv] = ConfigOptions.globalNdv
# Reset variables to free up memory
DECLIN = None
SOLCON = None
coszen_loc = None
hrang_loc = None
indNdv = None
def weighted_average(input_forcings, ConfigOptions, MpiConfig):
"""
Function for setting the current output regridded fields as a weighted
average between the previous output step and the next output step.
:param input_forcings:
:param ConfigOptions:
:param MpiConfig:
:return:
"""
# Check to make sure we have valid grids.
if input_forcings.regridded_forcings2 is None:
input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv
return
if input_forcings.regridded_forcings1 is None:
input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv
return
# If we are running CFSv2 with bias correction, bypass as temporal interpolation is done
# internally (NWM-only).
if ConfigOptions.runCfsNldasBiasCorrect and input_forcings.productName == "CFSv2_6Hr_Global_GRIB2":
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Bypassing temporal interpolation routine due to NWM bias correction for CFSv2"
err_handler.log_msg(ConfigOptions, MpiConfig)
return
# Calculate the difference between the current output timestep,
# and the previous input forecast output step. Use this to calculate a fraction
# of the previous forcing output to use in the final output for this step.
dtFromPrevious = ConfigOptions.current_output_date - input_forcings.fcst_date1
weight1 = 1 - (abs(dtFromPrevious.total_seconds()) /
(input_forcings.outFreq * 60.0))
# Calculate the difference between the current output timesetp,
# and the next forecast output step. Use this to calculate a fraction of
# the next forcing output to use in the final output for this step.
dtFromNext = ConfigOptions.current_output_date - input_forcings.fcst_date2
weight2 = 1 - (abs(dtFromNext.total_seconds()) /
(input_forcings.outFreq * 60.0))
# Calculate where we have missing data in either the previous or next forcing dataset.
ind1Ndv = np.where(
input_forcings.regridded_forcings1 == ConfigOptions.globalNdv)
ind2Ndv = np.where(
input_forcings.regridded_forcings2 == ConfigOptions.globalNdv)
input_forcings.final_forcings[:,:,:] = input_forcings.regridded_forcings1[:,:,:]*weight1 + \
input_forcings.regridded_forcings2[:,:,:]*weight2
# Set any pixel cells that were missing for either window to missing value.
input_forcings.final_forcings[ind1Ndv] = ConfigOptions.globalNdv
input_forcings.final_forcings[ind2Ndv] = ConfigOptions.globalNdv
# Reset for memory efficiency.
ind1Ndv = None
ind2Ndv = None
def simple_lapse(input_forcings, ConfigOptions, GeoMetaWrfHydro, MpiConfig):
"""
Function that applies a single lapse rate adjustment to modeled
2-meter temperature by taking the difference of the native
input elevation and the WRF-hydro elevation.
:param inpute_forcings:
:param ConfigOptions:
:param GeoMetaWrfHydro:
:return:
"""
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Applying simple lapse rate to temperature downscaling"
err_handler.log_msg(ConfigOptions, MpiConfig)
# Calculate the elevation difference.
if input_forcings.height is None:
ConfigOptions.errMsg = "Unable to perform downscaling without terrain height input"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
elevDiff = input_forcings.height - GeoMetaWrfHydro.height
# Assign existing, un-downscaled temperatures to a temporary placeholder, which
# will be used for specific humidity downscaling.
if input_forcings.q2dDownscaleOpt > 0:
input_forcings.t2dTmp[:, :] = input_forcings.final_forcings[4, :, :]
# Apply single lapse rate value to the input 2-meter
# temperature values.
try:
indNdv = np.where(
input_forcings.final_forcings == ConfigOptions.globalNdv)
except:
ConfigOptions.errMsg = "Unable to perform NDV search on input forcings"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
try:
input_forcings.final_forcings[4,:,:] = input_forcings.final_forcings[4,:,:] + \
(6.49/1000.0)*elevDiff
except:
ConfigOptions.errMsg = "Unable to apply lapse rate to input 2-meter temperatures."
err_handler.log_critical(ConfigOptions, MpiConfig)
return
input_forcings.final_forcings[indNdv] = ConfigOptions.globalNdv
# Reset for memory efficiency
indNdv = None
def q2_down_classic(input_forcings, ConfigOptions, GeoMetaWrfHydro, MpiConfig):
"""
NCAR function for downscaling 2-meter specific humidity using already downscaled
2-meter temperature, unadjusted surface pressure, and downscaled surface
pressure.
:param input_forcings:
:param ConfigOptions:
:param GeoMetaWrfHydro:
:return:
"""
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Performing topographic adjustment to specific humidity."
err_handler.log_msg(ConfigOptions, MpiConfig)
# Establish where we have missing values.
try:
indNdv = np.where(
input_forcings.final_forcings == ConfigOptions.globalNdv)
except:
ConfigOptions.errMsg = "Unable to perform NDV search on input forcings"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
# First calculate relative humidity given original surface pressure and 2-meter
# temperature
try:
relHum = rel_hum(input_forcings, ConfigOptions)
except:
ConfigOptions.errMsg = "Unable to perform topographic downscaling of incoming " \
"specific humidity to relative humidity"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
# Downscale 2-meter specific humidity
try:
q2Tmp = mixhum_ptrh(input_forcings, relHum, 2, ConfigOptions)
except:
ConfigOptions.errMsg = "Unable to perform topographic downscaling of " \
"incoming specific humidity"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
input_forcings.final_forcings[5, :, :] = q2Tmp
input_forcings.final_forcings[indNdv] = ConfigOptions.globalNdv
q2Tmp = None
indNdv = None
def pressure_down_classic(input_forcings, ConfigOptions, GeoMetaWrfHydro,
MpiConfig):
"""
Generic function to downscale surface pressure to the WRF-Hydro domain.
:param input_forcings:
:param ConfigOptions:
:param GeoMetaWrfHydro:
:return:
"""
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Performing topographic adjustment to surface pressure."
err_handler.log_msg(ConfigOptions, MpiConfig)
# Calculate the elevation difference.
if input_forcings.height is None:
ConfigOptions.errMsg = "Unable to perform downscaling without terrain height input"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
elevDiff = input_forcings.height - GeoMetaWrfHydro.height
# Assign existing, un-downscaled pressure values to a temporary placeholder, which
# will be used for specific humidity downscaling.
if input_forcings.q2dDownscaleOpt > 0:
input_forcings.psfcTmp[:, :] = input_forcings.final_forcings[6, :, :]
try:
indNdv = np.where(
input_forcings.final_forcings == ConfigOptions.globalNdv)
except:
ConfigOptions.errMsg = "Unable to perform NDV search on input forcings"
err_handler.log_critical(ConfigOptions, MpiConfig)
return
try:
input_forcings.final_forcings[6,:,:] = input_forcings.final_forcings[6,:,:] +\
(input_forcings.final_forcings[6,:,:]*elevDiff*9.8)/\
(input_forcings.final_forcings[4,:,:]*287.05)
except:
ConfigOptions.errMsg = "Unable to downscale surface pressure to input forcings."
err_handler.log_critical(ConfigOptions, MpiConfig)
return
input_forcings.final_forcings[indNdv] = ConfigOptions.globalNdv
# Reset for memory efficiency
indNdv = None
def nearest_neighbor(input_forcings, ConfigOptions, MpiConfig):
"""
Function for setting the current output regridded forcings to the nearest
input forecast step.
:param input_forcings:
:param ConfigOptions:
:param MpiConfig:
:return:
"""
# If we are running CFSv2 with bias correction, bypass as temporal interpolation is done
# internally (NWM-only).
if ConfigOptions.runCfsNldasBiasCorrect and input_forcings.productName == "CFSv2_6Hr_Global_GRIB2":
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Bypassing temporal interpolation routine due to NWM bias correction for CFSv2"
err_handler.log_msg(ConfigOptions, MpiConfig)
return
# Calculate the difference between the current output timestep,
# and the previous input forecast output step.
dtFromPrevious = ConfigOptions.current_output_date - input_forcings.fcst_date1
# Calculate the difference between the current output timesetp,
# and the next forecast output step.
dtFromNext = ConfigOptions.current_output_date - input_forcings.fcst_date2
if abs(dtFromNext.total_seconds()) <= abs(dtFromPrevious.total_seconds()):
# Default to the regridded states from the next forecast output step.
if input_forcings.regridded_forcings2 is None:
input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv
else:
input_forcings.final_forcings[:, :, :] = input_forcings.regridded_forcings2[:, :, :]
else:
# Default to the regridded states from the previous forecast output
# step.
if input_forcings.regridded_forcings1 is None:
input_forcings.final_forcings[:, :, :] = ConfigOptions.globalNdv
else:
input_forcings.final_forcings[:, :, :] = input_forcings.regridded_forcings1[:, :, :]
def process_forecasts(ConfigOptions, wrfHydroGeoMeta, inputForcingMod, suppPcpMod, MpiConfig, OutputObj):
"""
Main calling module for running realtime forecasts and re-forecasts.
:param jobMeta:
:return:
"""
# Loop through each WRF-Hydro forecast cycle being processed. Within
# each cycle, perform the following tasks:
# 1.) Loop over each output frequency
# 2.) Determine the input forcing cycle dates (both before and after)
# for temporal interpolation, downscaling, and bias correction reasons.
# 3.) If the input forcings haven't been opened and read into memory,
# open them.
# 4.) Check to see if the ESMF objects for input forcings have been
# created. If not, create them, including the regridding object.
# 5.) Regrid forcing grids for input cycle dates surrounding the
# current output timestep if they haven't been regridded.
# 6.) Perform bias correction and/or downscaling.
# 7.) Output final grids to LDASIN NetCDF files with associated
# WRF-Hydro geospatial metadata to the final output directories.
# Throughout this entire process, log progress being made into LOG
# files. Once a forecast cycle is complete, we will touch an empty
# 'WrfHydroForcing.COMPLETE' flag in the directory. This will be
# checked upon the beginning of this program to see if we
# need to process any files.
for fcstCycleNum in range(ConfigOptions.nFcsts):
ConfigOptions.current_fcst_cycle = ConfigOptions.b_date_proc + datetime.timedelta(
seconds=ConfigOptions.fcst_freq * 60 * fcstCycleNum)
if ConfigOptions.first_fcst_cycle is None:
ConfigOptions.first_fcst_cycle = ConfigOptions.current_fcst_cycle
if ConfigOptions.ana_flag:
fcstCycleOutDir = ConfigOptions.output_dir + "/" + ConfigOptions.e_date_proc.strftime('%Y%m%d%H')
else:
fcstCycleOutDir = ConfigOptions.output_dir + "/" + ConfigOptions.current_fcst_cycle.strftime('%Y%m%d%H')
# put all AnA output in the same directory
if ConfigOptions.ana_flag:
if ConfigOptions.ana_out_dir is None:
ConfigOptions.ana_out_dir = fcstCycleOutDir
fcstCycleOutDir = ConfigOptions.ana_out_dir
# completeFlag = ConfigOptions.scratch_dir + "/WrfHydroForcing.COMPLETE"
completeFlag = fcstCycleOutDir + "/WrfHydroForcing.COMPLETE"
if os.path.isfile(completeFlag):
ConfigOptions.statusMsg = "Forecast Cycle: " + \
ConfigOptions.current_fcst_cycle.strftime('%Y-%m-%d %H:%M') + \
" has already completed."
err_handler.log_msg(ConfigOptions, MpiConfig)
# We have already completed processing this cycle,
# move on.
continue
if (not ConfigOptions.ana_flag) or (ConfigOptions.logFile is None):
if MpiConfig.rank == 0:
# If the cycle directory doesn't exist, create it.
if not os.path.isdir(fcstCycleOutDir):
try:
os.mkdir(fcstCycleOutDir)
except:
ConfigOptions.errMsg = "Unable to create output " \
"directory: " + fcstCycleOutDir
err_handler.err_out_screen_para(ConfigOptions.errMsg, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Compose a path to a log file, which will contain information
# about this forecast cycle.
# ConfigOptions.logFile = ConfigOptions.output_dir + "/LOG_" + \
if ConfigOptions.ana_flag:
log_time = ConfigOptions.e_date_proc
else:
log_time = ConfigOptions.current_fcst_cycle
ConfigOptions.logFile = ConfigOptions.scratch_dir + "/LOG_" + ConfigOptions.nwmConfig + \
ConfigOptions.d_program_init.strftime('%Y%m%d%H%M') + \
"_" + log_time.strftime('%Y%m%d%H%M')
# Initialize the log file.
try:
err_handler.init_log(ConfigOptions, MpiConfig)
except:
err_handler.err_out_screen_para(ConfigOptions.errMsg, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Log information about this forecast cycle
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
err_handler.log_msg(ConfigOptions, MpiConfig)
ConfigOptions.statusMsg = 'Processing Forecast Cycle: ' + \
ConfigOptions.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')
err_handler.log_msg(ConfigOptions, MpiConfig)
ConfigOptions.statusMsg = 'Forecast Cycle Length is: ' + \
str(ConfigOptions.cycle_length_minutes) + " minutes"
err_handler.log_msg(ConfigOptions, MpiConfig)
# MpiConfig.comm.barrier()
# Loop through each output timestep. Perform the following functions:
# 1.) Calculate all necessary input files per user options.
# 2.) Read in input forcings from GRIB/NetCDF files.
# 3.) Regrid the forcings, and temporally interpolate.
# 4.) Downscale.
# 5.) Layer, and output as necessary.
ana_factor = 1 if ConfigOptions.ana_flag is False else 0
for outStep in range(1, ConfigOptions.num_output_steps + 1):
# Reset out final grids to missing values.
OutputObj.output_local[:, :, :] = -9999.0
ConfigOptions.current_output_step = outStep
OutputObj.outDate = ConfigOptions.current_fcst_cycle + datetime.timedelta(
seconds=ConfigOptions.output_freq * 60 * outStep
)
ConfigOptions.current_output_date = OutputObj.outDate
# if AnA, adjust file date for analysis vs forecast
if ConfigOptions.ana_flag:
file_date = OutputObj.outDate - datetime.timedelta(seconds=ConfigOptions.output_freq * 60)
else:
file_date = OutputObj.outDate
# Calculate the previous output timestep. This is used in potential downscaling routines.
if outStep == ana_factor:
ConfigOptions.prev_output_date = ConfigOptions.current_output_date
else:
ConfigOptions.prev_output_date = ConfigOptions.current_output_date - datetime.timedelta(
seconds=ConfigOptions.output_freq * 60
)
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = '========================================='
err_handler.log_msg(ConfigOptions, MpiConfig)
ConfigOptions.statusMsg = "Processing for output timestep: " + \
file_date.strftime('%Y-%m-%d %H:%M')
err_handler.log_msg(ConfigOptions, MpiConfig)
# MpiConfig.comm.barrier()
# Compose the expected path to the output file. Check to see if the file exists,
# if so, continue to the next time step. Also initialize our output arrays if necessary.
OutputObj.outPath = fcstCycleOutDir + "/" + file_date.strftime('%Y%m%d%H%M') + \
".LDASIN_DOMAIN1"
# MpiConfig.comm.barrier()
if os.path.isfile(OutputObj.outPath):
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Output file: " + OutputObj.outPath + " exists. Moving " + \
" to the next output timestep."
err_handler.log_msg(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
continue
else:
ConfigOptions.currentForceNum = 0
ConfigOptions.currentCustomForceNum = 0
# Loop over each of the input forcings specifed.
for forceKey in ConfigOptions.input_forcings:
input_forcings = inputForcingMod[forceKey]
# Calculate the previous and next input cycle files from the inputs.
input_forcings.calc_neighbor_files(ConfigOptions, OutputObj.outDate, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Regrid forcings.
input_forcings.regrid_inputs(ConfigOptions, wrfHydroGeoMeta, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Run check on regridded fields for reasonable values that are not missing values.
err_handler.check_forcing_bounds(ConfigOptions, input_forcings, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# If we are restarting a forecast cycle, re-calculate the neighboring files, and regrid the
# next set of forcings as the previous step just regridded the previous forcing.
if input_forcings.rstFlag == 1:
if input_forcings.regridded_forcings1 is not None and \
input_forcings.regridded_forcings2 is not None:
# Set the forcings back to reflect we just regridded the previous set of inputs, not the next.
input_forcings.regridded_forcings1[:, :, :] = \
input_forcings.regridded_forcings2[:, :, :]
# Re-calculate the neighbor files.
input_forcings.calc_neighbor_files(ConfigOptions, OutputObj.outDate, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Regrid the forcings for the end of the window.
input_forcings.regrid_inputs(ConfigOptions, wrfHydroGeoMeta, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
input_forcings.rstFlag = 0
# Run temporal interpolation on the grids.
input_forcings.temporal_interpolate_inputs(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Run bias correction.
bias_correction.run_bias_correction(input_forcings, ConfigOptions,
wrfHydroGeoMeta, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Run downscaling on grids for this output timestep.
downscale.run_downscaling(input_forcings, ConfigOptions,
wrfHydroGeoMeta, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Layer in forcings from this product.
layeringMod.layer_final_forcings(OutputObj, input_forcings, ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
ConfigOptions.currentForceNum = ConfigOptions.currentForceNum + 1
if forceKey == 10:
ConfigOptions.currentCustomForceNum = ConfigOptions.currentCustomForceNum + 1
# Process supplemental precipitation if we specified in the configuration file.
if ConfigOptions.number_supp_pcp > 0:
for suppPcpKey in ConfigOptions.supp_precip_forcings:
# Like with input forcings, calculate the neighboring files to use.
suppPcpMod[suppPcpKey].calc_neighbor_files(ConfigOptions, OutputObj.outDate, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Regrid the supplemental precipitation.
suppPcpMod[suppPcpKey].regrid_inputs(ConfigOptions, wrfHydroGeoMeta, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
if suppPcpMod[suppPcpKey].regridded_precip1 is not None \
and suppPcpMod[suppPcpKey].regridded_precip2 is not None:
# if np.any(suppPcpMod[suppPcpKey].regridded_precip1) and \
# np.any(suppPcpMod[suppPcpKey].regridded_precip2):
# Run check on regridded fields for reasonable values that are not missing values.
err_handler.check_supp_pcp_bounds(ConfigOptions, suppPcpMod[suppPcpKey], MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Run temporal interpolation on the grids.
suppPcpMod[suppPcpKey].temporal_interpolate_inputs(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Layer in the supplemental precipitation into the current output object.
layeringMod.layer_supplemental_forcing(OutputObj, suppPcpMod[suppPcpKey],
ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Call the output routines
# adjust date for AnA if necessary
if ConfigOptions.ana_flag:
OutputObj.outDate = file_date
OutputObj.output_final_ldasin(ConfigOptions, wrfHydroGeoMeta, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
if (not ConfigOptions.ana_flag) or (fcstCycleNum == (ConfigOptions.nFcsts - 1)):
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Forcings complete for forecast cycle: " + \
ConfigOptions.current_fcst_cycle.strftime('%Y-%m-%d %H:%M')
err_handler.log_msg(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
if MpiConfig.rank == 0:
# Close the log file.
try:
err_handler.close_log(ConfigOptions, MpiConfig)
except:
err_handler.err_out_screen_para(ConfigOptions.errMsg, MpiConfig)
# Success.... Now touch an empty complete file for this forecast cycle to indicate
# completion in case the code is re-ran.
try:
open(completeFlag, 'a').close()
except:
ConfigOptions.errMsg = "Unable to create completion file: " + completeFlag
err_handler.log_critical(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
def nwm_monthly_PRISM_downscale(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig):
"""
NCAR/OWP function for downscaling precipitation using monthly PRISM climatology in a
mountain-mapper like fashion.
:param input_forcings:
:param ConfigOptions:
:param GeoMetaWrfHydro:
:return:
"""
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Performing NWM Monthly PRISM Mountain Mapper " \
"Downscaling of Precipitation"
err_handler.log_msg(ConfigOptions, MpiConfig)
# Establish whether or not we need to read in new PRISM monthly climatology:
# 1.) This is the first output timestep, and no grids have been initialized.
# 2.) We have switched months from the last timestep. In this case, we need
# to re-initialize the grids for the current month.
initialize_flag = False
if input_forcings.nwmPRISM_denGrid is None and input_forcings.nwmPRISM_numGrid is None:
# We are on situation 1 - This is the first output step.
initialize_flag = True
# print('WE NEED TO READ IN PRISM GRIDS')
if ConfigOptions.current_output_date.month != ConfigOptions.prev_output_date.month:
# We are on situation #2 - The month has changed so we need to reinitialize the
# PRISM grids.
initialize_flag = True
# print('MONTH CHANGE.... NEED TO READ IN NEW PRISM GRIDS.')
if initialize_flag is True:
while (True):
# First reset the local PRISM grids to be safe.
input_forcings.nwmPRISM_numGrid = None
input_forcings.nwmPRISM_denGrid = None
# Compose paths to the expected files.
numeratorPath = input_forcings.paramDir + "/PRISM_Precip_Clim_" + \
ConfigOptions.current_output_date.strftime('%h') + '_NWM_Mtn_Mapper_Numer.nc'
denominatorPath = input_forcings.paramDir + "/PRISM_Precip_Clim_" + \
ConfigOptions.current_output_date.strftime('%h') + '_NWM_Mtn_Mapper_Denom.nc'
#print(numeratorPath)
#print(denominatorPath)
# Make sure files exist.
if not os.path.isfile(numeratorPath):
ConfigOptions.errMsg = "Expected parameter file: " + numeratorPath + \
" for mountain mapper downscaling of precipitation not found."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if not os.path.isfile(denominatorPath):
ConfigOptions.errMsg = "Expected parameter file: " + denominatorPath + \
" for mountain mapper downscaling of precipitation not found."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if MpiConfig.rank == 0:
# Open the NetCDF parameter files. Check to make sure expected dimension
# sizes are in place, along with variable names, etc.
try:
idNum = Dataset(numeratorPath,'r')
except:
ConfigOptions.errMsg = "Unable to open parameter file: " + numeratorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
try:
idDenom = Dataset(denominatorPath,'r')
except:
ConfigOptions.errMsg = "Unable to open parameter file: " + denominatorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Check to make sure expected names, dimension sizes are present.
if 'x' not in idNum.variables.keys():
ConfigOptions.errMsg = "Expected 'x' variable not found in parameter file: " + numeratorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if 'x' not in idDenom.variables.keys():
ConfigOptions.errMsg = "Expected 'x' variable not found in parameter file: " + denominatorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if 'y' not in idNum.variables.keys():
ConfigOptions.errMsg = "Expected 'y' variable not found in parameter file: " + numeratorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if 'y' not in idDenom.variables.keys():
ConfigOptions.errMsg = "Expected 'y' variable not found in parameter file: " + denominatorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if 'Data' not in idNum.variables.keys():
ConfigOptions.errMsg = "Expected 'Data' variable not found in parameter file: " + numeratorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if 'Data' not in idDenom.variables.keys():
ConfigOptions.errMsg = "Expected 'Data' variable not found in parameter file: " + denominatorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if idNum.variables['Data'].shape[0] != GeoMetaWrfHydro.ny_global:
ConfigOptions.errMsg = "Input Y dimension for: " + numeratorPath + \
" does not match the output WRF-Hydro Y dimension size."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if idDenom.variables['Data'].shape[0] != GeoMetaWrfHydro.ny_global:
ConfigOptions.errMsg = "Input Y dimension for: " + denominatorPath + \
" does not match the output WRF-Hydro Y dimension size."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if idNum.variables['Data'].shape[1] != GeoMetaWrfHydro.nx_global:
ConfigOptions.errMsg = "Input X dimension for: " + numeratorPath + \
" does not match the output WRF-Hydro X dimension size."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if idDenom.variables['Data'].shape[1] != GeoMetaWrfHydro.nx_global:
ConfigOptions.errMsg = "Input X dimension for: " + denominatorPath + \
" does not match the output WRF-Hydro X dimension size."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Read in the PRISM grid on the output grid. Then scatter the array out to the processors.
try:
numDataTmp = idNum.variables['Data'][:,:]
except:
ConfigOptions.errMsg = "Unable to extract 'Data' from parameter file: " + numeratorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
try:
denDataTmp = idDenom.variables['Data'][:,:]
except:
ConfigOptions.errMsg = "Unable to extract 'Data' from parameter file: " + denominatorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Close the parameter files.
try:
idNum.close()
except:
ConfigOptions.errMsg = "Unable to close parameter file: " + numeratorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
try:
idDenom.close()
except:
ConfigOptions.errMsg = "Unable to close parameter file: " + denominatorPath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
else:
numDataTmp = None
denDataTmp = None
break
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Scatter the array out to the local processors
input_forcings.nwmPRISM_numGrid = MpiConfig.scatter_array(GeoMetaWrfHydro, numDataTmp, ConfigOptions)
err_handler.check_program_status(ConfigOptions, MpiConfig)
input_forcings.nwmPRISM_denGrid = MpiConfig.scatter_array(GeoMetaWrfHydro, denDataTmp, ConfigOptions)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Create temporary grids from the local slabs of params/precip forcings.
localRainRate = input_forcings.final_forcings[3,:,:]
numLocal = input_forcings.nwmPRISM_numGrid[:,:]
denLocal = input_forcings.nwmPRISM_denGrid[:,:]
# Establish index of where we have valid data.
try:
indValid = np.where((localRainRate > 0.0) & (denLocal > 0.0) & (numLocal > 0.0))
except:
ConfigOptions.errMsg = "Unable to run numpy search for valid values on precip and " \
"param grid in mountain mapper downscaling"
err_handler.log_critical(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Convert precipitation rate, which is mm/s to mm, which is needed to run the PRISM downscaling.
try:
localRainRate[indValid] = localRainRate[indValid]*3600.0
except:
ConfigOptions.errMsg = "Unable to convert temporary precip rate from mm/s to mm."
err_handler.log_critical(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
try:
localRainRate[indValid] = localRainRate[indValid] * numLocal[indValid]
except:
ConfigOptions.errMsg = "Unable to multiply precip by numerator in mountain mapper downscaling"
err_handler.log_critical(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
try:
localRainRate[indValid] = localRainRate[indValid] / denLocal[indValid]
except:
ConfigOptions.errMsg = "Unable to divide precip by denominator in mountain mapper downscaling"
err_handler.log_critical(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Convert local precip back to a rate (mm/s)
try:
localRainRate[indValid] = localRainRate[indValid]/3600.0
except:
ConfigOptions.errMsg = "Unable to convert temporary precip rate from mm to mm/s."
err_handler.log_critical(ConfigOptions, MpiConfig)
err_handler.check_program_status(ConfigOptions, MpiConfig)
input_forcings.final_forcings[3, :, :] = localRainRate
# Reset variables for memory efficiency
idDenom = None
idNum = None
localRainRate = None
numLocal = None
denLocal = None
def param_lapse(input_forcings,ConfigOptions,GeoMetaWrfHydro,MpiConfig):
"""
Function that applies a apriori lapse rate adjustment to modeled
2-meter temperature by taking the difference of the native
input elevation and the WRF-hydro elevation. It's assumed this lapse
rate grid has already been regridded to the final output WRF-Hydro
grid.
:param inpute_forcings:
:param ConfigOptions:
:param GeoMetaWrfHydro:
:return:
"""
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Applying aprior lapse rate grid to temperature downscaling"
err_handler.log_msg(ConfigOptions, MpiConfig)
# Calculate the elevation difference.
elevDiff = input_forcings.height - GeoMetaWrfHydro.height
if input_forcings.lapseGrid is None:
#if not np.any(input_forcings.lapseGrid):
# We have not read in our lapse rate file. Read it in, do extensive checks,
# scatter the lapse rate grid out to individual processors, then apply the
# lapse rate to the 2-meter temperature grid.
if MpiConfig.rank == 0:
while (True):
# First ensure we have a parameter directory
if input_forcings.paramDir == "NONE":
ConfigOptions.errMsg = "User has specified spatial temperature lapse rate " \
"downscaling while no downscaling parameter directory " \
"exists."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Compose the path to the lapse rate grid file.
lapsePath = input_forcings.paramDir + "/lapse_param.nc"
if not os.path.isfile(lapsePath):
ConfigOptions.errMsg = "Expected lapse rate parameter file: " + \
lapsePath + " does not exist."
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Open the lapse rate file. Check for the expected variable, along with
# the dimension size to make sure everything matches up.
try:
idTmp = Dataset(lapsePath,'r')
except:
ConfigOptions.errMsg = "Unable to open parameter file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if not 'lapse' in idTmp.variables.keys():
ConfigOptions.errMsg = "Expected 'lapse' variable not located in parameter " \
"file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
try:
lapseTmp = idTmp.variables['lapse'][:,:]
except:
ConfigOptions.errMsg = "Unable to extracte 'lapse' variable from parameter: " \
"file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Check dimensions to ensure they match up to the output grid.
if lapseTmp.shape[1] != GeoMetaWrfHydro.nx_global:
ConfigOptions.errMsg = "X-Dimension size mismatch between output grid and lapse " \
"rate from parameter file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
if lapseTmp.shape[0] != GeoMetaWrfHydro.ny_global:
ConfigOptions.errMsg = "Y-Dimension size mismatch between output grid and lapse " \
"rate from parameter file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Perform a quick search to ensure we don't have radical values.
indTmp = np.where(lapseTmp < -10.0)
if len(indTmp[0]) > 0:
ConfigOptions.errMsg = "Found anomolous negative values in the lapse rate grid from " \
"parameter file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
indTmp = np.where(lapseTmp > 100.0)
if len(indTmp[0]) > 0:
ConfigOptions.errMsg = "Found excessively high values in the lapse rate grid from " \
"parameter file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
# Close the parameter lapse rate file.
try:
idTmp.close()
except:
ConfigOptions.errMsg = "Unable to close parameter file: " + lapsePath
err_handler.log_critical(ConfigOptions, MpiConfig)
break
break
else:
lapseTmp = None
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Scatter the lapse rate grid to the other processors.
input_forcings.lapseGrid = MpiConfig.scatter_array(GeoMetaWrfHydro,lapseTmp,ConfigOptions)
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Apply the local lapse rate grid to our local slab of 2-meter temperature data.
temperature_grid_tmp = input_forcings.final_forcings[4, :, :]
try:
indNdv = np.where(input_forcings.final_forcings == ConfigOptions.globalNdv)
except:
ConfigOptions.errMsg = "Unable to perform NDV search on input " + \
input_forcings.productName + " regridded forcings."
err_handler.log_critical(ConfigOptions, MpiConfig)
return
try:
indValid = np.where(temperature_grid_tmp != ConfigOptions.globalNdv)
except:
ConfigOptions.errMsg = "Unable to perform search for valid values on input " + \
input_forcings.productName + " regridded temperature forcings."
err_handler.log_critical(ConfigOptions, MpiConfig)
return
try:
temperature_grid_tmp[indValid] = temperature_grid_tmp[indValid] + \
((input_forcings.lapseGrid[indValid]/1000.0) * elevDiff[indValid])
except:
ConfigOptions.errMsg = "Unable to apply spatial lapse rate values to input " + \
input_forcings.productName + " regridded temperature forcings."
err_handler.log_critical(ConfigOptions, MpiConfig)
return
input_forcings.final_forcings[4,:,:] = temperature_grid_tmp
input_forcings.final_forcings[indNdv] = ConfigOptions.globalNdv
# Reset for memory efficiency
indTmp = None
indNdv = None
indValid = None
elevDiff = None
temperature_grid_tmp = None
def read_rqi_monthly_climo(ConfigOptions, MpiConfig, supplemental_precip,
GeoMetaWrfHydro):
"""
Function to read in monthly RQI grids on the NWM grid. This is an NWM ONLY
option. Please do not activate if not executing on the NWM conus grid.
:param ConfigOptions:
:param MpiConfig:
:param supplemental_precip:
:return:
"""
# Ensure all processors are synced up before proceeding.
#MpiConfig.comm.barrier()
# First check to see if the RQI grids have valid values in them. There should
# be NO NDV values if the grids have properly been read in.
indTmp = np.where(
supplemental_precip.regridded_rqi2 != ConfigOptions.globalNdv)
rqiPath = ConfigOptions.supp_precip_param_dir + "/MRMS_WGT_RQI0.9_m" + \
supplemental_precip.pcp_date2.strftime('%m') + '_v1.1_geosmth.nc'
if len(indTmp[0]) == 0:
# We haven't initialized the RQI fields. We need to do this.....
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Reading in RQI Parameter File: " + rqiPath
err_handler.log_msg(ConfigOptions, MpiConfig)
# First make sure the RQI file exists.
if not os.path.isfile(rqiPath):
ConfigOptions.errMsg = "Expected RQI parameter file: " + rqiPath + " not found."
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
# Open the Parameter file.
try:
idTmp = Dataset(rqiPath, 'r')
except:
ConfigOptions.errMsg = "Unable to open parameter file: " + rqiPath
pass
# Extract out the RQI grid.
try:
varTmp = idTmp.variables['POP_0mabovemeansealevel'][0, :, :]
except:
ConfigOptions.errMsg = "Unable to extract POP_0mabovemeansealevel from parameter file: " + rqiPath
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
# Sanity checking on grid size.
if varTmp.shape[0] != GeoMetaWrfHydro.ny_global or varTmp.shape[
1] != GeoMetaWrfHydro.nx_global:
ConfigOptions.errMsg = "Improper dimension sizes for POP_0mabovemeansealevel " \
"in parameter file: " + rqiPath
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
else:
idTmp = None
varTmp = None
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Scatter the array out to the local processors
varSubTmp = MpiConfig.scatter_array(GeoMetaWrfHydro, varTmp,
ConfigOptions)
err_handler.check_program_status(ConfigOptions, MpiConfig)
supplemental_precip.regridded_rqi2[:, :] = varSubTmp
# Reset variables for memory purposes
varSubTmp = None
varTmp = None
# Close the RQI NetCDF file
if MpiConfig.rank == 0:
try:
idTmp.close()
except:
ConfigOptions.errMsg = "Unable to close parameter file: " + rqiPath
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Also check to see if we have switched to a new month based on the previous
# MRMS step and the current one.
if supplemental_precip.pcp_date2.month != supplemental_precip.pcp_date1.month:
# We need to read in a new RQI monthly grid.
if MpiConfig.rank == 0:
ConfigOptions.statusMsg = "Reading in RQI Parameter File: " + rqiPath
err_handler.log_msg(ConfigOptions, MpiConfig)
# First make sure the RQI file exists.
if not os.path.isfile(rqiPath):
ConfigOptions.errMsg = "Expected RQI parameter file: " + rqiPath + " not found."
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
# Open the Parameter file.
try:
idTmp = Dataset(rqiPath, 'r')
except:
ConfigOptions.errMsg = "Unable to open parameter file: " + rqiPath
pass
# Extract out the RQI grid.
try:
varTmp = idTmp.variables['POP_0mabovemeansealevel'][0, :, :]
except:
ConfigOptions.errMsg = "Unable to extract POP_0mabovemeansealevel from parameter file: " + rqiPath
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
# Sanity checking on grid size.
if varTmp.shape[0] != GeoMetaWrfHydro.ny_global or varTmp.shape[
1] != GeoMetaWrfHydro.nx_global:
ConfigOptions.errMsg = "Improper dimension sizes for POP_0mabovemeansealevel " \
"in parameter file: " + rqiPath
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
else:
idTmp = None
varTmp = None
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Scatter the array out to the local processors
varSubTmp = MpiConfig.scatter_array(GeoMetaWrfHydro, varTmp,
ConfigOptions)
err_handler.check_program_status(ConfigOptions, MpiConfig)
supplemental_precip.regridded_rqi2[:, :] = varSubTmp
# Reset variables for memory purposes
varSubTmp = None
varTmp = None
# Close the RQI NetCDF file
if MpiConfig.rank == 0:
try:
idTmp.close()
except:
ConfigOptions.errMsg = "Unable to close parameter file: " + rqiPath
err_handler.log_critical(ConfigOptions, MpiConfig)
pass
err_handler.check_program_status(ConfigOptions, MpiConfig)
def open_grib2(GribFileIn, NetCdfFileOut, Wgrib2Cmd, ConfigOptions, MpiConfig,
inputVar):
"""
Generic function to convert a GRIB2 file into a NetCDF file. Function
will also open the NetCDF file, and ensure all necessary inputs are
in file.
:param GribFileIn:
:param NetCdfFileOut:
:param ConfigOptions:
:return:
"""
# Ensure all processors are synced up before outputting.
# MpiConfig.comm.barrier()
# Run wgrib2 command to convert GRIB2 file to NetCDF.
if MpiConfig.rank == 0:
# Check to see if output file already exists. If so, delete it and
# override.
ConfigOptions.statusMsg = "Reading in GRIB2 file: " + GribFileIn
err_handler.log_msg(ConfigOptions, MpiConfig)
if os.path.isfile(NetCdfFileOut):
ConfigOptions.statusMsg = "Overriding temporary NetCDF file: " + NetCdfFileOut
err_handler.log_warning(ConfigOptions, MpiConfig)
try:
# WCOSS fix for WGRIB2 crashing when called on the same file twice in python
if not os.environ.get('MFE_SILENT'):
print("command: " + Wgrib2Cmd)
# set up GRIB2TABLE if needed:
if not os.environ.get('GRIB2TABLE'):
g2path = os.path.join(ConfigOptions.scratch_dir, "grib2.tbl")
with open(g2path, 'wt') as g2t:
g2t.write(
"209:1:0:0:161:1:6:30:MultiSensorQPE01H:"
"Multi-sensor estimated precipitation accumulation 1-hour:mm\n"
"209:1:0:0:161:1:6:37:MultiSensorQPE01H:"
"Multi-sensor estimated precipitation accumulation 1-hour:mm\n"
)
os.environ['GRIB2TABLE'] = g2path
exitcode = subprocess.call(Wgrib2Cmd, shell=True)
#print("exitcode: " + str(exitcode))
# Call WGRIB2 with subprocess.Popen
#cmdOutput = subprocess.Popen([Wgrib2Cmd], stdout=subprocess.PIPE,
# stderr=subprocess.PIPE, shell=True)
#out, err = cmdOutput.communicate()
#exitcode = cmdOutput.returncode
except:
ConfigOptions.errMsg = "Unable to convert: " + GribFileIn + " to " + \
NetCdfFileOut
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
# Reset temporary subprocess variables.
out = None
err = None
exitcode = None
# Ensure file exists.
if not os.path.isfile(NetCdfFileOut):
ConfigOptions.errMsg = "Expected NetCDF file: " + NetCdfFileOut + \
" not found. It's possible the GRIB2 variable was not found."
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
# Open the NetCDF file.
try:
idTmp = Dataset(NetCdfFileOut, 'r')
except:
ConfigOptions.errMsg = "Unable to open input NetCDF file: " + \
NetCdfFileOut
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
if idTmp is not None:
# Check for expected lat/lon variables.
if 'latitude' not in idTmp.variables.keys():
ConfigOptions.statusMsg = "Unable to locate latitude from: " + \
GribFileIn
err_handler.log_warning(ConfigOptions, MpiConfig)
# idTmp = None
pass
if idTmp is not None:
if 'longitude' not in idTmp.variables.keys():
ConfigOptions.statusMsg = "Unable to locate longitude from: " + \
GribFileIn
err_handler.log_warning(ConfigOptions, MpiConfig)
# idTmp = None
pass
if idTmp is not None and inputVar is not None:
# Loop through all the expected variables.
if inputVar not in idTmp.variables.keys():
ConfigOptions.errMsg = "Unable to locate expected variable: " + \
inputVar + " in: " + NetCdfFileOut
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
else:
idTmp = None
# Ensure all processors are synced up before outputting.
# MpiConfig.comm.barrier() ## THIS HAPPENS IN check_program_status
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Return the NetCDF file handle back to the user.
return idTmp
def open_grib2(GribFileIn, NetCdfFileOut, Wgrib2Cmd, ConfigOptions, MpiConfig,
inputVar):
"""
Generic function to convert a GRIB2 file into a NetCDF file. Function
will also open the NetCDF file, and ensure all necessary inputs are
in file.
:param GribFileIn:
:param NetCdfFileOut:
:param ConfigOptions:
:return:
"""
# Ensure all processors are synced up before outputting.
# MpiConfig.comm.barrier()
# Run wgrib2 command to convert GRIB2 file to NetCDF.
if MpiConfig.rank == 0:
# Check to see if output file already exists. If so, delete it and
# override.
ConfigOptions.statusMsg = "Reading in GRIB2 file: " + GribFileIn
err_handler.log_msg(ConfigOptions, MpiConfig)
if os.path.isfile(NetCdfFileOut):
ConfigOptions.statusMsg = "Overriding temporary NetCDF file: " + NetCdfFileOut
err_handler.log_warning(ConfigOptions, MpiConfig)
try:
cmdOutput = subprocess.Popen([Wgrib2Cmd],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
out, err = cmdOutput.communicate()
exitcode = cmdOutput.returncode
#subprocess.run([Wgrib2Cmd],shell=True)
except:
ConfigOptions.errMsg = "Unable to convert: " + GribFileIn + " to " + \
NetCdfFileOut
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
# Reset temporary subprocess variables.
out = None
err = None
exitcode = None
# Ensure file exists.
if not os.path.isfile(NetCdfFileOut):
ConfigOptions.errMsg = "Expected NetCDF file: " + NetCdfFileOut + \
" not found. It's possible the GRIB2 variable was not found."
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
# Open the NetCDF file.
try:
idTmp = Dataset(NetCdfFileOut, 'r')
except:
ConfigOptions.errMsg = "Unable to open input NetCDF file: " + \
NetCdfFileOut
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
if idTmp is not None:
# Check for expected lat/lon variables.
if 'latitude' not in idTmp.variables.keys():
ConfigOptions.errMsg = "Unable to locate latitude from: " + \
GribFileIn
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
if idTmp is not None:
if 'longitude' not in idTmp.variables.keys():
ConfigOptions.errMsg = "Unable t locate longitude from: " + \
GribFileIn
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
if idTmp is not None:
# Loop through all the expected variables.
if inputVar not in idTmp.variables.keys():
ConfigOptions.errMsg = "Unable to locate expected variable: " + \
inputVar + " in: " + NetCdfFileOut
err_handler.log_critical(ConfigOptions, MpiConfig)
idTmp = None
pass
else:
idTmp = None
# Ensure all processors are synced up before outputting.
# MpiConfig.comm.barrier() ## THIS HAPPENS IN check_program_status
err_handler.check_program_status(ConfigOptions, MpiConfig)
# Return the NetCDF file handle back to the user.
return idTmp
def ext_ana_disaggregate(input_forcings, supplemental_precip, config_options, mpi_config):
"""
Function for disaggregating 6hr SuppPcp data to 1hr Input data
:param input_forcings:
:param supplemental_precip:
:param config_options:
:param mpi_config:
:return:
"""
# Check to make sure we have valid grids.
if input_forcings.regridded_forcings2 is None or supplemental_precip.regridded_precip2 is None:
if mpi_config.rank == 0:
config_options.statusMsg = "Bypassing ext_ana_disaggregation routine due to missing input or supp pcp data"
err_handler.log_warning(config_options, mpi_config)
return
if supplemental_precip.ext_ana != "STAGE4":
if mpi_config.rank == 0:
config_options.statusMsg = f"Bypassing ext_ana_disaggregation routine due to supplemental_precip.ext_ana = {supplemental_precip.ext_ana}"
err_handler.log_warning(config_options, mpi_config)
return
#print("ext_ana_disaggregate RAINRATE input_forcings.regridded_forcings2[3,:,:]")
#print(input_forcings.regridded_forcings2[3,:,:])
#print("ext_ana_disaggregate supplemental_precip.regridded_precip2[:,:]")
#print(supplemental_precip.regridded_precip2[:,:])
#print("supplemental_precip.regridded_precip2[:,:].shape")
#print(supplemental_precip.regridded_precip2[:,:].shape)
read_hours = 0
found_target_hh = False
ana_data = []
if mpi_config.rank == 0:
target_hh = Path(input_forcings.file_in2).stem[-4:-2]
_,_,_,beg_hh,end_hh,yyyymmdd = Path(supplemental_precip.file_in2).stem.split('_')
date_iter = datetime.strptime(f"{yyyymmdd}{beg_hh}", '%Y%m%d%H')
end_date = date_iter + timedelta(hours=6)
#Advance the date_iter by 1 hour since the beginning of the Stage IV data in date range is excluded, the end is included
#(begin_date,end_date]
date_iter += timedelta(hours=1)
while date_iter <= end_date:
tmp_file = f"{input_forcings.inDir}/{date_iter.strftime('%Y%m%d%H')}/{date_iter.strftime('%Y%m%d%H')}00.LDASIN_DOMAIN1"
if os.path.exists(tmp_file):
config_options.statusMsg = f"Reading {input_forcings.netcdf_var_names[3]} from {tmp_file} for disaggregation"
err_handler.log_msg(config_options, mpi_config)
with Dataset(tmp_file,'r') as ds:
try:
#Read in rainrate
data = ds.variables[input_forcings.netcdf_var_names[3]][0, :, :]
data[data == config_options.globalNdv] = np.nan
ana_data.append(data)
read_hours += 1
if date_iter.hour == int(target_hh):
found_target_hh = True
except (ValueError, KeyError, AttributeError) as err:
config_options.errMsg = f"Unable to extract: RAINRATE from: {input_forcings.file_in2} ({str(err)})"
err_handler.log_critical(config_options, mpi_config)
else:
config_options.statusMsg = f"Input file missing {tmp_file}"
err_handler.log_warning(config_options, mpi_config)
date_iter += timedelta(hours=1)
found_target_hh = mpi_config.broadcast_parameter(found_target_hh, config_options, param_type=bool)
err_handler.check_program_status(config_options, mpi_config)
if not found_target_hh:
if mpi_config.rank == 0:
config_options.statusMsg = f"Could not find AnA target_hh = {target_hh} for disaggregation. Setting output values to {config_options.globalNdv}."
err_handler.log_warning(config_options, mpi_config)
supplemental_precip.regridded_precip2[:,:] = config_options.globalNdv
return
read_hours = mpi_config.broadcast_parameter(read_hours, config_options, param_type=int)
err_handler.check_program_status(config_options, mpi_config)
if read_hours != 6:
if mpi_config.rank == 0:
config_options.statusMsg = f"Could not find all 6 AnA files for disaggregation. Only found {read_hours} hours. Setting output values to {config_options.globalNdv}."
err_handler.log_warning(config_options, mpi_config)
supplemental_precip.regridded_precip2[:,:] = config_options.globalNdv
return
ana_sum = np.array([],dtype=np.float32)
target_data = np.array([],dtype=np.float32)
ana_all_zeros = np.array([],dtype=np.bool)
ana_no_zeros = np.array([],dtype=np.bool)
target_data_no_zeros = np.array([],dtype=np.bool)
if mpi_config.rank == 0:
config_options.statusMsg = f"Performing hourly disaggregation of {supplemental_precip.file_in2}"
err_handler.log_msg(config_options, mpi_config)
ana_sum = sum(ana_data)
target_data = ana_data[(int(target_hh)-1)%6]
ana_zeros = [(a == 0).astype(int) for a in ana_data]
target_data_zeros = (target_data == 0)
target_data_no_zeros = ~target_data_zeros
ana_zeros_sum = sum(ana_zeros)
ana_all_zeros = (ana_zeros_sum == 6)
ana_no_zeros = (ana_zeros_sum == 0)
err_handler.check_program_status(config_options, mpi_config)
ana_sum = mpi_config.scatter_array(input_forcings, ana_sum, config_options)
err_handler.check_program_status(config_options, mpi_config)
target_data = mpi_config.scatter_array(input_forcings, target_data, config_options)
err_handler.check_program_status(config_options, mpi_config)
ana_all_zeros = mpi_config.scatter_array(input_forcings, ana_all_zeros, config_options)
err_handler.check_program_status(config_options, mpi_config)
ana_no_zeros = mpi_config.scatter_array(input_forcings, ana_no_zeros, config_options)
err_handler.check_program_status(config_options, mpi_config)
target_data_no_zeros = mpi_config.scatter_array(input_forcings, target_data_no_zeros, config_options)
err_handler.check_program_status(config_options, mpi_config)
if mpi_config.comm.Get_size() == 1 and test_enabled:
test_file = f"{config_options.scratch_dir}/stage_4_A_PCP_GDS5_SFC_acc6h_{yyyymmdd}_{beg_hh}_{end_hh}.txt"
np.savetxt(test_file,supplemental_precip.regridded_precip2)
test_file = f"{config_options.scratch_dir}/disaggregation_factors_{target_hh}_{yyyymmdd}{beg_hh}_{end_date.strftime('%Y%m%d%H')}.txt"
np.savetxt(test_file,np.nan_to_num(np.select([ana_all_zeros,
(ana_no_zeros | target_data_no_zeros)],
[1/6.0*np.ones(supplemental_precip.regridded_precip2[:,:].shape),
target_data/ana_sum],
0),nan=config_options.globalNdv))
#supplemental_precip.regridded_precip2[(0.0 < supplemental_precip.regridded_precip2) & (supplemental_precip.regridded_precip2 < 0.00003)] = 0.0
supplemental_precip.regridded_precip2[:,:] = np.select([ana_all_zeros,
(ana_no_zeros | target_data_no_zeros)],
[1/6.0*supplemental_precip.regridded_precip2[:,:],
supplemental_precip.regridded_precip2[:,:] * target_data/ana_sum],
0)
np.nan_to_num(supplemental_precip.regridded_precip2[:,:], copy=False, nan=config_options.globalNdv)
if mpi_config.comm.Get_size() == 1 and test_enabled:
test_file = f"{config_options.scratch_dir}/stage_4_A_PCP_GDS5_SFC_acc6_disaggregation_{target_hh}_{yyyymmdd}{beg_hh}_{end_date.strftime('%Y%m%d%H')}.txt"
np.savetxt(test_file,supplemental_precip.regridded_precip2)
