def createStateFile(parms, realtime):
""" Called if there is no state file, look at data dirs and create state
Parameters
----------
parms: Parms
Parameter settings
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
state = State("")
if (realtime):
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
for f in cfs._content:
f.debugPrint("Newest files: CFS")
state.initialize(cfs)
# write out file
state.write(parms._stateFile)
def run(configFile, realtime):
if not os.path.exists(configFile):
print 'ERROR forcing engine config file not found.'
return 1
# read in fixed main params
parms = parmRead(configFile, realtime)
#parms.debugPrint()
#if there is not a state file, create one now using newest
if (not os.path.exists(parms._stateFile)):
parms.debugPrint()
createStateFile(parms, realtime)
# begin normal processing situation
WhfLog.debug("....Check for new input data to regid")
# read in state
state = State(parms._stateFile)
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
# Same with CFS
toProcess = state.updateWithNew(cfs, parms._hoursBackCfs)
for f in toProcess:
regridCFS(configFile, f)
# write out state and exit
#state.debugPrint()
state.write(parms._stateFile)
return 0
def layerIfReady(self, parms, configFile):
""" Perform layering if state is such that it should be done
Parameters
----------
parms : Parms
Parameters
configFile : string
name of file with settings
Returns
-------
bool
True if layering was done, or had previously been done
"""
if (self._layered == 1):
return 1
if (self._hrrr == 0 and self._rap == 0):
return 0
if (self._hrrr == 1 and self._rap == 1):
self.layer(parms, configFile)
self._layered = 1
return 1
if (self._hrrr == 0 and self._rap == 1):
ntime = datetime.datetime.utcnow()
diff = ntime - self._clockTime
idiff = diff.total_seconds()
if (idiff > parms._maxWaitSeconds):
WhfLog.debug("timeout..Should layer, dt=%d", idiff)
self.passthroughRap(parms)
self._layered = 1
return 1
return 0
def regridIfZeroHr(configFile, fileType, fname):
"""If it is a 0 hour forecast (RAP or HRRR) regrid in a special way
Parameters
----------
configFile : str
configuration file with all settings
fileType: str
HRRR, RAP, ... string
fname: str
name of file to regrid and downscale, with yyyymmdd parent dir
Returns
-------
None
"""
# check for 0 hour by creating a DataFile and checking forecast hour
try:
f = df.DataFile(fname[0:8], fname[9:], fileType)
except FilenameMatchError as fe:
WhfLog.debug("Cannot check for 0 hour data due to %s", fe)
raise
except InvalidArgumentError as ie:
WhfLog.debug("Cannot check for 0 hour data due to %s", ie)
raise
if (f._time._forecastHour == 0):
WhfLog.setConfigType('AA')
WhfLog.debug("SPECIAL 0 hour case %s", fname[9:0])
aaf.forcing(configFile, 'regrid', fileType, fname[9:])
WhfLog.setConfigType('Short')
def parmRead(fname, fileType, realtime):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
fileType: str
'RAP', 'HRRR', 'MRMS', 'GFS'
realtime: boolean
True if realtime, False for archive mode
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
label = 'Regrid' + fileType
if (realtime):
WhfLog.init(parser, label, False)
else:
WhfLog.set(label)
dataDir = parser.get('data_dir', fileType + '_data')
maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max'))
hoursBack = int(parser.get('triggering', fileType + '_hours_back'))
stateFile = parser.get('triggering', fileType + '_regrid_state_file')
parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile)
return parms
def obsExists(dir, issueTime):
""" Check if obs exists
Parameters
----------
dir : str
Full path to the MRMS directories
issueTime : datetime
The issue time (y,m,d,h)
Returns
-------
bool
True if the data does exist on disk
"""
path = dir + "/"
path += issueTime.strftime("%Y%m%d%H")
if (os.path.isdir(path)):
fname = issueTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
names = df.getFileNames(path)
for n in names:
if (n == fname):
WhfLog.debug("Found %s in %s", fname, path)
return True
return False
def withinNHours(self, ftime, N):
""" Check if input time is within some number of issue hours of self
Parameters
----------
ftime: ForecastTime
N:int
Number of hours to check
Returns
-------
True if input time - local time <= N hours
"""
# same day, issue hour match
# same day, issue hour input - issue hour <= N
if (self._fcstTime == ftime._fcstTime):
if (self._issueHour == ftime._issueHour):
return True
elif (ftime._issueHour-self._issueHour <= N):
return True
else:
return False
else:
# create a full time from local and input states
timeIn = ftime.ymdh()
timeLoc = self.ymdh()
diff = (timeIn - timeLoc).total_seconds()
maxSeconds = N*3600
if (diff < 0):
WhfLog.error("Unexpected data newer than newest")
return False
else:
return (diff <= maxSeconds)
def regridIfZeroHr(configFile, fileType, fname):
"""If it is a 0 hour forecast (RAP or HRRR) regrid in a special way
Parameters
----------
configFile : str
configuration file with all settings
fileType: str
HRRR, RAP, ... string
fname: str
name of file to regrid and downscale, with yyyymmdd parent dir
Returns
-------
None
"""
# check for 0 hour by creating a DataFile and checking forecast hour
try:
f = df.DataFile(fname, fileType)
except FilenameMatchError as fe:
WhfLog.debug("Cannot check for 0 hour data due to %s", fe)
raise
except InvalidArgumentError as ie:
WhfLog.debug("Cannot check for 0 hour data due to %s", ie)
raise
if (f._time._forecastHour == 0):
WhfLog.setConfigType('AA')
WhfLog.debug("SPECIAL 0 hour case %s", fname[9:0])
aaf.forcing(configFile, 'regrid', 'HRRR', fname[9:])
WhfLog.setConfigType('Short')
def _forecastExists(self, dir):
""" Check if forecast indicated by local state exists
Parameters
----------
dir : str
Full path to the issue time directories
Returns
-------
bool
True if the forecast does exist on disk
"""
path = dir + "/"
path += self._issue.strftime("%Y%m%d%H")
if (os.path.isdir(path)):
fname = self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
names = df.getFileNames(path)
for n in names:
if (n == fname):
WhfLog.debug("Found %s in %s", fname, path)
return True
return False
else:
return False
def lookForNew(self, data, hoursBack, fileType):
""" See if new data has arrived compared to state.
If a new issue time, purge older stuff from state.
Parameters
----------
data: DataFiles
The newest data
hoursBack: int
Maximum number of hours back to keep data in state
fileType : str
'HRRR', 'RAP', ...
Returns
-------
list[str]
The data file names that are are to be added to state
"""
ret = []
fnames = data.getFnames()
if (not fnames):
return ret
if (self.isEmpty()):
WhfLog.debug("Adding to empty list")
else:
sname = self.newest()
if (not sname):
WhfLog.error("Expected file, got none")
return ret
self._analyzeNewest(fnames[-1], sname, hoursBack, fileType)
for f in fnames:
if (self._isNew(f)):
ret.append(f)
return ret
def forecastExists(self, dir):
""" Check if forecast indicated by local state exists
Parameters
----------
dir : str
Full path to the issue time directories
Returns
-------
bool
True if the forecast does exist on disk
"""
path = dir + "/"
path += self._issue.strftime("%Y%m%d%H")
if (os.path.isdir(path)):
fname = self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
names = df.getFileNames(path)
for n in names:
if (n == fname):
WhfLog.debug("Found %s in %s", fname, path)
return 1
return 0
else:
return 0
def parmRead(fname):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "LongRangeRegridDriver"
WhfLog.init(parser, forcing_config_label, 'Long', 'Regrid', 'CFS')
cfsDir = parser.get('data_dir', 'CFS_data')
cfsNumEnsemble = int(parser.get('data_dir', 'CFS_num_ensemble'))
maxFcstHourCfs = int(parser.get('fcsthr_max', 'CFS_fcsthr_max'))
hoursBackCfs = int(parser.get('triggering', 'CFS_hours_back'))
stateFile = parser.get('triggering', 'long_range_regrid_state_file')
parms = Parms(cfsDir, cfsNumEnsemble, maxFcstHourCfs, hoursBackCfs,
stateFile)
return parms
def layerIfReady(self, parms, configFile):
""" Perform layering if state is such that it should be done
Parameters
----------
parms : Parms
Parameters
configFile : string
name of file with settings
Returns
-------
bool
True if layering was done, or had previously been done
"""
if (self._layered):
return True
if ((not self._hrrr) and (not self._rap)):
return False
if (self._hrrr and self._rap):
self._layer(parms, configFile)
self._layered = True
return True
if (self._rap and not self._hrrr):
ntime = datetime.datetime.utcnow()
diff = ntime - self._clockTime
idiff = diff.total_seconds()
if (idiff > parms._maxWaitSeconds):
WhfLog.debug("timeout..Should layer, dt=%d", idiff)
self._passthroughRap(parms)
self._layered = True
return True
return False
def obsExists(dir, issueTime):
""" Check if obs exists
Parameters
----------
dir : str
Full path to the MRMS directories
issueTime : datetime
The issue time (y,m,d,h)
Returns
-------
bool
True if the data does exist on disk
"""
path = dir + "/"
path += issueTime.strftime("%Y%m%d%H")
if (os.path.isdir(path)):
fname = issueTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
names = df.getFileNames(path)
for n in names:
if (n == fname):
WhfLog.debug("Found %s in %s", fname, path)
return True
return False
def createStateFile(parms, fileType, realtime):
""" Called if there is no state file, look at data dirs and create state
in realtime, in non-realtime create an empty state. Write to file.
Parameters
----------
parms: Parms
Parameter settings
fileType: str
'HRRR', ...
realtime: boolean
True if realtime, False for archive mode
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
state = State("")
if (realtime):
# query each directory and get newest model run file for each, then
# get all for that and previous issue time, this becomes state that
# is not re-processed, we only look for new stuff
data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType)
data.setNewestFiles(parms._hoursBack)
for f in data._content:
f.debugPrint("Newest files: " + fileType)
state.initialize(data)
# write out file (at least try to)
state.write(parms._stateFile, fileType)
def forecastExists(dir, issueTime, fcstHour):
""" Check if forecast exists
Parameters
----------
dir : str
Full path to the issue time directories
issueTime : datetime
The issue time (y,m,d,h)
fcstHour: int
should be 0 or 3
Returns
-------
bool
True if the forecast does exist on disk
"""
path = dir + "/"
path += issueTime.strftime("%Y%m%d%H")
if (os.path.isdir(path)):
validTime = issueTime + datetime.timedelta(hours=fcstHour)
fname = validTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
names = df.getFileNames(path)
for n in names:
if (n == fname):
WhfLog.debug("Found %s in %s", fname, path)
return True
return False
def filterWithinNHours(files, type, ftime, N):
"""Filter a list of file names with assumed format to those in a time range
Parameters
----------
files: list[str]
file names, each with parent directory: 'yyyymmdd/<file>
type: str
file type string, 'RAP', 'HRRR', 'MRMS', GFS'
ftime: ForecastTime
time to compare against, assumed most recent time
N: int
Number of hours back from ftime to keep
Returns
-------
list[str]
subset of input files with issue time is in the range [ftime-N,ftime]
"""
ret = []
for f in files:
df = DataFile(f[0:8], f[9:], type)
if (df._ok):
ithFtime = df._time
if (ithFtime.withinNHours(ftime, N)):
ret.append(f)
else:
WhfLog.debug("Did not append file, too old compared to %s file=%s", ftime.debugString(), f)
WhfLog.debug("filtering within %d hours, input length %d output %d",
N, len(files), len(ret))
return ret
def parmRead(fname):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "LongRangeRegridDriver"
WhfLog.init(parser, forcing_config_label, 'Long', 'Regrid', 'CFS')
cfsDir = parser.get('data_dir', 'CFS_data')
cfsNumEnsemble = int(parser.get('data_dir', 'CFS_num_ensemble'))
maxFcstHourCfs = int(parser.get('fcsthr_max', 'CFS_fcsthr_max'))
hoursBackCfs = int(parser.get('triggering', 'CFS_hours_back'))
stateFile = parser.get('triggering', 'long_range_regrid_state_file')
parms = Parms(cfsDir, cfsNumEnsemble, maxFcstHourCfs, hoursBackCfs,
stateFile)
return parms
def forecastExists(dir, issueTime, fcstHour):
""" Check if forecast exists
Parameters
----------
dir : str
Full path to the issue time directories
issueTime : datetime
The issue time (y,m,d,h)
fcstHour: int
should be 0 or 3
Returns
-------
bool
True if the forecast does exist on disk
"""
path = dir + "/"
path += issueTime.strftime("%Y%m%d%H")
if (os.path.isdir(path)):
validTime = issueTime + datetime.timedelta(hours=fcstHour)
fname = validTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
names = df.getFileNames(path)
for n in names:
if (n == fname):
WhfLog.debug("Found %s in %s", fname, path)
return True
return False
def _allDataFiles(self):
"""Return all data files on disk, in order
Parameters
----------
none
Returns
-------
list[DataFile]
The DataFile specs, in order oldest to newest
"""
# get the yyyymmdd subdirs
dirs = getYyyymmddSubdirectories(self._topDir)
# sort them into ascending order
dirs = sorted(dirs)
if not dirs:
# nothing there
WhfLog.debug("_allDataFiles: No data in %s", self._topDir)
return []
else:
# make one big array
ret = []
for d in dirs:
ret.extend(self._allDataFilesInDir(d))
return ret
def debugPrint(self):
""" WhfLog debug of content
"""
WhfLog.debug("Model: empty=%d", df.boolToInt(self._empty))
if (self._empty):
return
WhfLog.debug("Model: Issue=%s clockTime=%s",
self._issue.strftime("%Y%m%d%H"),
self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self):
""" WhfLog debug of content
"""
WhfLog.debug("Model: empty=%d", df.boolToInt(self._empty))
if (self._empty):
return
WhfLog.debug("Model: Issue=%s clockTime=%s",
self._issue.strftime("%Y%m%d%H"),
self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self):
""" logging debug of content
"""
WhfLog.debug("FcstStep: empty=%d", self._empty)
if (self._empty):
return
WhfLog.debug("FcstStep[%d] hrrr0:%d hrrr3:%d rap0:%d rap3:%d mrms:%d lay:%d",
self._step, self._hrrr0, self._hrrr3, self._rap0,
self._rap3, self._mrms, self._layered)
def debugPrint(self):
""" logging debug of content
"""
WhfLog.debug("FcstStep: empty=%d", self._empty)
if (self._empty):
return
WhfLog.debug(
"FcstStep[%d] hrrr0:%d hrrr3:%d rap0:%d rap3:%d mrms:%d lay:%d",
self._step, self._hrrr0, self._hrrr3, self._rap0, self._rap3,
self._mrms, self._layered)
def debugPrint(self):
""" WhfLog debug of content
"""
WhfLog.debug("Fcst: empty=%d", int(self._empty))
if (self._empty):
return
WhfLog.debug("Fcst: I:%s V:%s hrrr:%d rap:%d layered:%d clockTime=%s",
self._issue.strftime("%Y%m%d%H"),
self._valid.strftime("%Y%m%d%H"), df.boolToInt(self._hrrr),
df.boolToInt(self._rap), df.boolToInt(self._layered),
self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self):
""" logging debug of content
"""
WhfLog.debug("Fcst: empty=%d first=%d", self._empty, self._first)
if (self._empty):
return
WhfLog.debug(
"Fcst: I:%s step[0]:%s step[1]:%s step[2]:%s layered:%d clockTime=%s",
self._issue.strftime("%Y%m%d%H"), self._step[0].debugPrintString(),
self._step[1].debugPrintString(), self._step[2].debugPrintString(),
self._layered, self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self):
""" logging debug of content
"""
WhfLog.debug("Fcst: empty=%d first=%d", self._empty, self._first)
if (self._empty):
return
WhfLog.debug("Fcst: I:%s step[0]:%s step[1]:%s step[2]:%s layered:%d clockTime=%s",
self._issue.strftime("%Y%m%d%H"),
self._step[0].debugPrintString(),
self._step[1].debugPrintString(),
self._step[2].debugPrintString(),
self._layered,
self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self):
""" Debug logging of content
"""
WhfLog.debug("%s data = %s", self._dataType, self._dataDir)
WhfLog.debug("%s source = %s", self._dataType, self._sourceDataDir)
WhfLog.debug("%s statefile = %s", self._dataType, self._stateFile)
WhfLog.debug("%s format = %s", self._dataType, self._format)
def _analyzeNewest(self, dataNewest, stateNewest, hoursBack, fileType):
if (dataNewest <= stateNewest):
return
# see if issue time has increased and if so, purge old stuff
# create DataFile objects, which requires breaking the full
# file into yymmdd/filename
sind = stateNewest.find('/')
if (sind < 0):
raise FileNameMatchError('Cannot parse directory from ' + stateNewest)
nind = dataNewest.find('/')
if (nind < 0):
raise FileNameMatchError('Cannot parse directory from ' + dataNewest)
symd = stateNewest[:sind]
sfile = stateNewest[sind+1:]
nymd = dataNewest[:nind]
nfile = dataNewest[nind+1:]
WhfLog.debug("Checking %s / %s against %s / %s", symd, sfile, nymd, nfile)
try:
df0 = df.DataFile(symd, sfile, fileType)
df1 = df.DataFile(nymd, nfile, fileType)
except FilenameMatchError as fe:
WhfLog.debug("Cannot update due to %s", fe)
except InvalidArgumentError as ie:
WhfLog.debug("Cannot update due to %s", ie)
if (df0._time.inputIsNewerIssueHour(df1._time)):
WhfLog.debug("%s Issue hour has increased, purge now", fileType)
self._update(df1._time, hoursBack, fileType)
def debugPrint(self):
""" Debug logging of content
"""
WhfLog.debug("%s data = %s", self._dataType, self._dataDir)
WhfLog.debug("%s source = %s", self._dataType, self._sourceDataDir)
WhfLog.debug("%s statefile = %s", self._dataType, self._stateFile)
WhfLog.debug("%s format = %s", self._dataType, self._format)
def debugPrint(self):
""" Debug logging of content
"""
WhfLog.debug("Parms: CFS_data = %s", self._cfsDir)
WhfLog.debug("Parms: CFS_num_ensembles = %d", self._cfsNumEnsemble)
WhfLog.debug("Parms: MaxFcstHourCfs = %d", self._maxFcstHourCfs)
WhfLog.debug("Parms: StateFile = %s", self._stateFile)
def setCurrentModelAvailability(self, parms, config):
""" Change availability status when appropriate by looking at disk
Parameters
----------
parms : Parms
parameters
config : str
Config file name
Returns
-------
none
"""
if (self._layered):
# no need to do more, already layered
return
if (self._empty):
WhfLog.error("Empty when not expected")
return
# make note if going from nothing to something
nothing = True
for f in self._step:
if (f._layered):
nothing = False
#if (nothing):
#WhfLog.debug("Nothing, so trying to get stuff")
# first time only, try the -1 and -2 steps, force with what we have
if (self._first):
self._step[2].forceLayer(parms, config, self._issue)
self._step[1].forceLayer(parms, config, self._issue)
self._first = False
self._step[0].layerIfReady(parms, config, self._issue)
self._layered = self._step[0]._layered
if (not self._layered):
tnow = datetime.datetime.utcnow()
diff = tnow - self._clockTime
idiff = (diff.microseconds +
(diff.seconds + diff.days*24*3600)*10**6)/10**6
if (idiff > parms._veryLateSeconds):
WhfLog.warning("WARNING: Inputs for layering timeout Issue:%s",
self._issue.strftime("%Y%m%d%H"))
self._step[0].forceLayer(parms, config, self._issue)
self._layered = True
def debugPrint(self):
""" Debug logging of content
"""
WhfLog.debug("Parms: CFS_data = %s", self._cfsDir)
WhfLog.debug("Parms: CFS_num_ensembles = %d", self._cfsNumEnsemble)
WhfLog.debug("Parms: MaxFcstHourCfs = %d", self._maxFcstHourCfs)
WhfLog.debug("Parms: StateFile = %s", self._stateFile)
def _analyzeNewest(self, dataNewest, stateNewest, hoursBack, fileType):
if (dataNewest <= stateNewest):
return
# see if issue time has increased and if so, purge old stuff
# create DataFile objects, which requires breaking the full
# file into yymmdd/filename
sind = stateNewest.find('/')
if (sind < 0):
raise FileNameMatchError('Cannot parse directory from ' + stateNewest)
nind = dataNewest.find('/')
if (nind < 0):
raise FileNameMatchError('Cannot parse directory from ' + dataNewest)
symd = stateNewest[:sind]
sfile = stateNewest[sind+1:]
nymd = dataNewest[:nind]
nfile = dataNewest[nind+1:]
WhfLog.debug("Checking %s / %s against %s / %s", symd, sfile, nymd, nfile)
try:
df0 = df.DataFile(symd, sfile, fileType)
df1 = df.DataFile(nymd, nfile, fileType)
except FilenameMatchError as fe:
WhfLog.debug("Cannot update due to %s", fe)
except InvalidArgumentError as ie:
WhfLog.debug("Cannot update due to %s", ie)
if (df0._time.inputIsNewerIssueHour(df1._time)):
WhfLog.debug("%s Issue hour has increased, purge now", fileType)
self._update(df1._time, hoursBack, fileType)
def setCurrentModelAvailability(self, parms, config):
""" Change availability status when appropriate by looking at disk
Parameters
----------
parms : Parms
parameters
config : str
Config file name
Returns
-------
none
"""
if (self._layered):
# no need to do more, already layered
return
if (self._empty):
WhfLog.error("Empty when not expected")
return
# make note if going from nothing to something
nothing = True
for f in self._step:
if (f._layered):
nothing = False
#if (nothing):
#WhfLog.debug("Nothing, so trying to get stuff")
# first time only, try the -1 and -2 steps, force with what we have
if (self._first):
self._step[2].forceLayer(parms, config, self._issue)
self._step[1].forceLayer(parms, config, self._issue)
self._first = False
self._step[0].layerIfReady(parms, config, self._issue)
self._layered = self._step[0]._layered
if (not self._layered):
tnow = datetime.datetime.utcnow()
diff = tnow - self._clockTime
idiff = (diff.microseconds +
(diff.seconds + diff.days * 24 * 3600) * 10**6) / 10**6
if (idiff > parms._veryLateSeconds):
WhfLog.warning("WARNING: Inputs for layering timeout Issue:%s",
self._issue.strftime("%Y%m%d%H"))
self._step[0].forceLayer(parms, config, self._issue)
self._layered = True
def debugPrint(self, name):
"""WhfLog.debug call with description of this data
Parameters
----------
name: string
name that is included in the description
Returns
-------
none
"""
WhfLog.debug("%s[%s]=%s, %s, %s", name, self._yyyymmddDir,
self._name, self._fileType,
self._time.debugString())
def debugPrint(self, name):
"""WhfLog.debug call with description of this data
Parameters
----------
name: string
name that is included in the description
Returns
-------
none
"""
WhfLog.debug("%s[%s]=%s, %s, %s", name, self._yyyymmddDir,
self._name, self._fileType,
self._time.debugString())
def setCurrentModelAvailability(self, parms, model):
""" Change availability status when appropriate by looking at disk
Parameters
----------
parms : Parms
parameters
model : Model
overall status for this model run, used for clock time
Returns
-------
none
"""
if (self._layered):
# no need to do more, already layered
return
# make note if going from nothing to something
nothing = (not self._hrrr) and (not self._rap)
#if (nothing):
#WhfLog.debug("Nothing, so trying to get stuff")
if (not self._hrrr):
# update HRRR status
self._hrrr = self._forecastExists(parms._hrrrDir)
if (not self._rap):
# update RAP status
self._rap = self._forecastExists(parms._rapDir)
if (nothing and (self._hrrr or self._rap)):
# went from nothing to something, so start the clock
WhfLog.debug("Starting clock now, hrrr=%d, rap=%d",
df.boolToInt(self._hrrr), df.boolToInt(self._rap))
self._clockTime = datetime.datetime.utcnow()
else:
if (nothing and ((not self._hrrr) and (not self._rap))):
# nothing to nothing, compare current time to time from
# model input, and squeaky wheel if too long
tnow = datetime.datetime.utcnow()
diff = tnow - model._clockTime
idiff = diff.total_seconds()
if (idiff > parms._veryLateSeconds):
WhfLog.warning(
"Inputs for short range layering are very late Issue:%s Valid:%s",
self._issue.strftime("%Y%m%d%H"),
self._valid.strftime("%Y%m%d%H"))
def layer(parms, itime, step, which, config):
""" Perform layering
NOTE: here is where returns status will be added and used
Parameters
----------
parms : Parms
parameters
"""
WhfLog.debug("LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"),
step, which)
aaf.anal_assim_layer(itime.strftime('%Y%m%d%H'), '-%01d'%(step), which, config)
WhfLog.debug("DONE LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"),
step, which)
def setCurrentModelAvailability(self, parms, model):
""" Change availability status when appropriate by looking at disk
Parameters
----------
parms : Parms
parameters
model : Model
overall status for this model run, used for clock time
Returns
-------
none
"""
if (self._layered):
# no need to do more, already layered
return
# make note if going from nothing to something
nothing = (not self._hrrr) and (not self._rap)
#if (nothing):
#WhfLog.debug("Nothing, so trying to get stuff")
if (not self._hrrr):
# update HRRR status
self._hrrr = self._forecastExists(parms._hrrrDir)
if (not self._rap):
# update RAP status
self._rap = self._forecastExists(parms._rapDir)
if (nothing and (self._hrrr or self._rap)):
# went from nothing to something, so start the clock
WhfLog.debug("Starting clock now, hrrr=%d, rap=%d", df.boolToInt(self._hrrr),
df.boolToInt(self._rap))
self._clockTime = datetime.datetime.utcnow()
else:
if (nothing and ((not self._hrrr) and (not self._rap))):
# nothing to nothing, compare current time to time from
# model input, and squeaky wheel if too long
tnow = datetime.datetime.utcnow()
diff = tnow - model._clockTime
idiff = diff.total_seconds()
if (idiff > parms._veryLateSeconds):
WhfLog.warning("Inputs for short range layering are very late Issue:%s Valid:%s",
self._issue.strftime("%Y%m%d%H"),
self._valid.strftime("%Y%m%d%H"))
def _layer(self, parms, configFile):
""" Perform layering
NOTE: here is where returns status will be added and used
Parameters
----------
parms : Parms
parameters
configFile : string
name of file with settings
"""
path = self._issue.strftime("%Y%m%d%H") + "/"
path += self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
WhfLog.info("LAYERING %s ", path)
srf.forcing(configFile, 'layer', 'HRRR', path, 'RAP', path)
WhfLog.info("DONE LAYERING file=%s", path)
def setCurrentModelAvailability(self, parms):
""" Change availability status when appropriate by looking at disk
Parameters
----------
parms : Parms
Params
"""
for f in self._fState:
# find matching model
didSet = False
for m in self._model:
if (f.matches(m)):
didSet = True
# need to pass that model in to check for 'very late'
f.setCurrentModelAvailability(parms, m)
if (not didSet):
WhfLog.error("No matching model for forecast")
def layer(parms, itime, step, which, config):
""" Perform layering
NOTE: here is where returns status will be added and used
Parameters
----------
parms : Parms
parameters
"""
WhfLog.debug("LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"), step,
which)
aaf.anal_assim_layer(itime.strftime('%Y%m%d%H'), '-%01d' % (step), which,
config)
WhfLog.debug("DONE LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"), step,
which)
def process(configFile, parm):
'''Moves all the files from the source directory to the destination directory.
Args:
fileType (string) :
source_dir (string): Full path to the source directory
destination_dir (string): Full path to the destination directory
extension (string): File extension of files to be moved
Returns:
None
'''
# initialize the logging first
parser = SafeConfigParser()
parser.read(configFile)
WhfLog.init(parser, 'RegridMRMS', True) # just use any valid 2nd arg
for day in parm._days:
processDay(configFile, day, parm)
def process(configFile, parm):
'''Moves all the files from the source directory to the destination directory.
Args:
fileType (string) :
source_dir (string): Full path to the source directory
destination_dir (string): Full path to the destination directory
extension (string): File extension of files to be moved
Returns:
None
'''
# initialize the logging first
parser = SafeConfigParser()
parser.read(configFile)
WhfLog.init(parser, 'RegridMRMS', True) # just use any valid 2nd arg
for day in parm._days:
processDay(configFile, day, parm)
def setCurrentModelAvailability(self, parms):
""" Change availability status when appropriate by looking at disk
Parameters
----------
parms : Parms
Params
"""
for f in self._fState:
# find matching model
didSet = False
for m in self._model:
if (f.matches(m)):
didSet = True
# need to pass that model in to check for 'very late'
f.setCurrentModelAvailability(parms, m)
if (not didSet):
WhfLog.error("No matching model for forecast")
def _layer(self, parms, configFile):
""" Perform layering
NOTE: here is where returns status will be added and used
Parameters
----------
parms : Parms
parameters
configFile : string
name of file with settings
"""
path = self._issue.strftime("%Y%m%d%H") + "/"
path += self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc"
WhfLog.info("LAYERING %s ", path)
srf.forcing(configFile, 'layer', 'HRRR', path, 'RAP', path)
WhfLog.info("DONE LAYERING file=%s", path)
def createStateFile(parms):
""" Called if there is no state file, look at data dirs and create state
Parameters
----------
parms: Parms
Parameter settings
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
for f in cfs._content:
f.debugPrint("Newest files: CFS")
state = State("")
state.initialize(cfs)
# maybe back up and regrid that entire issue time
# maybe redo this exact set of inputs only
# maybe do nothing
# maybe do all of them..for now do nothing as its easiest, just move on
#files = hrrr.getFnames()
#for f in files:
# regridHRRR(f)
#files = rap.getFnames()
#for f in files:
# regridRAP(f)
#files = mrms.getFnames()
#for f in files:
# regridMRMS(f)
# write out file
state.write(parms._stateFile)
def createStateFile(parms):
""" Called if there is no state file, look at data dirs and create state
Parameters
----------
parms: Parms
Parameter settings
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
for f in cfs._content:
f.debugPrint("Newest files: CFS")
state = State("")
state.initialize(cfs)
# maybe back up and regrid that entire issue time
# maybe redo this exact set of inputs only
# maybe do nothing
# maybe do all of them..for now do nothing as its easiest, just move on
#files = hrrr.getFnames()
#for f in files:
# regridHRRR(f)
#files = rap.getFnames()
#for f in files:
# regridRAP(f)
#files = mrms.getFnames()
#for f in files:
# regridMRMS(f)
# write out file
state.write(parms._stateFile)
def parmRead(fname):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "AnalysisAssimLayeringDriver"
WhfLog.init(parser, forcing_config_label, 'AA', 'Layer', 'RAP/HRRR/MRMS')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
hrrr0hrDir = parser.get('downscaling',
'HRRR_finished_output_dir_0hr') # placeholder
mrmsDir = parser.get('regridding',
'MRMS_finished_output_dir') # maybe incorrect
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
rap0hrDir = parser.get('downscaling',
'RAP_finished_output_dir_0hr') # placeholder
layerDir = parser.get('layering', 'analysis_assimilation_output')
maxFcstHour = 3
hoursBack = 5 # 3 hours back at 0, -1, and -2 (-2 -3 = -5)
maxWaitMinutes = 15 #int(parser.get('triggering',
# 'short_range_fcst_max_wait_minutes'))
veryLateMinutes = 20 #int(parser.get('triggering',
# 'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'analysis_assim_layering_state_file')
parms = Parms(hrrrDir, hrrr0hrDir, rapDir, rap0hrDir, mrmsDir, layerDir,
maxFcstHour, hoursBack, maxWaitMinutes, veryLateMinutes,
stateFile)
return parms
def parmRead(fname, fileType):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
fileType: str
'RAP', 'HRRR', 'MRMS', 'GFS'
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "Regrid" + fileType
if (fileType == 'HRRR'):
configType = 'Short'
elif (fileType == 'RAP'):
configType = 'Short'
elif (fileType == 'GFS'):
configType = 'Medium'
elif (fileType == 'MRMS'):
configType = 'AA'
else:
configType = '???'
WhfLog.init(parser, forcing_config_label, configType, 'Regrid', fileType)
dataDir = parser.get('data_dir', fileType + '_data')
maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max'))
hoursBack = int(parser.get('triggering', fileType + '_hours_back'))
stateFile = parser.get('triggering', fileType + '_regrid_state_file')
parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile)
return parms
def parmRead(fname, fileType):
"""Read in the main config file, return needed parameters
Parameters
----------
fname: str
name of parameter file to read in
fileType: str
'RAP', 'HRRR', 'MRMS', 'GFS'
Returns
-------
Parms
values as pulled from the file
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "Regrid" + fileType
if (fileType == 'HRRR'):
configType = 'Short'
elif (fileType == 'RAP'):
configType = 'Short'
elif (fileType == 'GFS'):
configType = 'Medium'
elif (fileType == 'MRMS'):
configType = 'AA'
else:
configType = '???'
WhfLog.init(parser, forcing_config_label, configType, 'Regrid', fileType)
dataDir = parser.get('data_dir', fileType + '_data')
maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max'))
hoursBack = int(parser.get('triggering', fileType + '_hours_back'))
stateFile = parser.get('triggering', fileType + '_regrid_state_file')
parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile)
return parms
def main(argv):
# User must pass the config file into the main driver.
configFile = argv[0]
if not os.path.exists(configFile):
print 'ERROR forcing engine config file not found.'
return 1
# read in fixed main params
parms = parmRead(configFile)
#parms.debugPrint()
#if there is not a state file, create one now using newest
if (not os.path.exists(parms._stateFile)):
parms.debugPrint()
createStateFile(parms)
# begin normal processing situation
WhfLog.debug("....Check for new input data to regid")
# read in state
state = State(parms._stateFile)
if state.isEmpty():
# error return here
return 0
#state.debugPrint()
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
# Same with CFS
toProcess = state.updateWithNew(cfs, parms._hoursBackCfs)
for f in toProcess:
regridCFS(configFile, f)
# write out state and exit
#state.debugPrint()
state.write(parms._stateFile)
return 0
def olderIssueHour(self, hoursBack):
""" Return the ForecastTime that has hoursBack hours earlier issue time
Parameters
----------
hoursBack: int
Number of hours back to look
Returns
-------
ForecastTime
Forecast with issue time hoursBack earlier than local, forecastHour=0
"""
if (self.isEmpty()):
WhfLog.debug("ERROR empty input to olderIssueHour")
else:
timeBack = self.ymdh() - datetime.timedelta(hours=hoursBack)
self._fcstTime = datetime.datetime(timeBack.year, timeBack.month,
timeBack.day, 0, 0, 0)
self._issueHour = timeBack.hour
self._forecastHour = 0
def olderIssueHour(self, hoursBack):
""" Return the ForecastTime that has hoursBack hours earlier issue time
Parameters
----------
hoursBack: int
Number of hours back to look
Returns
-------
ForecastTime
Forecast with issue time hoursBack earlier than local, forecastHour=0
"""
if (self.isEmpty()):
WhfLog.debug("ERROR empty input to olderIssueHour")
else:
timeBack = self.ymdh() - datetime.timedelta(hours=hoursBack)
self._fcstTime = datetime.datetime(timeBack.year, timeBack.month,
timeBack.day, 0, 0, 0)
self._issueHour = timeBack.hour
self._forecastHour = 0
def run(fileType, configFile, realtime):
""" Run the script, process any new data
Parameters
----------
fileType: str
'HRRR', ...
configFile : str
Name of the file with settings
realtime : boolean
True if this is realtime
Returns
-------
1 for error, 0 for success
"""
good = False
regriddable = ['HRRR', 'RAP', 'MRMS', 'GFS']
if (fileType not in regriddable):
print 'ERROR unknown file type command arg ', fileType
return 1
# User must pass the config file into the main driver.
if not os.path.exists(configFile):
print 'ERROR forcing engine config file not found:', configFile
return 1
# read in fixed main params
parms = parmRead(configFile, fileType, realtime)
#if there is not a state file, create one now using newest
if (not os.path.exists(parms._stateFile)):
parms.debugPrint()
createStateFile(parms, fileType, realtime)
# read in state
state = State(parms._stateFile, fileType)
# query each directory and get newest model run file for each, then
# get all for that and previous issue time
data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType)
data.setNewestFiles(parms._hoursBack)
# Update the state to reflect changes, returning those files to regrid
# Regrid 'em
toProcess = state.lookForNew(data, parms._hoursBack, fileType)
for f in toProcess:
try:
regrid(f, fileType, configFile);
except:
WhfLog.error("Could not regrid/downscale %s", f)
else:
WhfLog.debug("Adding new file %s, and writing state file", f)
if (not state.addFileIfNew(f)):
WhfLog.error("File %s was not new", f)
else:
state.write(parms._stateFile, fileType);
# write out state (in case it has not been written yet) and exit
#state.debugPrint()
state.write(parms._stateFile, fileType)
return 0
def parmRead(fname):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
forcing_config_label = "ShortRangeLayeringDriver"
WhfLog.init(parser, forcing_config_label, 'Short', 'Layer', 'RAP/HRRR')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
#mrmsDir = parser.get('data_dir', 'MRMS_data')
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
layerDir = parser.get('layering', 'short_range_output')
maxFcstHour = int(parser.get('fcsthr_max', 'HRRR_fcsthr_max'))
# go with HRRR
hoursBack = int(parser.get('triggering', 'HRRR_hours_back'))
maxWaitMinutes=int(parser.get('triggering',
'short_range_fcst_max_wait_minutes'))
veryLateMinutes=int(parser.get('triggering',
'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'short_range_layering_state_file')
parms = Parms(hrrrDir, rapDir, layerDir, maxFcstHour, hoursBack,
maxWaitMinutes, veryLateMinutes, stateFile)
return parms
def parmRead(fname, realtime):
""" Read in a param file
Parameters
---------
fname : str
Name of the file to read
realtime: boolean
True if realtime, False for archive mode
Returns
-------
Parms
The parameters that were read in
"""
parser = SafeConfigParser()
parser.read(fname)
if (realtime):
WhfLog.init(parser, 'ShortLayer', False)
else:
WhfLog.set('ShortLayer')
hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir')
rapDir = parser.get('downscaling', 'RAP_finished_output_dir')
layerDir = parser.get('layering', 'short_range_output')
maxFcstHour = int(parser.get('fcsthr_max', 'HRRR_fcsthr_max'))
# go with HRRR
hoursBack = int(parser.get('triggering', 'HRRR_hours_back'))
maxWaitMinutes = int(
parser.get('triggering', 'short_range_fcst_max_wait_minutes'))
veryLateMinutes = int(
parser.get('triggering', 'short_range_fcst_very_late_minutes'))
stateFile = parser.get('triggering', 'short_range_layering_state_file')
parms = Parms(hrrrDir, rapDir, layerDir, maxFcstHour, hoursBack,
maxWaitMinutes, veryLateMinutes, stateFile)
return parms
def makeDirIfNeeded(path):
""" Input is a to be a directory, make it if it does not exist.
Parameters
----------
path : str
full path
Returns
-------
bool
True if directory exists, or was created, false if error
"""
try:
os.makedirs(path)
return True
except OSError as exception:
if exception.errno != errno.EEXIST:
WhfLog.error("ERROR creating %s", path)
return False
else:
return True
