def createStateFile(parms, fileType, realtime):
""" Called if there is no state file, look at data dirs and create state
in realtime, in non-realtime create an empty state. Write to file.
Parameters
----------
parms: Parms
Parameter settings
fileType: str
'HRRR', ...
realtime: boolean
True if realtime, False for archive mode
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
state = State("")
if (realtime):
# query each directory and get newest model run file for each, then
# get all for that and previous issue time, this becomes state that
# is not re-processed, we only look for new stuff
data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType)
data.setNewestFiles(parms._hoursBack)
for f in data._content:
f.debugPrint("Newest files: " + fileType)
state.initialize(data)
# write out file (at least try to)
state.write(parms._stateFile, fileType)
def run(fileType, configFile, realtime):
""" Run the script, process any new data
Parameters
----------
fileType: str
'HRRR', ...
configFile : str
Name of the file with settings
realtime : boolean
True if this is realtime
Returns
-------
1 for error, 0 for success
"""
good = False
regriddable = ['HRRR', 'RAP', 'MRMS', 'GFS']
if (fileType not in regriddable):
print 'ERROR unknown file type command arg ', fileType
return 1
# User must pass the config file into the main driver.
if not os.path.exists(configFile):
print 'ERROR forcing engine config file not found:', configFile
return 1
# read in fixed main params
parms = parmRead(configFile, fileType, realtime)
#if there is not a state file, create one now using newest
if (not os.path.exists(parms._stateFile)):
parms.debugPrint()
createStateFile(parms, fileType, realtime)
# read in state
state = State(parms._stateFile, fileType)
# query each directory and get newest model run file for each, then
# get all for that and previous issue time
data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType)
data.setNewestFiles(parms._hoursBack)
# Update the state to reflect changes, returning those files to regrid
# Regrid 'em
toProcess = state.lookForNew(data, parms._hoursBack, fileType)
for f in toProcess:
try:
regrid(f, fileType, configFile);
except:
WhfLog.error("Could not regrid/downscale %s", f)
else:
WhfLog.debug("Adding new file %s, and writing state file", f)
if (not state.addFileIfNew(f)):
WhfLog.error("File %s was not new", f)
else:
state.write(parms._stateFile, fileType);
# write out state (in case it has not been written yet) and exit
#state.debugPrint()
state.write(parms._stateFile, fileType)
return 0
def run(configFile, realtime):
if not os.path.exists(configFile):
print 'ERROR forcing engine config file not found.'
return 1
# read in fixed main params
parms = parmRead(configFile, realtime)
#parms.debugPrint()
#if there is not a state file, create one now using newest
if (not os.path.exists(parms._stateFile)):
parms.debugPrint()
createStateFile(parms, realtime)
# begin normal processing situation
WhfLog.debug("....Check for new input data to regid")
# read in state
state = State(parms._stateFile)
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
# Same with CFS
toProcess = state.updateWithNew(cfs, parms._hoursBackCfs)
for f in toProcess:
regridCFS(configFile, f)
# write out state and exit
#state.debugPrint()
state.write(parms._stateFile)
return 0
def createStateFile(parms, realtime):
""" Called if there is no state file, look at data dirs and create state
Parameters
----------
parms: Parms
Parameter settings
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
state = State("")
if (realtime):
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
for f in cfs._content:
f.debugPrint("Newest files: CFS")
state.initialize(cfs)
# write out file
state.write(parms._stateFile)
def main(argv):
fileType = argv[0]
good = False
if (fileType == 'HRRR' or fileType == 'RAP' or fileType == 'MRMS' or
fileType == 'GFS'):
good = True
if (not good):
print 'ERROR unknown file type command arg ', fileType
return 1
# User must pass the config file into the main driver.
configFile = argv[1]
if not os.path.exists(configFile):
print 'ERROR forcing engine config file not found:', configFile
return 1
# read in fixed main params
parms = parmRead(configFile, fileType)
#parms.debugPrint()
#if there is not a state file, create one now using newest
if (not os.path.exists(parms._stateFile)):
parms.debugPrint()
createStateFile(parms, fileType)
# begin normal processing situation
#WhfLog.debug("....Check for new input data to regid")
# read in state
state = State(parms._stateFile, fileType)
if state.isEmpty():
# error return here
return 0
#state.debugPrint()
# query each directory and get newest model run file for each, then
# get all for that and previous issue time
data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType)
data.setNewestFiles(parms._hoursBack)
# Update the state to reflect changes, returning those files to regrid
# Regrid 'em
toProcess = state.updateWithNew(data, parms._hoursBack, fileType)
for f in toProcess:
regrid(f, fileType, configFile)
# write out state and exit
#state.debugPrint()
state.write(parms._stateFile, fileType)
return 0
def createStateFile(parms):
""" Called if there is no state file, look at data dirs and create state
Parameters
----------
parms: Parms
Parameter settings
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
# query directory and get newest model run file, then
# get all for that and previous issue time
cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS")
cfs.setNewestFiles(parms._hoursBackCfs)
for f in cfs._content:
f.debugPrint("Newest files: CFS")
state = State("")
state.initialize(cfs)
# maybe back up and regrid that entire issue time
# maybe redo this exact set of inputs only
# maybe do nothing
# maybe do all of them..for now do nothing as its easiest, just move on
#files = hrrr.getFnames()
#for f in files:
# regridHRRR(f)
#files = rap.getFnames()
#for f in files:
# regridRAP(f)
#files = mrms.getFnames()
#for f in files:
# regridMRMS(f)
# write out file
state.write(parms._stateFile)
def createStateFile(parms, fileType):
""" Called if there is no state file, look at data dirs and create state
Parameters
----------
parms: Parms
Parameter settings
fileType: str
'HRRR', ...
Returns
-------
none
Writes out the state file after creating it
"""
WhfLog.info("Initializing")
# query each directory and get newest model run file for each, then
# get all for that and previous issue time
data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType)
data.setNewestFiles(parms._hoursBack)
for f in data._content:
f.debugPrint("Newest files: " + fileType)
state = State("")
state.initialize(data, fileType)
# maybe back up and regrid that entire issue time
# maybe redo this exact set of inputs only
# maybe do nothing
# maybe do all of them..for now do nothing as its easiest, just move on
# write out file (at least try to)
state.write(parms._stateFile, fileType)
