def createStateFile(parms, fileType, realtime): """ Called if there is no state file, look at data dirs and create state in realtime, in non-realtime create an empty state. Write to file. Parameters ---------- parms: Parms Parameter settings fileType: str 'HRRR', ... realtime: boolean True if realtime, False for archive mode Returns ------- none Writes out the state file after creating it """ WhfLog.info("Initializing") state = State("") if (realtime): # query each directory and get newest model run file for each, then # get all for that and previous issue time, this becomes state that # is not re-processed, we only look for new stuff data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType) data.setNewestFiles(parms._hoursBack) for f in data._content: f.debugPrint("Newest files: " + fileType) state.initialize(data) # write out file (at least try to) state.write(parms._stateFile, fileType)
def run(fileType, configFile, realtime): """ Run the script, process any new data Parameters ---------- fileType: str 'HRRR', ... configFile : str Name of the file with settings realtime : boolean True if this is realtime Returns ------- 1 for error, 0 for success """ good = False regriddable = ['HRRR', 'RAP', 'MRMS', 'GFS'] if (fileType not in regriddable): print 'ERROR unknown file type command arg ', fileType return 1 # User must pass the config file into the main driver. if not os.path.exists(configFile): print 'ERROR forcing engine config file not found:', configFile return 1 # read in fixed main params parms = parmRead(configFile, fileType, realtime) #if there is not a state file, create one now using newest if (not os.path.exists(parms._stateFile)): parms.debugPrint() createStateFile(parms, fileType, realtime) # read in state state = State(parms._stateFile, fileType) # query each directory and get newest model run file for each, then # get all for that and previous issue time data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType) data.setNewestFiles(parms._hoursBack) # Update the state to reflect changes, returning those files to regrid # Regrid 'em toProcess = state.lookForNew(data, parms._hoursBack, fileType) for f in toProcess: try: regrid(f, fileType, configFile); except: WhfLog.error("Could not regrid/downscale %s", f) else: WhfLog.debug("Adding new file %s, and writing state file", f) if (not state.addFileIfNew(f)): WhfLog.error("File %s was not new", f) else: state.write(parms._stateFile, fileType); # write out state (in case it has not been written yet) and exit #state.debugPrint() state.write(parms._stateFile, fileType) return 0
def run(configFile, realtime): if not os.path.exists(configFile): print 'ERROR forcing engine config file not found.' return 1 # read in fixed main params parms = parmRead(configFile, realtime) #parms.debugPrint() #if there is not a state file, create one now using newest if (not os.path.exists(parms._stateFile)): parms.debugPrint() createStateFile(parms, realtime) # begin normal processing situation WhfLog.debug("....Check for new input data to regid") # read in state state = State(parms._stateFile) # query directory and get newest model run file, then # get all for that and previous issue time cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS") cfs.setNewestFiles(parms._hoursBackCfs) # Same with CFS toProcess = state.updateWithNew(cfs, parms._hoursBackCfs) for f in toProcess: regridCFS(configFile, f) # write out state and exit #state.debugPrint() state.write(parms._stateFile) return 0
def createStateFile(parms, realtime): """ Called if there is no state file, look at data dirs and create state Parameters ---------- parms: Parms Parameter settings Returns ------- none Writes out the state file after creating it """ WhfLog.info("Initializing") state = State("") if (realtime): # query directory and get newest model run file, then # get all for that and previous issue time cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS") cfs.setNewestFiles(parms._hoursBackCfs) for f in cfs._content: f.debugPrint("Newest files: CFS") state.initialize(cfs) # write out file state.write(parms._stateFile)
def main(argv): fileType = argv[0] good = False if (fileType == 'HRRR' or fileType == 'RAP' or fileType == 'MRMS' or fileType == 'GFS'): good = True if (not good): print 'ERROR unknown file type command arg ', fileType return 1 # User must pass the config file into the main driver. configFile = argv[1] if not os.path.exists(configFile): print 'ERROR forcing engine config file not found:', configFile return 1 # read in fixed main params parms = parmRead(configFile, fileType) #parms.debugPrint() #if there is not a state file, create one now using newest if (not os.path.exists(parms._stateFile)): parms.debugPrint() createStateFile(parms, fileType) # begin normal processing situation #WhfLog.debug("....Check for new input data to regid") # read in state state = State(parms._stateFile, fileType) if state.isEmpty(): # error return here return 0 #state.debugPrint() # query each directory and get newest model run file for each, then # get all for that and previous issue time data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType) data.setNewestFiles(parms._hoursBack) # Update the state to reflect changes, returning those files to regrid # Regrid 'em toProcess = state.updateWithNew(data, parms._hoursBack, fileType) for f in toProcess: regrid(f, fileType, configFile) # write out state and exit #state.debugPrint() state.write(parms._stateFile, fileType) return 0
def createStateFile(parms): """ Called if there is no state file, look at data dirs and create state Parameters ---------- parms: Parms Parameter settings Returns ------- none Writes out the state file after creating it """ WhfLog.info("Initializing") # query directory and get newest model run file, then # get all for that and previous issue time cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS") cfs.setNewestFiles(parms._hoursBackCfs) for f in cfs._content: f.debugPrint("Newest files: CFS") state = State("") state.initialize(cfs) # maybe back up and regrid that entire issue time # maybe redo this exact set of inputs only # maybe do nothing # maybe do all of them..for now do nothing as its easiest, just move on #files = hrrr.getFnames() #for f in files: # regridHRRR(f) #files = rap.getFnames() #for f in files: # regridRAP(f) #files = mrms.getFnames() #for f in files: # regridMRMS(f) # write out file state.write(parms._stateFile)
def createStateFile(parms, fileType): """ Called if there is no state file, look at data dirs and create state Parameters ---------- parms: Parms Parameter settings fileType: str 'HRRR', ... Returns ------- none Writes out the state file after creating it """ WhfLog.info("Initializing") # query each directory and get newest model run file for each, then # get all for that and previous issue time data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType) data.setNewestFiles(parms._hoursBack) for f in data._content: f.debugPrint("Newest files: " + fileType) state = State("") state.initialize(data, fileType) # maybe back up and regrid that entire issue time # maybe redo this exact set of inputs only # maybe do nothing # maybe do all of them..for now do nothing as its easiest, just move on # write out file (at least try to) state.write(parms._stateFile, fileType)