def createStateFile(parms, realtime): """ Called if there is no state file, look at data dirs and create state Parameters ---------- parms: Parms Parameter settings Returns ------- none Writes out the state file after creating it """ WhfLog.info("Initializing") state = State("") if (realtime): # query directory and get newest model run file, then # get all for that and previous issue time cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS") cfs.setNewestFiles(parms._hoursBackCfs) for f in cfs._content: f.debugPrint("Newest files: CFS") state.initialize(cfs) # write out file state.write(parms._stateFile)
def run(configFile, realtime): if not os.path.exists(configFile): print 'ERROR forcing engine config file not found.' return 1 # read in fixed main params parms = parmRead(configFile, realtime) #parms.debugPrint() #if there is not a state file, create one now using newest if (not os.path.exists(parms._stateFile)): parms.debugPrint() createStateFile(parms, realtime) # begin normal processing situation WhfLog.debug("....Check for new input data to regid") # read in state state = State(parms._stateFile) # query directory and get newest model run file, then # get all for that and previous issue time cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS") cfs.setNewestFiles(parms._hoursBackCfs) # Same with CFS toProcess = state.updateWithNew(cfs, parms._hoursBackCfs) for f in toProcess: regridCFS(configFile, f) # write out state and exit #state.debugPrint() state.write(parms._stateFile) return 0
def layerIfReady(self, parms, configFile): """ Perform layering if state is such that it should be done Parameters ---------- parms : Parms Parameters configFile : string name of file with settings Returns ------- bool True if layering was done, or had previously been done """ if (self._layered == 1): return 1 if (self._hrrr == 0 and self._rap == 0): return 0 if (self._hrrr == 1 and self._rap == 1): self.layer(parms, configFile) self._layered = 1 return 1 if (self._hrrr == 0 and self._rap == 1): ntime = datetime.datetime.utcnow() diff = ntime - self._clockTime idiff = diff.total_seconds() if (idiff > parms._maxWaitSeconds): WhfLog.debug("timeout..Should layer, dt=%d", idiff) self.passthroughRap(parms) self._layered = 1 return 1 return 0
def regridIfZeroHr(configFile, fileType, fname): """If it is a 0 hour forecast (RAP or HRRR) regrid in a special way Parameters ---------- configFile : str configuration file with all settings fileType: str HRRR, RAP, ... string fname: str name of file to regrid and downscale, with yyyymmdd parent dir Returns ------- None """ # check for 0 hour by creating a DataFile and checking forecast hour try: f = df.DataFile(fname[0:8], fname[9:], fileType) except FilenameMatchError as fe: WhfLog.debug("Cannot check for 0 hour data due to %s", fe) raise except InvalidArgumentError as ie: WhfLog.debug("Cannot check for 0 hour data due to %s", ie) raise if (f._time._forecastHour == 0): WhfLog.setConfigType('AA') WhfLog.debug("SPECIAL 0 hour case %s", fname[9:0]) aaf.forcing(configFile, 'regrid', fileType, fname[9:]) WhfLog.setConfigType('Short')
def parmRead(fname, fileType, realtime): """Read in the main config file, return needed parameters Parameters ---------- fname: str name of parameter file to read in fileType: str 'RAP', 'HRRR', 'MRMS', 'GFS' realtime: boolean True if realtime, False for archive mode Returns ------- Parms values as pulled from the file """ parser = SafeConfigParser() parser.read(fname) label = 'Regrid' + fileType if (realtime): WhfLog.init(parser, label, False) else: WhfLog.set(label) dataDir = parser.get('data_dir', fileType + '_data') maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max')) hoursBack = int(parser.get('triggering', fileType + '_hours_back')) stateFile = parser.get('triggering', fileType + '_regrid_state_file') parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile) return parms
def obsExists(dir, issueTime): """ Check if obs exists Parameters ---------- dir : str Full path to the MRMS directories issueTime : datetime The issue time (y,m,d,h) Returns ------- bool True if the data does exist on disk """ path = dir + "/" path += issueTime.strftime("%Y%m%d%H") if (os.path.isdir(path)): fname = issueTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc" names = df.getFileNames(path) for n in names: if (n == fname): WhfLog.debug("Found %s in %s", fname, path) return True return False
def withinNHours(self, ftime, N): """ Check if input time is within some number of issue hours of self Parameters ---------- ftime: ForecastTime N:int Number of hours to check Returns ------- True if input time - local time <= N hours """ # same day, issue hour match # same day, issue hour input - issue hour <= N if (self._fcstTime == ftime._fcstTime): if (self._issueHour == ftime._issueHour): return True elif (ftime._issueHour-self._issueHour <= N): return True else: return False else: # create a full time from local and input states timeIn = ftime.ymdh() timeLoc = self.ymdh() diff = (timeIn - timeLoc).total_seconds() maxSeconds = N*3600 if (diff < 0): WhfLog.error("Unexpected data newer than newest") return False else: return (diff <= maxSeconds)
def regridIfZeroHr(configFile, fileType, fname): """If it is a 0 hour forecast (RAP or HRRR) regrid in a special way Parameters ---------- configFile : str configuration file with all settings fileType: str HRRR, RAP, ... string fname: str name of file to regrid and downscale, with yyyymmdd parent dir Returns ------- None """ # check for 0 hour by creating a DataFile and checking forecast hour try: f = df.DataFile(fname, fileType) except FilenameMatchError as fe: WhfLog.debug("Cannot check for 0 hour data due to %s", fe) raise except InvalidArgumentError as ie: WhfLog.debug("Cannot check for 0 hour data due to %s", ie) raise if (f._time._forecastHour == 0): WhfLog.setConfigType('AA') WhfLog.debug("SPECIAL 0 hour case %s", fname[9:0]) aaf.forcing(configFile, 'regrid', 'HRRR', fname[9:]) WhfLog.setConfigType('Short')
def _forecastExists(self, dir): """ Check if forecast indicated by local state exists Parameters ---------- dir : str Full path to the issue time directories Returns ------- bool True if the forecast does exist on disk """ path = dir + "/" path += self._issue.strftime("%Y%m%d%H") if (os.path.isdir(path)): fname = self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc" names = df.getFileNames(path) for n in names: if (n == fname): WhfLog.debug("Found %s in %s", fname, path) return True return False else: return False
def lookForNew(self, data, hoursBack, fileType): """ See if new data has arrived compared to state. If a new issue time, purge older stuff from state. Parameters ---------- data: DataFiles The newest data hoursBack: int Maximum number of hours back to keep data in state fileType : str 'HRRR', 'RAP', ... Returns ------- list[str] The data file names that are are to be added to state """ ret = [] fnames = data.getFnames() if (not fnames): return ret if (self.isEmpty()): WhfLog.debug("Adding to empty list") else: sname = self.newest() if (not sname): WhfLog.error("Expected file, got none") return ret self._analyzeNewest(fnames[-1], sname, hoursBack, fileType) for f in fnames: if (self._isNew(f)): ret.append(f) return ret
def forecastExists(self, dir): """ Check if forecast indicated by local state exists Parameters ---------- dir : str Full path to the issue time directories Returns ------- bool True if the forecast does exist on disk """ path = dir + "/" path += self._issue.strftime("%Y%m%d%H") if (os.path.isdir(path)): fname = self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc" names = df.getFileNames(path) for n in names: if (n == fname): WhfLog.debug("Found %s in %s", fname, path) return 1 return 0 else: return 0
def parmRead(fname): """Read in the main config file, return needed parameters Parameters ---------- fname: str name of parameter file to read in Returns ------- Parms values as pulled from the file """ parser = SafeConfigParser() parser.read(fname) forcing_config_label = "LongRangeRegridDriver" WhfLog.init(parser, forcing_config_label, 'Long', 'Regrid', 'CFS') cfsDir = parser.get('data_dir', 'CFS_data') cfsNumEnsemble = int(parser.get('data_dir', 'CFS_num_ensemble')) maxFcstHourCfs = int(parser.get('fcsthr_max', 'CFS_fcsthr_max')) hoursBackCfs = int(parser.get('triggering', 'CFS_hours_back')) stateFile = parser.get('triggering', 'long_range_regrid_state_file') parms = Parms(cfsDir, cfsNumEnsemble, maxFcstHourCfs, hoursBackCfs, stateFile) return parms
def layerIfReady(self, parms, configFile): """ Perform layering if state is such that it should be done Parameters ---------- parms : Parms Parameters configFile : string name of file with settings Returns ------- bool True if layering was done, or had previously been done """ if (self._layered): return True if ((not self._hrrr) and (not self._rap)): return False if (self._hrrr and self._rap): self._layer(parms, configFile) self._layered = True return True if (self._rap and not self._hrrr): ntime = datetime.datetime.utcnow() diff = ntime - self._clockTime idiff = diff.total_seconds() if (idiff > parms._maxWaitSeconds): WhfLog.debug("timeout..Should layer, dt=%d", idiff) self._passthroughRap(parms) self._layered = True return True return False
def createStateFile(parms, fileType, realtime): """ Called if there is no state file, look at data dirs and create state in realtime, in non-realtime create an empty state. Write to file. Parameters ---------- parms: Parms Parameter settings fileType: str 'HRRR', ... realtime: boolean True if realtime, False for archive mode Returns ------- none Writes out the state file after creating it """ WhfLog.info("Initializing") state = State("") if (realtime): # query each directory and get newest model run file for each, then # get all for that and previous issue time, this becomes state that # is not re-processed, we only look for new stuff data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType) data.setNewestFiles(parms._hoursBack) for f in data._content: f.debugPrint("Newest files: " + fileType) state.initialize(data) # write out file (at least try to) state.write(parms._stateFile, fileType)
def forecastExists(dir, issueTime, fcstHour): """ Check if forecast exists Parameters ---------- dir : str Full path to the issue time directories issueTime : datetime The issue time (y,m,d,h) fcstHour: int should be 0 or 3 Returns ------- bool True if the forecast does exist on disk """ path = dir + "/" path += issueTime.strftime("%Y%m%d%H") if (os.path.isdir(path)): validTime = issueTime + datetime.timedelta(hours=fcstHour) fname = validTime.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc" names = df.getFileNames(path) for n in names: if (n == fname): WhfLog.debug("Found %s in %s", fname, path) return True return False
def filterWithinNHours(files, type, ftime, N): """Filter a list of file names with assumed format to those in a time range Parameters ---------- files: list[str] file names, each with parent directory: 'yyyymmdd/<file> type: str file type string, 'RAP', 'HRRR', 'MRMS', GFS' ftime: ForecastTime time to compare against, assumed most recent time N: int Number of hours back from ftime to keep Returns ------- list[str] subset of input files with issue time is in the range [ftime-N,ftime] """ ret = [] for f in files: df = DataFile(f[0:8], f[9:], type) if (df._ok): ithFtime = df._time if (ithFtime.withinNHours(ftime, N)): ret.append(f) else: WhfLog.debug("Did not append file, too old compared to %s file=%s", ftime.debugString(), f) WhfLog.debug("filtering within %d hours, input length %d output %d", N, len(files), len(ret)) return ret
def _allDataFiles(self): """Return all data files on disk, in order Parameters ---------- none Returns ------- list[DataFile] The DataFile specs, in order oldest to newest """ # get the yyyymmdd subdirs dirs = getYyyymmddSubdirectories(self._topDir) # sort them into ascending order dirs = sorted(dirs) if not dirs: # nothing there WhfLog.debug("_allDataFiles: No data in %s", self._topDir) return [] else: # make one big array ret = [] for d in dirs: ret.extend(self._allDataFilesInDir(d)) return ret
def debugPrint(self): """ WhfLog debug of content """ WhfLog.debug("Model: empty=%d", df.boolToInt(self._empty)) if (self._empty): return WhfLog.debug("Model: Issue=%s clockTime=%s", self._issue.strftime("%Y%m%d%H"), self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self): """ logging debug of content """ WhfLog.debug("FcstStep: empty=%d", self._empty) if (self._empty): return WhfLog.debug("FcstStep[%d] hrrr0:%d hrrr3:%d rap0:%d rap3:%d mrms:%d lay:%d", self._step, self._hrrr0, self._hrrr3, self._rap0, self._rap3, self._mrms, self._layered)
def debugPrint(self): """ logging debug of content """ WhfLog.debug("FcstStep: empty=%d", self._empty) if (self._empty): return WhfLog.debug( "FcstStep[%d] hrrr0:%d hrrr3:%d rap0:%d rap3:%d mrms:%d lay:%d", self._step, self._hrrr0, self._hrrr3, self._rap0, self._rap3, self._mrms, self._layered)
def debugPrint(self): """ WhfLog debug of content """ WhfLog.debug("Fcst: empty=%d", int(self._empty)) if (self._empty): return WhfLog.debug("Fcst: I:%s V:%s hrrr:%d rap:%d layered:%d clockTime=%s", self._issue.strftime("%Y%m%d%H"), self._valid.strftime("%Y%m%d%H"), df.boolToInt(self._hrrr), df.boolToInt(self._rap), df.boolToInt(self._layered), self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self): """ logging debug of content """ WhfLog.debug("Fcst: empty=%d first=%d", self._empty, self._first) if (self._empty): return WhfLog.debug( "Fcst: I:%s step[0]:%s step[1]:%s step[2]:%s layered:%d clockTime=%s", self._issue.strftime("%Y%m%d%H"), self._step[0].debugPrintString(), self._step[1].debugPrintString(), self._step[2].debugPrintString(), self._layered, self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self): """ logging debug of content """ WhfLog.debug("Fcst: empty=%d first=%d", self._empty, self._first) if (self._empty): return WhfLog.debug("Fcst: I:%s step[0]:%s step[1]:%s step[2]:%s layered:%d clockTime=%s", self._issue.strftime("%Y%m%d%H"), self._step[0].debugPrintString(), self._step[1].debugPrintString(), self._step[2].debugPrintString(), self._layered, self._clockTime.strftime("%Y-%m-%d_%H:%M:%S"))
def debugPrint(self): """ Debug logging of content """ WhfLog.debug("%s data = %s", self._dataType, self._dataDir) WhfLog.debug("%s source = %s", self._dataType, self._sourceDataDir) WhfLog.debug("%s statefile = %s", self._dataType, self._stateFile) WhfLog.debug("%s format = %s", self._dataType, self._format)
def _analyzeNewest(self, dataNewest, stateNewest, hoursBack, fileType): if (dataNewest <= stateNewest): return # see if issue time has increased and if so, purge old stuff # create DataFile objects, which requires breaking the full # file into yymmdd/filename sind = stateNewest.find('/') if (sind < 0): raise FileNameMatchError('Cannot parse directory from ' + stateNewest) nind = dataNewest.find('/') if (nind < 0): raise FileNameMatchError('Cannot parse directory from ' + dataNewest) symd = stateNewest[:sind] sfile = stateNewest[sind+1:] nymd = dataNewest[:nind] nfile = dataNewest[nind+1:] WhfLog.debug("Checking %s / %s against %s / %s", symd, sfile, nymd, nfile) try: df0 = df.DataFile(symd, sfile, fileType) df1 = df.DataFile(nymd, nfile, fileType) except FilenameMatchError as fe: WhfLog.debug("Cannot update due to %s", fe) except InvalidArgumentError as ie: WhfLog.debug("Cannot update due to %s", ie) if (df0._time.inputIsNewerIssueHour(df1._time)): WhfLog.debug("%s Issue hour has increased, purge now", fileType) self._update(df1._time, hoursBack, fileType)
def debugPrint(self): """ Debug logging of content """ WhfLog.debug("Parms: CFS_data = %s", self._cfsDir) WhfLog.debug("Parms: CFS_num_ensembles = %d", self._cfsNumEnsemble) WhfLog.debug("Parms: MaxFcstHourCfs = %d", self._maxFcstHourCfs) WhfLog.debug("Parms: StateFile = %s", self._stateFile)
def setCurrentModelAvailability(self, parms, config): """ Change availability status when appropriate by looking at disk Parameters ---------- parms : Parms parameters config : str Config file name Returns ------- none """ if (self._layered): # no need to do more, already layered return if (self._empty): WhfLog.error("Empty when not expected") return # make note if going from nothing to something nothing = True for f in self._step: if (f._layered): nothing = False #if (nothing): #WhfLog.debug("Nothing, so trying to get stuff") # first time only, try the -1 and -2 steps, force with what we have if (self._first): self._step[2].forceLayer(parms, config, self._issue) self._step[1].forceLayer(parms, config, self._issue) self._first = False self._step[0].layerIfReady(parms, config, self._issue) self._layered = self._step[0]._layered if (not self._layered): tnow = datetime.datetime.utcnow() diff = tnow - self._clockTime idiff = (diff.microseconds + (diff.seconds + diff.days*24*3600)*10**6)/10**6 if (idiff > parms._veryLateSeconds): WhfLog.warning("WARNING: Inputs for layering timeout Issue:%s", self._issue.strftime("%Y%m%d%H")) self._step[0].forceLayer(parms, config, self._issue) self._layered = True
def setCurrentModelAvailability(self, parms, config): """ Change availability status when appropriate by looking at disk Parameters ---------- parms : Parms parameters config : str Config file name Returns ------- none """ if (self._layered): # no need to do more, already layered return if (self._empty): WhfLog.error("Empty when not expected") return # make note if going from nothing to something nothing = True for f in self._step: if (f._layered): nothing = False #if (nothing): #WhfLog.debug("Nothing, so trying to get stuff") # first time only, try the -1 and -2 steps, force with what we have if (self._first): self._step[2].forceLayer(parms, config, self._issue) self._step[1].forceLayer(parms, config, self._issue) self._first = False self._step[0].layerIfReady(parms, config, self._issue) self._layered = self._step[0]._layered if (not self._layered): tnow = datetime.datetime.utcnow() diff = tnow - self._clockTime idiff = (diff.microseconds + (diff.seconds + diff.days * 24 * 3600) * 10**6) / 10**6 if (idiff > parms._veryLateSeconds): WhfLog.warning("WARNING: Inputs for layering timeout Issue:%s", self._issue.strftime("%Y%m%d%H")) self._step[0].forceLayer(parms, config, self._issue) self._layered = True
def debugPrint(self, name): """WhfLog.debug call with description of this data Parameters ---------- name: string name that is included in the description Returns ------- none """ WhfLog.debug("%s[%s]=%s, %s, %s", name, self._yyyymmddDir, self._name, self._fileType, self._time.debugString())
def setCurrentModelAvailability(self, parms, model): """ Change availability status when appropriate by looking at disk Parameters ---------- parms : Parms parameters model : Model overall status for this model run, used for clock time Returns ------- none """ if (self._layered): # no need to do more, already layered return # make note if going from nothing to something nothing = (not self._hrrr) and (not self._rap) #if (nothing): #WhfLog.debug("Nothing, so trying to get stuff") if (not self._hrrr): # update HRRR status self._hrrr = self._forecastExists(parms._hrrrDir) if (not self._rap): # update RAP status self._rap = self._forecastExists(parms._rapDir) if (nothing and (self._hrrr or self._rap)): # went from nothing to something, so start the clock WhfLog.debug("Starting clock now, hrrr=%d, rap=%d", df.boolToInt(self._hrrr), df.boolToInt(self._rap)) self._clockTime = datetime.datetime.utcnow() else: if (nothing and ((not self._hrrr) and (not self._rap))): # nothing to nothing, compare current time to time from # model input, and squeaky wheel if too long tnow = datetime.datetime.utcnow() diff = tnow - model._clockTime idiff = diff.total_seconds() if (idiff > parms._veryLateSeconds): WhfLog.warning( "Inputs for short range layering are very late Issue:%s Valid:%s", self._issue.strftime("%Y%m%d%H"), self._valid.strftime("%Y%m%d%H"))
def layer(parms, itime, step, which, config): """ Perform layering NOTE: here is where returns status will be added and used Parameters ---------- parms : Parms parameters """ WhfLog.debug("LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"), step, which) aaf.anal_assim_layer(itime.strftime('%Y%m%d%H'), '-%01d'%(step), which, config) WhfLog.debug("DONE LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"), step, which)
def setCurrentModelAvailability(self, parms, model): """ Change availability status when appropriate by looking at disk Parameters ---------- parms : Parms parameters model : Model overall status for this model run, used for clock time Returns ------- none """ if (self._layered): # no need to do more, already layered return # make note if going from nothing to something nothing = (not self._hrrr) and (not self._rap) #if (nothing): #WhfLog.debug("Nothing, so trying to get stuff") if (not self._hrrr): # update HRRR status self._hrrr = self._forecastExists(parms._hrrrDir) if (not self._rap): # update RAP status self._rap = self._forecastExists(parms._rapDir) if (nothing and (self._hrrr or self._rap)): # went from nothing to something, so start the clock WhfLog.debug("Starting clock now, hrrr=%d, rap=%d", df.boolToInt(self._hrrr), df.boolToInt(self._rap)) self._clockTime = datetime.datetime.utcnow() else: if (nothing and ((not self._hrrr) and (not self._rap))): # nothing to nothing, compare current time to time from # model input, and squeaky wheel if too long tnow = datetime.datetime.utcnow() diff = tnow - model._clockTime idiff = diff.total_seconds() if (idiff > parms._veryLateSeconds): WhfLog.warning("Inputs for short range layering are very late Issue:%s Valid:%s", self._issue.strftime("%Y%m%d%H"), self._valid.strftime("%Y%m%d%H"))
def _layer(self, parms, configFile): """ Perform layering NOTE: here is where returns status will be added and used Parameters ---------- parms : Parms parameters configFile : string name of file with settings """ path = self._issue.strftime("%Y%m%d%H") + "/" path += self._valid.strftime("%Y%m%d%H%M") + ".LDASIN_DOMAIN1.nc" WhfLog.info("LAYERING %s ", path) srf.forcing(configFile, 'layer', 'HRRR', path, 'RAP', path) WhfLog.info("DONE LAYERING file=%s", path)
def setCurrentModelAvailability(self, parms): """ Change availability status when appropriate by looking at disk Parameters ---------- parms : Parms Params """ for f in self._fState: # find matching model didSet = False for m in self._model: if (f.matches(m)): didSet = True # need to pass that model in to check for 'very late' f.setCurrentModelAvailability(parms, m) if (not didSet): WhfLog.error("No matching model for forecast")
def layer(parms, itime, step, which, config): """ Perform layering NOTE: here is where returns status will be added and used Parameters ---------- parms : Parms parameters """ WhfLog.debug("LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"), step, which) aaf.anal_assim_layer(itime.strftime('%Y%m%d%H'), '-%01d' % (step), which, config) WhfLog.debug("DONE LAYERING: %s %d %s", itime.strftime("%Y%m%d%H"), step, which)
def process(configFile, parm): '''Moves all the files from the source directory to the destination directory. Args: fileType (string) : source_dir (string): Full path to the source directory destination_dir (string): Full path to the destination directory extension (string): File extension of files to be moved Returns: None ''' # initialize the logging first parser = SafeConfigParser() parser.read(configFile) WhfLog.init(parser, 'RegridMRMS', True) # just use any valid 2nd arg for day in parm._days: processDay(configFile, day, parm)
def createStateFile(parms): """ Called if there is no state file, look at data dirs and create state Parameters ---------- parms: Parms Parameter settings Returns ------- none Writes out the state file after creating it """ WhfLog.info("Initializing") # query directory and get newest model run file, then # get all for that and previous issue time cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS") cfs.setNewestFiles(parms._hoursBackCfs) for f in cfs._content: f.debugPrint("Newest files: CFS") state = State("") state.initialize(cfs) # maybe back up and regrid that entire issue time # maybe redo this exact set of inputs only # maybe do nothing # maybe do all of them..for now do nothing as its easiest, just move on #files = hrrr.getFnames() #for f in files: # regridHRRR(f) #files = rap.getFnames() #for f in files: # regridRAP(f) #files = mrms.getFnames() #for f in files: # regridMRMS(f) # write out file state.write(parms._stateFile)
def parmRead(fname): """ Read in a param file Parameters --------- fname : str Name of the file to read Returns ------- Parms The parameters that were read in """ parser = SafeConfigParser() parser.read(fname) forcing_config_label = "AnalysisAssimLayeringDriver" WhfLog.init(parser, forcing_config_label, 'AA', 'Layer', 'RAP/HRRR/MRMS') hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir') hrrr0hrDir = parser.get('downscaling', 'HRRR_finished_output_dir_0hr') # placeholder mrmsDir = parser.get('regridding', 'MRMS_finished_output_dir') # maybe incorrect rapDir = parser.get('downscaling', 'RAP_finished_output_dir') rap0hrDir = parser.get('downscaling', 'RAP_finished_output_dir_0hr') # placeholder layerDir = parser.get('layering', 'analysis_assimilation_output') maxFcstHour = 3 hoursBack = 5 # 3 hours back at 0, -1, and -2 (-2 -3 = -5) maxWaitMinutes = 15 #int(parser.get('triggering', # 'short_range_fcst_max_wait_minutes')) veryLateMinutes = 20 #int(parser.get('triggering', # 'short_range_fcst_very_late_minutes')) stateFile = parser.get('triggering', 'analysis_assim_layering_state_file') parms = Parms(hrrrDir, hrrr0hrDir, rapDir, rap0hrDir, mrmsDir, layerDir, maxFcstHour, hoursBack, maxWaitMinutes, veryLateMinutes, stateFile) return parms
def parmRead(fname, fileType): """Read in the main config file, return needed parameters Parameters ---------- fname: str name of parameter file to read in fileType: str 'RAP', 'HRRR', 'MRMS', 'GFS' Returns ------- Parms values as pulled from the file """ parser = SafeConfigParser() parser.read(fname) forcing_config_label = "Regrid" + fileType if (fileType == 'HRRR'): configType = 'Short' elif (fileType == 'RAP'): configType = 'Short' elif (fileType == 'GFS'): configType = 'Medium' elif (fileType == 'MRMS'): configType = 'AA' else: configType = '???' WhfLog.init(parser, forcing_config_label, configType, 'Regrid', fileType) dataDir = parser.get('data_dir', fileType + '_data') maxFcstHour = int(parser.get('fcsthr_max', fileType + '_fcsthr_max')) hoursBack = int(parser.get('triggering', fileType + '_hours_back')) stateFile = parser.get('triggering', fileType + '_regrid_state_file') parms = Parms(dataDir, maxFcstHour, hoursBack, stateFile) return parms
def main(argv): # User must pass the config file into the main driver. configFile = argv[0] if not os.path.exists(configFile): print 'ERROR forcing engine config file not found.' return 1 # read in fixed main params parms = parmRead(configFile) #parms.debugPrint() #if there is not a state file, create one now using newest if (not os.path.exists(parms._stateFile)): parms.debugPrint() createStateFile(parms) # begin normal processing situation WhfLog.debug("....Check for new input data to regid") # read in state state = State(parms._stateFile) if state.isEmpty(): # error return here return 0 #state.debugPrint() # query directory and get newest model run file, then # get all for that and previous issue time cfs = df.DataFiles(parms._cfsDir, parms._maxFcstHourCfs, "CFS") cfs.setNewestFiles(parms._hoursBackCfs) # Same with CFS toProcess = state.updateWithNew(cfs, parms._hoursBackCfs) for f in toProcess: regridCFS(configFile, f) # write out state and exit #state.debugPrint() state.write(parms._stateFile) return 0
def olderIssueHour(self, hoursBack): """ Return the ForecastTime that has hoursBack hours earlier issue time Parameters ---------- hoursBack: int Number of hours back to look Returns ------- ForecastTime Forecast with issue time hoursBack earlier than local, forecastHour=0 """ if (self.isEmpty()): WhfLog.debug("ERROR empty input to olderIssueHour") else: timeBack = self.ymdh() - datetime.timedelta(hours=hoursBack) self._fcstTime = datetime.datetime(timeBack.year, timeBack.month, timeBack.day, 0, 0, 0) self._issueHour = timeBack.hour self._forecastHour = 0
def run(fileType, configFile, realtime): """ Run the script, process any new data Parameters ---------- fileType: str 'HRRR', ... configFile : str Name of the file with settings realtime : boolean True if this is realtime Returns ------- 1 for error, 0 for success """ good = False regriddable = ['HRRR', 'RAP', 'MRMS', 'GFS'] if (fileType not in regriddable): print 'ERROR unknown file type command arg ', fileType return 1 # User must pass the config file into the main driver. if not os.path.exists(configFile): print 'ERROR forcing engine config file not found:', configFile return 1 # read in fixed main params parms = parmRead(configFile, fileType, realtime) #if there is not a state file, create one now using newest if (not os.path.exists(parms._stateFile)): parms.debugPrint() createStateFile(parms, fileType, realtime) # read in state state = State(parms._stateFile, fileType) # query each directory and get newest model run file for each, then # get all for that and previous issue time data = df.DataFiles(parms._dataDir, parms._maxFcstHour, fileType) data.setNewestFiles(parms._hoursBack) # Update the state to reflect changes, returning those files to regrid # Regrid 'em toProcess = state.lookForNew(data, parms._hoursBack, fileType) for f in toProcess: try: regrid(f, fileType, configFile); except: WhfLog.error("Could not regrid/downscale %s", f) else: WhfLog.debug("Adding new file %s, and writing state file", f) if (not state.addFileIfNew(f)): WhfLog.error("File %s was not new", f) else: state.write(parms._stateFile, fileType); # write out state (in case it has not been written yet) and exit #state.debugPrint() state.write(parms._stateFile, fileType) return 0
def parmRead(fname): """ Read in a param file Parameters --------- fname : str Name of the file to read Returns ------- Parms The parameters that were read in """ parser = SafeConfigParser() parser.read(fname) forcing_config_label = "ShortRangeLayeringDriver" WhfLog.init(parser, forcing_config_label, 'Short', 'Layer', 'RAP/HRRR') hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir') #mrmsDir = parser.get('data_dir', 'MRMS_data') rapDir = parser.get('downscaling', 'RAP_finished_output_dir') layerDir = parser.get('layering', 'short_range_output') maxFcstHour = int(parser.get('fcsthr_max', 'HRRR_fcsthr_max')) # go with HRRR hoursBack = int(parser.get('triggering', 'HRRR_hours_back')) maxWaitMinutes=int(parser.get('triggering', 'short_range_fcst_max_wait_minutes')) veryLateMinutes=int(parser.get('triggering', 'short_range_fcst_very_late_minutes')) stateFile = parser.get('triggering', 'short_range_layering_state_file') parms = Parms(hrrrDir, rapDir, layerDir, maxFcstHour, hoursBack, maxWaitMinutes, veryLateMinutes, stateFile) return parms
def parmRead(fname, realtime): """ Read in a param file Parameters --------- fname : str Name of the file to read realtime: boolean True if realtime, False for archive mode Returns ------- Parms The parameters that were read in """ parser = SafeConfigParser() parser.read(fname) if (realtime): WhfLog.init(parser, 'ShortLayer', False) else: WhfLog.set('ShortLayer') hrrrDir = parser.get('downscaling', 'HRRR_finished_output_dir') rapDir = parser.get('downscaling', 'RAP_finished_output_dir') layerDir = parser.get('layering', 'short_range_output') maxFcstHour = int(parser.get('fcsthr_max', 'HRRR_fcsthr_max')) # go with HRRR hoursBack = int(parser.get('triggering', 'HRRR_hours_back')) maxWaitMinutes = int( parser.get('triggering', 'short_range_fcst_max_wait_minutes')) veryLateMinutes = int( parser.get('triggering', 'short_range_fcst_very_late_minutes')) stateFile = parser.get('triggering', 'short_range_layering_state_file') parms = Parms(hrrrDir, rapDir, layerDir, maxFcstHour, hoursBack, maxWaitMinutes, veryLateMinutes, stateFile) return parms
def makeDirIfNeeded(path): """ Input is a to be a directory, make it if it does not exist. Parameters ---------- path : str full path Returns ------- bool True if directory exists, or was created, false if error """ try: os.makedirs(path) return True except OSError as exception: if exception.errno != errno.EEXIST: WhfLog.error("ERROR creating %s", path) return False else: return True