class FrameEtaDataBuffer(object): """A cached and threaded interface to reading the last log line""" maxCacheTime = 60 maxThreads = 2 maxQueue = 501 def __init__(self): from ThreadPool import ThreadPool self.__threadPool = ThreadPool(self.maxThreads, self.maxQueue) self.__currentJob = None self.__cache = {} self.__defaultETA = '' self.__TIME = 0 self.__ETA = 1 def getEtaFormatted(self, job, frame): result = self.getEta(job, frame) if result: return Utils.secondsToHHMMSS(result) return False def getEta(self, job, frame): __now = time.time() try: jobKey = Utils.getObjectKey(job) if self.__currentJob != jobKey: # New job so clear cache self.__cache.clear() self.__currentJob = jobKey frameKey = Utils.getObjectKey(frame) if frameKey in self.__cache: # Frame eta is cached __cached = self.__cache[frameKey] if __cached[self.__TIME] < __now - self.maxCacheTime: # It is an old cache, queue an update, reset the time until updated self.__cache[frameKey][0] = __now + 60 self.__threadPool.queue( self.__doWork, self.__saveWork, "getting data for %s" % self.__class__, frameKey, job, frame) # Return the cached results anyway if __cached[self.__ETA] is not None: return max( __cached[self.__ETA] - __now + __cached[self.__TIME], 0) else: # Queue an update, cache a blank entry until updated self.__cache[frameKey] = [__now + 60, None] self.__threadPool.queue(self.__doWork, self.__saveWork, "getting data for %s" % self.__class__, frameKey, job, frame) # Since nothing is updated yet, return a default except Exception, e: self.__cache[frameKey] = [__now, None] map(logger.warning, Utils.exceptionOutput(e)) return self.__defaultETA
class FrameLogDataBuffer(object): """A cached and threaded interface to reading the last log line""" maxCacheTime = 5 maxThreads = 2 maxQueue = 500 # Position of data from getLastLineData LASTLINE = 0 LLU = 1 # Notes for percentage: # PERCENT = 2 # go back 15 lines? # default to "" prior to checking # default to 0% if nothing found unless already a previous value def __init__(self): from ThreadPool import ThreadPool self.__threadPool = ThreadPool(self.maxThreads, self.maxQueue) self.__currentJob = None self.__cache = {} self.__queue = {} self.__defaultLine = "" self.__defaultLLU = "" # Position of data in self.__cache self.__TIME = 0 self.__PATH = 1 self.__LINE = 2 self.__LLU = 3 def getLastLineData(self, job, frame): """Returns the last line and LLU of the log file or queues a request to update it""" try: __now = time.time() jobKey = Utils.getObjectKey(job) if self.__currentJob != jobKey: # New job so clear cache self.__cache.clear() self.__queue.clear() self.__currentJob = jobKey if len(self.__queue) > len(self.__threadPool._q_queue): # Everything is hung up, start over self.__cache.clear() self.__queue.clear() frameKey = Utils.getObjectKey(frame) if frameKey in self.__cache: # Last line is cached __cached = self.__cache[frameKey] if __cached[self.__TIME] < __now - self.maxCacheTime: # Its an old cache, queue an update self.__cache[frameKey][0] = __now + 60 self.__queue[frameKey] = __cached[self.__PATH] self.__threadPool.queue( self.__doWork, self.__saveWork, "getting data for %s" % self.__class__) # Return the cached results anyway return (__cached[self.__LINE], __cached[self.__LLU]) else: __path = Utils.getFrameLogFile(job, frame) # Cache a blank entry until it is filled in self.__cache[frameKey] = [ __now + 60, __path, self.__defaultLine, self.__defaultLLU ] # Queue an update self.__queue[frameKey] = __path self.__threadPool.queue(self.__doWork, self.__saveWork, "getting data for %s" % self.__class__) # Since nothing is updated yet, return an empty string return (self.__defaultLine, self.__defaultLLU) except Exception, e: map(logger.warning, Utils.exceptionOutput(e))