def __init__(self):
RMParserManager.instance = self
self.parsers = {}
self.forceParsersRun = False
self.__maxFails = 100
self.__minDelayBetweenFails = 120 # 2 min
self.__maxDelayBetweenFails = 300 # 5 min
self.__stepDelayBetweenFails = 30 # 30 seconds
self.__delayAfterMaxFails = 86400 # 1 day
self.__lastRunningTimestamp = None # The timestamp when parsers last attempted to run
self.__lastUpdateTimestamp = 0 # The timestamp when parsers actually ran
self.__runningInterval = 60 # 1 minute
self.parserTable = RMParserTable(globalDbManager.parserDatabase)
self.parserDataTable = RMParserDataTable(
globalDbManager.parserDatabase)
self.forecastTable = RMForecastTable(globalDbManager.parserDatabase)
self.userDataTypeTable = RMUserDataTypeTable(
globalDbManager.parserDatabase)
self.parserUserDataTypeTable = RMParserUserDataTable(
globalDbManager.parserDatabase)
self.userDataTypeTable.buildCache()
self.mixer = None
self.__load(os.path.dirname(__file__) + '/parsers')
def __init__(self):
RMParserManager.instance = self
self.parsers = {}
self.forceParsersRun = False
self.__maxFails = 100
self.__minDelayBetweenFails = 120 # 2 min
self.__maxDelayBetweenFails = 300 # 5 min
self.__stepDelayBetweenFails = 30 # 30 seconds
self.__delayAfterMaxFails = 86400 # 1 day
self.__lastRunningTimestamp = None # The timestamp when parsers last attempted to run
self.__lastUpdateTimestamp = 0 # The timestamp when parsers actually ran
self.__runningInterval = 60 # 1 minute
self.parserTable = RMParserTable(globalDbManager.parserDatabase)
self.parserDataTable = RMParserDataTable(globalDbManager.parserDatabase)
self.forecastTable = RMForecastTable(globalDbManager.parserDatabase)
self.userDataTypeTable = RMUserDataTypeTable(globalDbManager.parserDatabase)
self.parserUserDataTypeTable = RMParserUserDataTable(globalDbManager.parserDatabase)
self.userDataTypeTable.buildCache()
self.mixer = None
self.__load(os.path.dirname(__file__) + '/parsers')
class RMParserManager:
IGNORE_PYC_MODULES = True
instance = None
def __init__(self):
RMParserManager.instance = self
self.parsers = {}
self.forceParsersRun = False
self.__maxFails = 100
self.__minDelayBetweenFails = 120 # 2 min
self.__maxDelayBetweenFails = 300 # 5 min
self.__stepDelayBetweenFails = 30 # 30 seconds
self.__delayAfterMaxFails = 86400 # 1 day
self.__lastRunningTimestamp = None # The timestamp when parsers last attempted to run
self.__lastUpdateTimestamp = 0 # The timestamp when parsers actually ran
self.__runningInterval = 60 # 1 minute
self.parserTable = RMParserTable(globalDbManager.parserDatabase)
self.parserDataTable = RMParserDataTable(
globalDbManager.parserDatabase)
self.forecastTable = RMForecastTable(globalDbManager.parserDatabase)
self.userDataTypeTable = RMUserDataTypeTable(
globalDbManager.parserDatabase)
self.parserUserDataTypeTable = RMParserUserDataTable(
globalDbManager.parserDatabase)
self.userDataTypeTable.buildCache()
self.mixer = None
self.__load(os.path.dirname(__file__) + '/parsers')
def preRun(self):
unmixedForecastAvailable = False
lastForecast = None
latestForecastByParser = self.parserDataTable.getLastForecastByParser()
for parserID in latestForecastByParser:
parserConfig = self.findParserConfig(parserID)
if parserConfig != None:
parserConfig.runtimeLastForecastInfo = latestForecastByParser[
parserID]
if not parserConfig.runtimeLastForecastInfo.processed:
unmixedForecastAvailable = True
if lastForecast == None:
lastForecast = parserConfig.runtimeLastForecastInfo
log.debug(
"*** All values are already mixed! No need to run the Mixer!")
for parserConfig in self.parsers:
self.parserDataTable.clearHistory(parserConfig.dbID, False)
globalDbManager.parserDatabase.commit()
globalDbManager.parserDatabase.vacuum()
return None, None
def run(self, parserId=None, forceRunParser=False, forceRunMixer=False):
currentTimestamp = rmCurrentTimestamp()
forceRunParser = True
if not forceRunParser and self.__lastRunningTimestamp is not None and (
currentTimestamp -
self.__lastRunningTimestamp) < self.__runningInterval:
# We want to run the parser only each N minutes. This condition is not met, try later.
log.debug(
"Parser %r not run lastRunning timestamp %s current %s" %
(parserId, self.__lastRunningTimestamp, currentTimestamp))
return None, None
self.__lastRunningTimestamp = currentTimestamp
newValuesAvailable = False
newForecast = RMForecastInfo(None, currentTimestamp)
log.debug("*** BEGIN Running parsers: %d (%s)" %
(newForecast.timestamp,
rmTimestampToDateAsString(newForecast.timestamp)))
for parserConfig in self.parsers:
if parserId is not None and parserId != parserConfig.dbID:
continue
log.debug(" * Parser: %s -> %s" %
(parserConfig, parserConfig.runtimeLastForecastInfo))
if parserConfig.enabled:
if parserConfig.failCounter >= self.__maxFails:
if forceRunParser or parserConfig.lastFailTimestamp is None or (
abs(newForecast.timestamp -
parserConfig.lastFailTimestamp) >=
self.__delayAfterMaxFails):
parserConfig.failCounter = 0
parserConfig.lastFailTimestamp = None
else:
if parserConfig.failCounter == self.__maxFails:
log.warning(
" * Parser: %s - ignored because of lack of data (failCounter=%s, lastFail=%s)!"
% (parserConfig, ` parserConfig.failCounter `,
rmTimestampToDateAsString(
parserConfig.lastFailTimestamp)))
parserConfig.failCounter += 1 # Increment this to get rid of the above message.
continue
elif parserConfig.failCounter > 0:
retryDelay = min(
self.__minDelayBetweenFails +
(parserConfig.failCounter - 1) *
self.__stepDelayBetweenFails,
self.__maxDelayBetweenFails)
nextRetryTimestamp = parserConfig.lastFailTimestamp + retryDelay
if newForecast.timestamp < nextRetryTimestamp:
log.debug(
" * Ignored because retry delay %d (sec) was not reached"
% retryDelay)
continue
log.debug(" * Parser retry after previous fail")
parser = self.parsers[parserConfig]
lastUpdate = None
if parserConfig.runtimeLastForecastInfo:
# Check if parser hasn't run with an invalid future date
if parserConfig.runtimeLastForecastInfo.timestamp <= currentTimestamp:
lastUpdate = parserConfig.runtimeLastForecastInfo.timestamp
# Save the newest parser run
if lastUpdate is not None and lastUpdate > self.__lastUpdateTimestamp:
self.__lastUpdateTimestamp = lastUpdate
if not forceRunParser and not self.forceParsersRun and (
lastUpdate != None and
(newForecast.timestamp - lastUpdate) <
parser.parserInterval):
log.debug(
" * Ignored because interval %d not expired for timestamp %d lastUpdate: %d"
% (parser.parserInterval, newForecast.timestamp,
lastUpdate))
continue
log.debug(" * Running parser %s with interval %d" %
(parser.parserName, parser.parserInterval))
parser.settings = globalSettings.getSettings()
parser.runtime[
RMParser.RuntimeDayTimestamp] = rmCurrentDayTimestamp()
try:
parser.lastKnownError = ''
parser.isRunning = True
parser.perform()
parser.isRunning = False
except Exception, e:
log.error(" * Cannot execute parser %s" %
parser.parserName)
log.exception(e)
parser.isRunning = False
if len(parser.lastKnownError) == 0:
parser.lastKnownError = 'Error: Failed to run'
if not parser.hasValues():
parserConfig.failCounter += 1
parserConfig.lastFailTimestamp = newForecast.timestamp
if len(parser.lastKnownError) == 0:
parser.lastKnownError = 'Error: parser returned no values'
parser.isRunning = False
if parserConfig.failCounter == 1:
log.warn(" * Parser %s returned no values" %
parser.parserName)
continue
parserConfig.failCounter = 0
parserConfig.lastFailTimestamp = None
if newForecast.id == None:
self.forecastTable.addRecordEx(newForecast)
parserConfig.runtimeLastForecastInfo = newForecast
if not globalSettings.vibration:
self.parserDataTable.removeEntriesWithParserIdAndTimestamp(
parserConfig.dbID, parser.getValues())
self.parserDataTable.addRecords(newForecast.id,
parserConfig.dbID,
parser.getValues())
parser.clearValues()
newValuesAvailable = True
mixerDataValues = None
if newValuesAvailable:
globalDbManager.parserDatabase.vacuum()
if not mixerDataValues is None:
for parserConfig in self.parsers:
if parserConfig.runtimeLastForecastInfo:
parserConfig.runtimeLastForecastInfo.processed = True
else:
log.debug(" * No new value available from parsers")
log.debug("*** END Running parsers: %s, %d (%s)" %
( ` newForecast.id `, newForecast.timestamp,
rmTimestampToDateAsString(newForecast.timestamp)))
return newForecast, mixerDataValues
class RMParserManager:
IGNORE_PYC_MODULES = True
instance = None
def __init__(self):
RMParserManager.instance = self
self.parsers = {}
self.forceParsersRun = False
self.__maxFails = 100
self.__minDelayBetweenFails = 120 # 2 min
self.__maxDelayBetweenFails = 300 # 5 min
self.__stepDelayBetweenFails = 30 # 30 seconds
self.__delayAfterMaxFails = 86400 # 1 day
self.__lastRunningTimestamp = None # The timestamp when parsers last attempted to run
self.__lastUpdateTimestamp = 0 # The timestamp when parsers actually ran
self.__runningInterval = 60 # 1 minute
self.parserTable = RMParserTable(globalDbManager.parserDatabase)
self.parserDataTable = RMParserDataTable(globalDbManager.parserDatabase)
self.forecastTable = RMForecastTable(globalDbManager.parserDatabase)
self.userDataTypeTable = RMUserDataTypeTable(globalDbManager.parserDatabase)
self.parserUserDataTypeTable = RMParserUserDataTable(globalDbManager.parserDatabase)
self.userDataTypeTable.buildCache()
self.mixer = None
self.__load(os.path.dirname(__file__) + '/parsers')
def preRun(self):
unmixedForecastAvailable = False
lastForecast = None
latestForecastByParser = self.parserDataTable.getLastForecastByParser()
for parserID in latestForecastByParser:
parserConfig = self.findParserConfig(parserID)
if parserConfig != None:
parserConfig.runtimeLastForecastInfo = latestForecastByParser[parserID]
if not parserConfig.runtimeLastForecastInfo.processed:
unmixedForecastAvailable = True
if lastForecast == None:
lastForecast = parserConfig.runtimeLastForecastInfo
log.debug("*** All values are already mixed! No need to run the Mixer!")
for parserConfig in self.parsers:
self.parserDataTable.clearHistory(parserConfig.dbID, False)
globalDbManager.parserDatabase.commit()
globalDbManager.parserDatabase.vacuum()
return None, None
def run(self, parserId = None, forceRunParser = False, forceRunMixer = False):
currentTimestamp = rmCurrentTimestamp()
forceRunParser = True
if not forceRunParser and self.__lastRunningTimestamp is not None and (currentTimestamp - self.__lastRunningTimestamp) < self.__runningInterval:
# We want to run the parser only each N minutes. This condition is not met, try later.
log.debug("Parser %r not run lastRunning timestamp %s current %s" % (parserId, self.__lastRunningTimestamp, currentTimestamp))
return None, None
self.__lastRunningTimestamp = currentTimestamp
newValuesAvailable = False
newForecast = RMForecastInfo(None, currentTimestamp)
log.debug("*** BEGIN Running parsers: %d (%s)" % (newForecast.timestamp, rmTimestampToDateAsString(newForecast.timestamp)))
for parserConfig in self.parsers:
if parserId is not None and parserId != parserConfig.dbID:
continue
log.debug(" * Parser: %s -> %s" % (parserConfig, parserConfig.runtimeLastForecastInfo))
if parserConfig.enabled:
if parserConfig.failCounter >= self.__maxFails:
if forceRunParser or parserConfig.lastFailTimestamp is None or (abs(newForecast.timestamp - parserConfig.lastFailTimestamp) >= self.__delayAfterMaxFails):
parserConfig.failCounter = 0
parserConfig.lastFailTimestamp = None
else:
if parserConfig.failCounter == self.__maxFails:
log.warning(" * Parser: %s - ignored because of lack of data (failCounter=%s, lastFail=%s)!" %
(parserConfig, `parserConfig.failCounter`, rmTimestampToDateAsString(parserConfig.lastFailTimestamp)))
parserConfig.failCounter += 1 # Increment this to get rid of the above message.
continue
elif parserConfig.failCounter > 0:
retryDelay = min(self.__minDelayBetweenFails + (parserConfig.failCounter - 1) * self.__stepDelayBetweenFails, self.__maxDelayBetweenFails)
nextRetryTimestamp = parserConfig.lastFailTimestamp + retryDelay
if newForecast.timestamp < nextRetryTimestamp:
log.debug(" * Ignored because retry delay %d (sec) was not reached" % retryDelay)
continue
log.debug(" * Parser retry after previous fail")
parser = self.parsers[parserConfig]
lastUpdate = None
if parserConfig.runtimeLastForecastInfo:
# Check if parser hasn't run with an invalid future date
if parserConfig.runtimeLastForecastInfo.timestamp <= currentTimestamp:
lastUpdate = parserConfig.runtimeLastForecastInfo.timestamp
# Save the newest parser run
if lastUpdate is not None and lastUpdate > self.__lastUpdateTimestamp:
self.__lastUpdateTimestamp = lastUpdate
if not forceRunParser and not self.forceParsersRun and (lastUpdate != None and (newForecast.timestamp - lastUpdate) < parser.parserInterval):
log.debug(" * Ignored because interval %d not expired for timestamp %d lastUpdate: %d" % (parser.parserInterval, newForecast.timestamp, lastUpdate))
continue
log.debug(" * Running parser %s with interval %d" % (parser.parserName, parser.parserInterval))
parser.settings = globalSettings.getSettings()
parser.runtime[RMParser.RuntimeDayTimestamp] = rmCurrentDayTimestamp()
try:
parser.lastKnownError = ''
parser.isRunning = True
parser.perform()
parser.isRunning = False
except Exception, e:
log.error(" * Cannot execute parser %s" % parser.parserName)
log.exception(e)
parser.isRunning = False
if len(parser.lastKnownError) == 0:
parser.lastKnownError = 'Error: Failed to run'
if not parser.hasValues():
parserConfig.failCounter += 1
parserConfig.lastFailTimestamp = newForecast.timestamp
if len(parser.lastKnownError) == 0:
parser.lastKnownError = 'Error: parser returned no values'
parser.isRunning = False
if parserConfig.failCounter == 1:
log.warn (" * Parser %s returned no values" % parser.parserName)
continue
parserConfig.failCounter = 0
parserConfig.lastFailTimestamp = None
if newForecast.id == None:
self.forecastTable.addRecordEx(newForecast)
parserConfig.runtimeLastForecastInfo = newForecast
if not globalSettings.vibration:
self.parserDataTable.removeEntriesWithParserIdAndTimestamp(parserConfig.dbID, parser.getValues())
self.parserDataTable.addRecords(newForecast.id, parserConfig.dbID, parser.getValues())
parser.clearValues()
newValuesAvailable = True
mixerDataValues = None
if newValuesAvailable:
globalDbManager.parserDatabase.vacuum()
if not mixerDataValues is None:
for parserConfig in self.parsers:
if parserConfig.runtimeLastForecastInfo:
parserConfig.runtimeLastForecastInfo.processed = True
else:
log.debug(" * No new value available from parsers")
log.debug("*** END Running parsers: %s, %d (%s)" % (`newForecast.id`, newForecast.timestamp, rmTimestampToDateAsString(newForecast.timestamp)))
return newForecast, mixerDataValues