def _checkAWOLTasks(self):
self._logger.debug('Checking AWOL tasks...')
for task in self._schedModule.getRunningList()[:]:
if not task.getOnRunningListSince():
self._logger.warning(
"Task %s is in the runningList but has no "
"onRunningListSince value! Removing from runningList "
"and relaunching..." % (task.id))
task.tearDown()
# relaunch it
self._db_moveTask(task, base.TASK_STATUS_RUNNING,
base.TASK_STATUS_QUEUED)
else:
runForSecs = int_timestamp(self._getCurrentDateTime()) - \
int_timestamp(task.getOnRunningListSince())
if runForSecs > task._AWOLThresold:
self._logger.warning(
"Task %s has been running for %d secs. "
"Assuming it has died abnormally and forcibly "
"calling its tearDown()..." % (task.id, runForSecs))
task.tearDown()
self._db_moveTask(task,
base.TASK_STATUS_RUNNING,
base.TASK_STATUS_TERMINATED,
setStatus=True)
self._logger.info("Task %s terminated." % (task.id))
def _checkAWOLTasks(self):
self._logger.debug('Checking AWOL tasks...')
for task in self._schedModule.getRunningList()[:]:
if not task.getOnRunningListSince():
self._logger.warning("Task %s is in the runningList but has no "
"onRunningListSince value! Removing from runningList "
"and relaunching..." % (task.id))
task.tearDown()
# relaunch it
self._db_moveTask(
task,
base.TASK_STATUS_RUNNING,
base.TASK_STATUS_QUEUED)
else:
runForSecs = int_timestamp(self._getCurrentDateTime()) - \
int_timestamp(task.getOnRunningListSince())
if runForSecs > task._AWOLThresold:
self._logger.warning("Task %s has been running for %d secs. "
"Assuming it has died abnormally and forcibly "
"calling its tearDown()..." % (task.id, runForSecs))
task.tearDown()
self._db_moveTask(
task,
base.TASK_STATUS_RUNNING,
base.TASK_STATUS_TERMINATED,
setStatus=True)
self._logger.info("Task %s terminated." % (task.id))
def run(self):
sm = SyncManager.getDBInstance()
logger = self.getLogger()
# go over all the agents
for agtName, agent in sm.getAllAgents().iteritems():
# skip agents if they're not active
if not agent.isActive():
logger.warning("Agent '%s' is not active - skipping" % agtName)
continue
logger.info("Starting agent '%s'" % agtName)
try:
dbi = DBMgr.getInstance()
# pass the current time and a logger
result = agent.run(int_timestamp(nowutc()), logger=logger, dbi=dbi, task=self)
except:
logger.exception("Problem running agent '%s'" % agtName)
return
if result:
logger.info("Acknowledged successful operation")
agent.acknowledge()
dbi.commit()
else:
logger.info("'Acknowledge' not done - no records?")
logger.info("Agent '%s' finished" % agtName)
def moveTask(self, task, moveFrom, status, occurrence=None, nocheck=False):
"""
Move a task somewhere
"""
if not occurrence:
occurrence = task
if not nocheck:
self._assertTaskStatus(task, moveFrom)
if moveFrom == base.TASK_STATUS_RUNNING:
# actually remove it from list
self.removeRunningTask(task)
elif moveFrom == base.TASK_STATUS_QUEUED:
idx_timestamp = int_timestamp(task.getStartOn())
self._waitingQueue.dequeue(idx_timestamp, task)
elif moveFrom == base.TASK_STATUS_FAILED:
self._failedIndex.unindex_obj(task)
# index it either in finished or failed
# (or queue it up again)
if status == base.TASK_STATUS_FINISHED:
self._finishedIndex.index_obj(occurrence)
elif status in [base.TASK_STATUS_FAILED,
base.TASK_STATUS_TERMINATED]:
self._failedIndex.index_obj(occurrence)
elif status == base.TASK_STATUS_QUEUED:
self.addTaskToWaitingQueue(occurrence)
def moveTask(self, task, moveFrom, status, occurrence=None, nocheck=False):
"""
Move a task somewhere
"""
if not occurrence:
occurrence = task
if not nocheck:
self._assertTaskStatus(task, moveFrom)
if moveFrom == base.TASK_STATUS_RUNNING:
# actually remove it from list
self.removeRunningTask(task)
elif moveFrom == base.TASK_STATUS_QUEUED:
idx_timestamp = int_timestamp(task.getStartOn())
self._waitingQueue.dequeue(idx_timestamp, task)
elif moveFrom == base.TASK_STATUS_FAILED:
self._failedIndex.unindex_obj(task)
# index it either in finished or failed
# (or queue it up again)
if status == base.TASK_STATUS_FINISHED:
self._finishedIndex.index_obj(occurrence)
elif status in [base.TASK_STATUS_FAILED,
base.TASK_STATUS_TERMINATED]:
self._failedIndex.index_obj(occurrence)
elif status == base.TASK_STATUS_QUEUED:
self.addTaskToWaitingQueue(occurrence)
def run(self):
sm = SyncManager.getDBInstance()
logger = self.getLogger()
# go over all the agents
for agtName, agent in sm.getAllAgents().iteritems():
# skip agents if they're not active
if not agent.isActive():
logger.warning("Agent '%s' is not active - skipping" % agtName)
continue
logger.info("Starting agent '%s'" % agtName)
try:
dbi = DBMgr.getInstance()
# pass the current time and a logger
result = agent.run(int_timestamp(nowutc()),
logger=logger,
dbi=dbi)
except:
logger.exception("Problem running agent '%s'" % agtName)
return
if result:
logger.info("Acknowledged successful operation")
agent.acknowledge()
dbi.commit()
else:
logger.info("'Acknowledge' not done - no records?")
logger.info("Agent '%s' finished" % agtName)
def _iterateTasks(self):
"""
Iterate over all the tasks in the waiting queue, blocking
in case there are none
"""
while True:
currentTimestamp = int_timestamp(self._getCurrentDateTime())
# this will basically abort the transaction, so, make sure
# everything important before this was committed
self._readFromDb()
# print a status message (only in debug mode)
self._printStatus(mode='debug')
# move the tasks in the spool to the waiting queue
try:
self._processSpool()
finally:
# this `finally` makes sure finished tasks are handled
# even if a shutdown order is sent
# process tasks that have finished meanwhile
# (tasks have been running in different processes, so, the sync
# thas was done above won't hurt)
self._checkFinishedTasks()
# get the next task in queue
res = self._schedModule.peekNextWaitingTask()
if res:
# it's actually a timestamp, task tuple
nextTS, nextTask = res
self._logger.debug(
(nextTS, currentTimestamp, self._getCurrentDateTime()))
# if it's time to execute the task
if (nextTS <= currentTimestamp):
yield nextTS, nextTask
# don't sleep, jump back to the beginning of the cycle
continue
# we assume the task cycle has been completed (if there was one)
# so, we can just reset the db status (sync)
self._readFromDb()
# we also check AWOL tasks from time to time
if random.random() < self._config.awol_tasks_check_probability:
self._checkAWOLTasks()
# if we get here, we have nothing else to do...
self._sleep('Nothing to do. Sleeping for %d secs...' %
self._config.sleep_interval)
# read from DB again after sleeping
self._readFromDb()
def _taskCycle(self, timestamp, curTask):
self._db_setTaskRunning(timestamp, curTask)
# Start a worker subprocess and add it to the worker dict
delay = int_timestamp(self._getCurrentDateTime()) - timestamp
self._runningWorkers[curTask.id] = ProcessWorker(curTask.id, self._config, delay)
self._runningWorkers[curTask.id].start()
def _iterateTasks(self):
"""
Iterate over all the tasks in the waiting queue, blocking
in case there are none
"""
while True:
currentTimestamp = int_timestamp(self._getCurrentDateTime())
# this will basically abort the transaction, so, make sure
# everything important before this was committed
self._readFromDb()
# print a status message (only in debug mode)
self._printStatus(mode = 'debug')
# move the tasks in the spool to the waiting queue
try:
self._processSpool()
finally:
# this `finally` makes sure finished tasks are handled
# even if a shutdown order is sent
# process tasks that have finished meanwhile
# (tasks have been running in different threads, so, the sync
# thas was done above won't hurt)
self._checkFinishedTasks()
# get the next task in queue
res = self._schedModule.peekNextWaitingTask()
if res:
# it's actually a timestamp, task tuple
nextTS, nextTask = res
self._logger.debug((nextTS, currentTimestamp, self._getCurrentDateTime()))
# if it's time to execute the task
if (nextTS <= currentTimestamp):
yield nextTS, nextTask
# don't sleep, jump back to the beginning of the cycle
continue
# we assume the task cycle has been completed (if there was one)
# so, we can just reset the db status (sync)
self._readFromDb()
# we also check AWOL tasks from time to time
if random.random() < self._config.awol_tasks_check_probability:
self._checkAWOLTasks()
# if we get here, we have nothing else to do...
self._sleep('Nothing to do. Sleeping for %d secs...' %
self._config.sleep_interval)
# read from DB again after sleeping
self._readFromDb()
def _taskCycle(self, timestamp, curTask):
self._db_setTaskRunning(timestamp, curTask)
# Start a worker subprocess and add it to the worker dict
delay = int_timestamp(self._getCurrentDateTime()) - timestamp
self._runningWorkers[curTask.id] = ProcessWorker(
curTask.id, self._config, delay)
self._runningWorkers[curTask.id].start()
def getVars(self):
tplVars = WTemplated.getVars(self)
smanager = SyncManager.getDBInstance()
tplVars["trackData"], tplVars["lastAgentTS"] = self._calculateTrackData(smanager)
tplVars["agents"] = smanager.getAllAgents()
tplVars["currentTS"] = int_timestamp(nowutc())
tplVars["granularity"] = smanager.getGranularity()
return tplVars
def getVars(self):
tplVars = WTemplated.getVars(self)
smanager = SyncManager.getDBInstance()
tplVars['trackData'], tplVars['lastAgentTS'] = \
self._calculateTrackData(smanager)
tplVars['agents'] = smanager.getAllAgents()
tplVars['currentTS'] = int_timestamp(nowutc())
tplVars['granularity'] = smanager.getGranularity()
return tplVars
def getVars(self):
tplVars = WTemplated.getVars(self)
smanager = SyncManager.getDBInstance()
tplVars['trackData'], tplVars['lastAgentTS'] = \
self._calculateTrackData(smanager)
tplVars['agents'] = smanager.getAllAgents()
tplVars['currentTS'] = int_timestamp(nowutc())
tplVars['granularity'] = smanager.getGranularity()
return tplVars
def prepare(self):
"""
This information will be saved regardless of the task being repeated or not
"""
curTime = self._getCurrentDateTime()
tsDiff = int_timestamp(self.getStartOn()) - int_timestamp(curTime)
if tsDiff > 0:
self.getLogger().debug('Task %s will wait for some time. (%s) > (%s)' % \
(self.id, self.getStartOn(), curTime))
base.TimeSource.get().sleep(tsDiff)
if self.expiryDate and curTime > self.expiryDate:
self.getLogger().warning(
'Task %s will not be executed, expiryDate (%s) < current time (%s)' % \
(self.id, self.expiryDate, curTime))
return False
self.startedOn = curTime
self.running = True
self.setStatus(base.TASK_STATUS_RUNNING)
def _taskCycle(self, timestamp, curTask):
self._db_setTaskRunning(timestamp, curTask)
# Start a worker subprocess
# Add it to the thread dict
if self._config.multitask_mode == 'processes':
wclass = ProcessWorker
else:
wclass = ThreadWorker
delay = int_timestamp(self._getCurrentDateTime()) - timestamp
self._runningWorkers[curTask.id] = wclass(curTask.id, self._config, delay)
self._runningWorkers[curTask.id].start()
def _taskCycle(self, timestamp, curTask):
self._db_setTaskRunning(timestamp, curTask)
# Start a worker subprocess
# Add it to the thread dict
if self._config.multitask_mode == 'processes':
wclass = ProcessWorker
else:
wclass = ThreadWorker
delay = int_timestamp(self._getCurrentDateTime()) - timestamp
self._runningWorkers[curTask.id] = wclass(curTask.id, self._config,
delay)
self._runningWorkers[curTask.id].start()
def addTaskToWaitingQueue(self, task, index=False):
if index:
self._indexTask(task)
# get an int timestamp
timestamp = int_timestamp(task.getStartOn())
self._waitingQueue.enqueue(timestamp, task)
# make it "officially" queued
task.setStatus(base.TASK_STATUS_QUEUED)
logging.getLogger('scheduler').debug(
'Added %s to waitingQueue..' % task)
def changeTaskStartDate(self, oldTS, task):
newTS = int_timestamp(task.getStartOn())
# enqueue-dequeue
try:
self._waitingQueue.dequeue(oldTS, task)
except:
logging.getLogger('scheduler').error(
"%s was supposed to be changed but it was not "
"found in the waiting queue!" % task)
return
self._waitingQueue.enqueue(newTS, task)
logging.getLogger('scheduler').info(
'%s moved from bin %s to %s...' % (task, oldTS, newTS))
def changeTaskStartDate(self, oldTS, task):
newTS = int_timestamp(task.getStartOn())
# enqueue-dequeue
try:
self._waitingQueue.dequeue(oldTS, task)
except:
logging.getLogger('scheduler').error(
"%s was supposed to be changed but it was not "
"found in the waiting queue!" % task)
return
self._waitingQueue.enqueue(newTS, task)
logging.getLogger('scheduler').info(
'%s moved from bin %s to %s...' % (task, oldTS, newTS))
def requestFinished(self, obj, req):
sm = SyncManager.getDBInstance()
cm = ContextManager.get('indico.ext.livesync:actions')
cm_ids = ContextManager.get('indico.ext.livesync:ids')
timestamp = int_timestamp(nowutc())
# if the returned context is a dummy one, there's nothing to do
if cm.__class__ == DummyDict:
return
# Insert the elements from the temporary index
# into the permanent one (MPT)
for obj, actions in cm.iteritems():
objId = cm_ids[obj]
for action in actions:
Logger.get('ext.livesync').debug((objId, action))
# TODO: remove redundant items
if not sm.objectExcluded(obj):
sm.add(timestamp, ActionWrapper(timestamp, obj, actions,
objId))
def requestFinished(self, obj, req):
sm = SyncManager.getDBInstance()
cm = ContextManager.get('indico.ext.livesync:actions')
cm_ids = ContextManager.get('indico.ext.livesync:ids')
timestamp = int_timestamp(nowutc())
# if the returned context is a dummy one, there's nothing to do
if cm.__class__ == DummyDict:
return
# Insert the elements from the temporary index
# into the permanent one (MPT)
for obj, actions in cm.iteritems():
objId = cm_ids[obj]
for action in actions:
Logger.get('ext.livesync').debug((objId, action))
# TODO: remove redundant items
if not sm.objectExcluded(obj):
sm.add(timestamp,
ActionWrapper(timestamp, obj, actions, objId))
def _getAnswer(self):
self._agent.preActivate(int_timestamp(nowutc()))
return True
def moveTask(self, task, newDate):
oldTS = int_timestamp(task.getStartOn())
task.setStartOn(newDate)
return self._schedMod.spool('change', (oldTS, task))
def _nextTS(self):
time.sleep(1)
return int_timestamp(nowutc())
def _getAnswer(self):
self._agent.preActivate(int_timestamp(nowutc()) / self._sm.getGranularity())
return True
def _nextTS(self):
time.sleep(1)
return int_timestamp(nowutc())
def _getAnswer(self):
self._agent.preActivate(
int_timestamp(nowutc()) / self._sm.getGranularity())
return True
def _getAnswer(self):
self._agent.preActivate(int_timestamp(nowutc()))
return True
def getIndexingDateTime(self):
return int_timestamp(self._getCurrentDateTime())