def __init__(self, commuChannel, taskBufferIF, ddmIF, vos,
prodSourceLabels):
self.vos = self.parseInit(vos)
self.prodSourceLabels = self.parseInit(prodSourceLabels)
self.pid = '{0}-{1}_{2}-con'.format(socket.getfqdn().split('.')[0],
os.getpid(), os.getpgrp())
JediKnight.__init__(self, commuChannel, taskBufferIF, ddmIF, logger)
def __init__(self,commuChannel,taskBufferIF,ddmIF,vos,prodSourceLabels):
self.vos = self.parseInit(vos)
self.prodSourceLabels = self.parseInit(prodSourceLabels)
self.pid = '{0}-{1}-dog'.format(socket.getfqdn().split('.')[0],os.getpid())
JediKnight.__init__(self,commuChannel,taskBufferIF,ddmIF,logger)
FactoryBase.__init__(self,self.vos,self.prodSourceLabels,logger,
jedi_config.watchdog.modConfig)
def __init__(self,commuChannel,taskBufferIF,ddmIF,vos,prodSourceLabels):
self.vos = self.parseInit(vos)
self.prodSourceLabels = self.parseInit(prodSourceLabels)
self.pid = '{0}-{1}-post'.format(socket.getfqdn().split('.')[0],os.getpid())
JediKnight.__init__(self,commuChannel,taskBufferIF,ddmIF,logger)
FactoryBase.__init__(self,self.vos,self.prodSourceLabels,logger,
jedi_config.postprocessor.modConfig)
def __init__(self, commuChannel, taskBufferIF, ddmIF, vos,
prodSourceLabels):
self.vos = self.parseInit(vos)
self.prodSourceLabels = self.parseInit(prodSourceLabels)
JediKnight.__init__(self, commuChannel, taskBufferIF, ddmIF, logger)
FactoryBase.__init__(self, self.vos, self.prodSourceLabels, logger,
jedi_config.taskbroker.modConfig)
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self,self.taskBufferIF,self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.info('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# prepare tasks to be finished
tmpLog.info('preparing tasks to be finished for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.prepareTasksToBeFinished_JEDI(vo,prodSourceLabel,
jedi_config.postprocessor.nTasks,
pid=self.pid)
if tmpRet == None:
# failed
tmpLog.error('failed to prepare tasks')
# get tasks to be finished
tmpLog.info('getting tasks to be finished')
tmpList = self.taskBufferIF.getTasksToBeFinished_JEDI(vo,prodSourceLabel,self.pid,
jedi_config.postprocessor.nTasks)
if tmpList == None:
# failed
tmpLog.error('failed to get tasks to be finished')
else:
tmpLog.info('got {0} tasks'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.postprocessor.nWorkers
for iWorker in range(nWorker):
thr = PostProcessorThread(taskList,threadPool,
self.taskBufferIF,
self.ddmIF,
self)
thr.start()
# join
threadPool.join()
tmpLog.info('done')
except:
errtype,errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,errtype.__name__,errvalue))
# sleep if needed
loopCycle = 60
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self,self.taskBufferIF,self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.info('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# rescue picked files
tmpLog.info('rescue tasks with picked files for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.rescuePickedFiles_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForPicked)
if tmpRet == None:
# failed
tmpLog.error('failed to rescue')
else:
tmpLog.info('rescued {0} tasks'.format(tmpRet))
# reactivate pending tasks
tmpLog.info('reactivate pending tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.reactivatePendingTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForPending,
jedi_config.watchdog.timeoutForPending)
if tmpRet == None:
# failed
tmpLog.error('failed to reactivate')
else:
tmpLog.info('reactivated {0} tasks'.format(tmpRet))
# vo/prodSourceLabel specific action
impl = self.getImpl(vo,prodSourceLabel)
if impl != None:
tmpLog.info('special action for vo={0} label={1} with {2}'.format(vo,prodSourceLabel,impl.__class__.__name__))
tmpStat = impl.doAction()
if tmpStat != Interaction.SC_SUCCEEDED:
tmpLog.error('failed to run special acction for vo={0} label={1}'.format(vo,prodSourceLabel))
else:
tmpLog.info('done for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpLog.info('done')
except:
errtype,errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,errtype.__name__,errvalue))
# sleep if needed
loopCycle = jedi_config.watchdog.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# get the list of tasks to refine
tmpList = self.taskBufferIF.getTasksToRefine_JEDI(vo, prodSourceLabel)
if tmpList == None:
# failed
tmpLog.error('failed to get the list of tasks to refine')
else:
tmpLog.debug('got {0} tasks'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# get work queue mapper
workQueueMapper = self.taskBufferIF.getWorkQueueMap()
# make workers
nWorker = jedi_config.taskrefine.nWorkers
for iWorker in range(nWorker):
thr = TaskRefinerThread(taskList, threadPool,
self.taskBufferIF,
self.ddmIF,
self, workQueueMapper)
thr.start()
# join
threadPool.join()
except:
errtype, errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,
errtype.__name__, errvalue))
tmpLog.error('Traceback: {0}'.format(traceback.format_exc()))
# sleep if needed
loopCycle = jedi_config.taskrefine.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# get the list of tasks to refine
tmpList = self.taskBufferIF.getTasksToRefine_JEDI(vo, prodSourceLabel)
if tmpList == None:
# failed
tmpLog.error('failed to get the list of tasks to refine')
else:
tmpLog.debug('got {0} tasks'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# get work queue mapper
workQueueMapper = self.taskBufferIF.getWorkQueueMap()
# make workers
nWorker = jedi_config.taskrefine.nWorkers
for iWorker in range(nWorker):
thr = TaskRefinerThread(taskList, threadPool,
self.taskBufferIF,
self.ddmIF,
self, workQueueMapper)
thr.start()
# join
threadPool.join()
except:
errtype, errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,
errtype.__name__, errvalue))
tmpLog.error('Traceback: {0}'.format(traceback.format_exc()))
# sleep if needed
loopCycle = jedi_config.taskrefine.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def start(self):
# start base class
JediKnight.start(self)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# get the list of datasets to feed contents to DB
tmpList = self.taskBufferIF.getDatasetsToFeedContents_JEDI(
vo, prodSourceLabel)
if tmpList == None:
# failed
logger.error(
'failed to get the list of datasets to feed contents'
)
else:
logger.debug('got %s datasets' % len(tmpList))
# put to a locked list
dsList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.confeeder.nWorkers
for iWorker in range(nWorker):
thr = ContentsFeederThread(
dsList, threadPool, self.taskBufferIF,
self.ddmIF, self.pid)
thr.start()
# join
threadPool.join()
except:
errtype, errvalue = sys.exc_info()[:2]
logger.error(
'failed in %s.start() with %s %s' %
(self.__class__.__name__, errtype.__name__, errvalue))
# sleep if needed
loopCycle = jedi_config.confeeder.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def start(self):
# start base class
JediKnight.start(self)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# get the list of datasets to feed contents to DB
tmpList = self.taskBufferIF.getDatasetsToFeedContents_JEDI(vo,prodSourceLabel)
if tmpList == None:
# failed
logger.error('failed to get the list of datasets to feed contents')
else:
logger.debug('got %s datasets' % len(tmpList))
# put to a locked list
dsList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.confeeder.nWorkers
for iWorker in range(nWorker):
thr = ContentsFeederThread(dsList,threadPool,
self.taskBufferIF,self.ddmIF,
self.pid)
thr.start()
# join
threadPool.join()
except:
errtype,errvalue = sys.exc_info()[:2]
logger.error('failed in %s.start() with %s %s' % (self.__class__.__name__,errtype.__name__,errvalue))
# sleep if needed
loopCycle = jedi_config.confeeder.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start TaskBroker')
# get work queue mapper
workQueueMapper = self.taskBufferIF.getWorkQueueMap()
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# loop over all work queues
for workQueue in workQueueMapper.getQueueListWithVoType(
vo, prodSourceLabel):
msgLabel = 'vo={0} label={1} queue={2}: '.format(
vo, prodSourceLabel, workQueue.queue_name)
tmpLog.debug(msgLabel + 'start')
# get the list of tasks to check
tmpList = self.taskBufferIF.getTasksToCheckAssignment_JEDI(
vo, prodSourceLabel, workQueue)
if tmpList == None:
# failed
tmpLog.error(
msgLabel +
'failed to get the list of tasks to check')
else:
tmpLog.debug(msgLabel +
'got {0} tasks to check'.format(
len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.taskbroker.nWorkers
for iWorker in range(nWorker):
thr = TaskCheckerThread(
taskList, threadPool,
self.taskBufferIF, self.ddmIF, self,
vo, prodSourceLabel)
thr.start()
# join
threadPool.join()
# get the list of tasks to assign
tmpList = self.taskBufferIF.getTasksToAssign_JEDI(
vo, prodSourceLabel, workQueue)
if tmpList == None:
# failed
tmpLog.error(
msgLabel +
'failed to get the list of tasks to assign'
)
else:
tmpLog.debug(msgLabel +
'got {0} tasks to assign'.format(
len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.taskbroker.nWorkers
for iWorker in range(nWorker):
thr = TaskBrokerThread(
taskList, threadPool,
self.taskBufferIF, self.ddmIF, self,
vo, prodSourceLabel, workQueue)
thr.start()
# join
threadPool.join()
tmpLog.debug(msgLabel + 'done')
except:
errtype, errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(
self.__class__.__name__, errtype.__name__, errvalue))
tmpLog.debug('done')
# sleep if needed
loopCycle = jedi_config.taskbroker.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def __init__(self,commuChannel,taskBufferIF,ddmIF,vos,prodSourceLabels):
self.vos = self.parseInit(vos)
self.prodSourceLabels = self.parseInit(prodSourceLabels)
self.pid = '{0}-{1}-dog'.format(socket.getfqdn().split('.')[0],os.getpid())
JediKnight.__init__(self,commuChannel,taskBufferIF,ddmIF,logger)
def __init__(self,commuChannel,taskBufferIF,ddmIF,vos,prodSourceLabels):
self.vos = self.parseInit(vos)
self.prodSourceLabels = self.parseInit(prodSourceLabels)
JediKnight.__init__(self,commuChannel,taskBufferIF,ddmIF,logger)
FactoryBase.__init__(self,self.vos,self.prodSourceLabels,logger,
jedi_config.taskrefine.modConfig)
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.info('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# rescue picked files
tmpLog.info(
'rescue tasks with picked files for vo={0} label={1}'
.format(vo, prodSourceLabel))
tmpRet = self.taskBufferIF.rescuePickedFiles_JEDI(
vo, prodSourceLabel,
jedi_config.watchdog.waitForPicked)
if tmpRet == None:
# failed
tmpLog.error('failed to rescue')
else:
tmpLog.info('rescued {0} tasks'.format(tmpRet))
# reactivate pending tasks
tmpLog.info(
'reactivate pending tasks for vo={0} label={1}'.
format(vo, prodSourceLabel))
tmpRet = self.taskBufferIF.reactivatePendingTasks_JEDI(
vo, prodSourceLabel,
jedi_config.watchdog.waitForPending,
jedi_config.watchdog.timeoutForPending)
if tmpRet == None:
# failed
tmpLog.error('failed to reactivate')
else:
tmpLog.info('reactivated {0} tasks'.format(tmpRet))
# unlock tasks
tmpLog.info('unlock tasks for vo={0} label={1}'.format(
vo, prodSourceLabel))
tmpRet = self.taskBufferIF.unlockTasks_JEDI(
vo, prodSourceLabel,
jedi_config.watchdog.waitForLocked)
if tmpRet == None:
# failed
tmpLog.error('failed to unlock')
else:
tmpLog.info('unlock {0} tasks'.format(tmpRet))
# restart contents update
tmpLog.info(
'restart contents update for vo={0} label={1}'.
format(vo, prodSourceLabel))
tmpRet = self.taskBufferIF.restartTasksForContentsUpdate_JEDI(
vo, prodSourceLabel)
if tmpRet == None:
# failed
tmpLog.error('failed to restart')
else:
tmpLog.info('restarted {0} tasks'.format(tmpRet))
# kick exhausted tasks
tmpLog.info(
'kick exhausted tasks for vo={0} label={1}'.format(
vo, prodSourceLabel))
tmpRet = self.taskBufferIF.kickExhaustedTasks_JEDI(
vo, prodSourceLabel,
jedi_config.watchdog.waitForExhausted)
if tmpRet == None:
# failed
tmpLog.error('failed to kick')
else:
tmpLog.info('kicked {0} tasks'.format(tmpRet))
# finish tasks when goal is reached
tmpLog.info(
'finish achieved tasks for vo={0} label={1}'.
format(vo, prodSourceLabel))
tmpRet = self.taskBufferIF.getAchievedTasks_JEDI(
vo, prodSourceLabel,
jedi_config.watchdog.waitForAchieved)
if tmpRet == None:
# failed
tmpLog.error('failed to finish')
else:
for jediTaskID in tmpRet:
self.taskBufferIF.sendCommandTaskPanda(
jediTaskID,
'JEDI. Goal reached',
True,
'finish',
comQualifier='soft')
tmpLog.info('finished {0} tasks'.format(tmpRet))
# vo/prodSourceLabel specific action
impl = self.getImpl(vo, prodSourceLabel)
if impl != None:
tmpLog.info(
'special action for vo={0} label={1} with {2}'.
format(vo, prodSourceLabel,
impl.__class__.__name__))
tmpStat = impl.doAction()
if tmpStat != Interaction.SC_SUCCEEDED:
tmpLog.error(
'failed to run special acction for vo={0} label={1}'
.format(vo, prodSourceLabel))
else:
tmpLog.info('done for vo={0} label={1}'.format(
vo, prodSourceLabel))
tmpLog.info('done')
except:
errtype, errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(
self.__class__.__name__, errtype.__name__, errvalue))
# sleep if needed
loopCycle = jedi_config.watchdog.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self,self.taskBufferIF,self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start TaskBroker')
# get work queue mapper
workQueueMapper = self.taskBufferIF.getWorkQueueMap()
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# loop over all work queues
for workQueue in workQueueMapper.getQueueListWithVoType(vo,prodSourceLabel):
msgLabel = 'vo={0} label={1} queue={2}: '.format(vo,prodSourceLabel,workQueue.queue_name)
tmpLog.debug(msgLabel+'start')
# get the list of tasks to check
tmpList = self.taskBufferIF.getTasksToCheckAssignment_JEDI(vo,prodSourceLabel,workQueue)
if tmpList == None:
# failed
tmpLog.error(msgLabel+'failed to get the list of tasks to check')
else:
tmpLog.debug(msgLabel+'got {0} tasks to check'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.taskbroker.nWorkers
for iWorker in range(nWorker):
thr = TaskCheckerThread(taskList,threadPool,
self.taskBufferIF,
self.ddmIF,self,
vo,prodSourceLabel)
thr.start()
# join
threadPool.join()
# get the list of tasks to assign
tmpList = self.taskBufferIF.getTasksToAssign_JEDI(vo,prodSourceLabel,workQueue)
if tmpList == None:
# failed
tmpLog.error(msgLabel+'failed to get the list of tasks to assign')
else:
tmpLog.debug(msgLabel+'got {0} tasks to assign'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.taskbroker.nWorkers
for iWorker in range(nWorker):
thr = TaskBrokerThread(taskList,threadPool,
self.taskBufferIF,
self.ddmIF,self,
vo,prodSourceLabel,
workQueue)
thr.start()
# join
threadPool.join()
tmpLog.debug(msgLabel+'done')
except:
errtype,errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,
errtype.__name__,errvalue))
tmpLog.debug('done')
# sleep if needed
loopCycle = jedi_config.taskbroker.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()
def __init__(self, commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels):
self.vos = self.parseInit(vos)
self.prodSourceLabels = self.parseInit(prodSourceLabels)
JediKnight.__init__(self, commuChannel, taskBufferIF, ddmIF, logger)
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self,self.taskBufferIF,self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.info('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# rescue picked files
tmpLog.info('rescue tasks with picked files for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.rescuePickedFiles_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForPicked)
if tmpRet == None:
# failed
tmpLog.error('failed to rescue')
else:
tmpLog.info('rescued {0} tasks'.format(tmpRet))
# reactivate pending tasks
tmpLog.info('reactivate pending tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
timeoutForPending = None
if hasattr(jedi_config.watchdog,'timeoutForPendingVoLabel'):
timeoutForPending = JediCoreUtils.getConfigParam(jedi_config.watchdog.timeoutForPendingVoLabel,vo,prodSourceLabel)
if timeoutForPending == None:
timeoutForPending = jedi_config.watchdog.timeoutForPending
timeoutForPending = int(timeoutForPending)
tmpRet = self.taskBufferIF.reactivatePendingTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForPending,
timeoutForPending)
if tmpRet == None:
# failed
tmpLog.error('failed to reactivate')
else:
tmpLog.info('reactivated {0} tasks'.format(tmpRet))
# unlock tasks
tmpLog.info('unlock tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.unlockTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForLocked)
if tmpRet == None:
# failed
tmpLog.error('failed to unlock')
else:
tmpLog.info('unlock {0} tasks'.format(tmpRet))
# restart contents update
tmpLog.info('restart contents update for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.restartTasksForContentsUpdate_JEDI(vo,prodSourceLabel)
if tmpRet == None:
# failed
tmpLog.error('failed to restart')
else:
tmpLog.info('restarted {0} tasks'.format(tmpRet))
# kick exhausted tasks
tmpLog.info('kick exhausted tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.kickExhaustedTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForExhausted)
if tmpRet == None:
# failed
tmpLog.error('failed to kick')
else:
tmpLog.info('kicked {0} tasks'.format(tmpRet))
# finish tasks when goal is reached
tmpLog.info('finish achieved tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.getAchievedTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForAchieved)
if tmpRet == None:
# failed
tmpLog.error('failed to finish')
else:
for jediTaskID in tmpRet:
self.taskBufferIF.sendCommandTaskPanda(jediTaskID,'JEDI. Goal reached',True,'finish',comQualifier='soft')
tmpLog.info('finished {0} tasks'.format(tmpRet))
# vo/prodSourceLabel specific action
impl = self.getImpl(vo,prodSourceLabel)
if impl != None:
tmpLog.info('special action for vo={0} label={1} with {2}'.format(vo,prodSourceLabel,impl.__class__.__name__))
tmpStat = impl.doAction()
if tmpStat != Interaction.SC_SUCCEEDED:
tmpLog.error('failed to run special acction for vo={0} label={1}'.format(vo,prodSourceLabel))
else:
tmpLog.info('done for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpLog.info('done')
except:
errtype,errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,errtype.__name__,errvalue))
# sleep if needed
loopCycle = jedi_config.watchdog.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()