def run(self):
# make sure that has_more_work() returns a correct value since any appmgr thread which may be started
# by the ITaskScheduler.run() below may use it
self.processing = True
# must call base class implementation first
ITaskScheduler.run(self)
from diane.util.periodic_call import PeriodicCall
msg = PeriodicCall(1,logger.info)
while self.appmgr.has_more_work() and not self.should_stop():
#print 'waiting workers:',self.job_master.worker_registry.waiting_workers
for w in self.job_master.worker_registry.waiting_workers.values():
#logger.debug('***** checking policy.WORKER_TIME_LIMIT %s %s',self.policy, dir(self.policy))
if self.policy.WORKER_TIME_LIMIT:
if time.time()-w.registration_time > self.policy.WORKER_TIME_LIMIT:
logger.info('removing worker %s due to exceeded WORKER_TIME_LIMIT',w.wid)
self.job_master.remove_worker(w)
continue
tasks = []
try:
for i in range(w.capacity()):
tasks.append(self.todo_tasks.get(w))
except Queue.Empty:
pass
if tasks:
logger.info('scheduling tasks tid=%s to worker wid=%d',[t.tid for t in tasks],w.wid)
self.appmgr.make_input(tasks,w)
self.worker_attempts[w.wid] += len(tasks)
self.job_master.schedule(w,tasks)
else:
break
try:
while True:
t = self.completed_tasks.get(False)
r = t.details.task_output
logger.info('consuming task tid=%d from worker wid=%d',t.tid,t.details.assigned_wid)
logger.debug('task tid=%d -> result=%s',t.tid,r)
self.appmgr.tasks_done([t])
# clean input and output associated with the task
if self.policy.TASK_INPUT_CLEANUP:
del t.task_input
if self.policy.TASK_OUTPUT_CLEANUP:
del t.details.task_output
except Queue.Empty:
pass
#msg("todo %s, completed %s",[t.tid for t in self.todo_tasks.queue], [t.tid for t in self.completed_tasks.queue])
time.sleep(0.1)
def dump_state(f):
print >>f, 'completed tasks:', len(self.completed_tasks.queue)
print >>f, 'todo tasks:', len(self.todo_tasks)
self.appmgr.finalize()
logger.info('dumping processing state to file: tasks.txt')
dump_state(file('tasks.txt','w'))
logger.info('terminating task manager...')
# we end processing AFTER appmgr.finalize() has completed (bugfix:http://savannah.cern.ch/bugs/?73322)
self.processing = False
def __init__(self,job_master,appmgr):
ITaskScheduler.__init__(self,job_master,appmgr)
self.todo_tasks = TodoTaskQueue()
self.completed_tasks = Queue.Queue()
self.worker_attempts = {}
self.processing = False
