def run(self): # make sure that has_more_work() returns a correct value since any appmgr thread which may be started # by the ITaskScheduler.run() below may use it self.processing = True # must call base class implementation first ITaskScheduler.run(self) from diane.util.periodic_call import PeriodicCall msg = PeriodicCall(1,logger.info) while self.appmgr.has_more_work() and not self.should_stop(): #print 'waiting workers:',self.job_master.worker_registry.waiting_workers for w in self.job_master.worker_registry.waiting_workers.values(): #logger.debug('***** checking policy.WORKER_TIME_LIMIT %s %s',self.policy, dir(self.policy)) if self.policy.WORKER_TIME_LIMIT: if time.time()-w.registration_time > self.policy.WORKER_TIME_LIMIT: logger.info('removing worker %s due to exceeded WORKER_TIME_LIMIT',w.wid) self.job_master.remove_worker(w) continue tasks = [] try: for i in range(w.capacity()): tasks.append(self.todo_tasks.get(w)) except Queue.Empty: pass if tasks: logger.info('scheduling tasks tid=%s to worker wid=%d',[t.tid for t in tasks],w.wid) self.appmgr.make_input(tasks,w) self.worker_attempts[w.wid] += len(tasks) self.job_master.schedule(w,tasks) else: break try: while True: t = self.completed_tasks.get(False) r = t.details.task_output logger.info('consuming task tid=%d from worker wid=%d',t.tid,t.details.assigned_wid) logger.debug('task tid=%d -> result=%s',t.tid,r) self.appmgr.tasks_done([t]) # clean input and output associated with the task if self.policy.TASK_INPUT_CLEANUP: del t.task_input if self.policy.TASK_OUTPUT_CLEANUP: del t.details.task_output except Queue.Empty: pass #msg("todo %s, completed %s",[t.tid for t in self.todo_tasks.queue], [t.tid for t in self.completed_tasks.queue]) time.sleep(0.1) def dump_state(f): print >>f, 'completed tasks:', len(self.completed_tasks.queue) print >>f, 'todo tasks:', len(self.todo_tasks) self.appmgr.finalize() logger.info('dumping processing state to file: tasks.txt') dump_state(file('tasks.txt','w')) logger.info('terminating task manager...') # we end processing AFTER appmgr.finalize() has completed (bugfix:http://savannah.cern.ch/bugs/?73322) self.processing = False
def __init__(self,job_master,appmgr): ITaskScheduler.__init__(self,job_master,appmgr) self.todo_tasks = TodoTaskQueue() self.completed_tasks = Queue.Queue() self.worker_attempts = {} self.processing = False