def __init__(self, *args, **kwargs): if kwargs is not None: if 'count' in kwargs: self._wcnt = kwargs['count'] else: self._wcnt = WorkerPool.DEFAULT_WORKER_COUNT if 'all_done' in kwargs: self._all_done = kwargs['all_done'] else: def ___pass(*args, **kwargs): pass self._all_done = ___pass if 'tick' in kwargs: self._tick = kwargs['tick'] self._wpol = list([]) # all workers self._wbsy = dict({}) # busy self._wfre = list([]) # free self._tque = list([]) # task queue self._tick = WorkerPool.DEFAULT_TICK # sleep time if all busy self._wrkr = SchedulerThread(0) # task schedule is worker self._task = Task() # taks for scheduler self._task.target = self._task_schedule self._task.args = tuple() self._task.kwargs = dict() self._ball_done = False def _on_worker_task_done(worker): """ default calback for scheduler when some worker is done with a task it is appended to free workers queue also marked free in busy dict """ self._wfre.append(worker) self._wbsy[worker._id] = False self.____f = _on_worker_task_done # pool initialization for i in xrange(1, self._wcnt + 1): w = WorkerThread(i) w._pool = self # back reference w._worker_task_done = self.____f self._wpol.append(w) self._wfre.append(w) self._wbsy[i] = False self._wrkr._target = self._task_schedule self._wrkr._args = tuple() self._wrkr._kwargs = dict()
def __init__(self,*args, **kwargs): if kwargs is not None: if 'count' in kwargs: self._wcnt = kwargs['count'] else: self._wcnt = WorkerPool.DEFAULT_WORKER_COUNT if 'all_done' in kwargs: self._all_done = kwargs['all_done'] else: def ___pass( *args, **kwargs ): pass self._all_done = ___pass if 'tick' in kwargs: self._tick = kwargs['tick'] self._wpol = list([]) # all workers self._wbsy = dict({}) # busy self._wfre = list([]) # free self._tque = list([]) # task queue self._tick = WorkerPool.DEFAULT_TICK # sleep time if all busy self._wrkr = SchedulerThread(0) # task schedule is worker self._task = Task() # taks for scheduler self._task.target = self._task_schedule self._task.args = tuple() self._task.kwargs = dict() self._ball_done = False def _on_worker_task_done( worker ): """ default calback for scheduler when some worker is done with a task it is appended to free workers queue also marked free in busy dict """ self._wfre.append(worker) self._wbsy[worker._id] = False self.____f = _on_worker_task_done # pool initialization for i in xrange(1,self._wcnt+1) : w = WorkerThread(i) w._pool = self # back reference w._worker_task_done = self.____f self._wpol.append(w) self._wfre.append(w) self._wbsy[i] = False self._wrkr._target = self._task_schedule self._wrkr._args = tuple() self._wrkr._kwargs = dict()
class WorkerPool(object): """ maintain asynchronous task distrubution over group ow working threads """ #time to sleep in scheduler when all workers are busy:bsy DEFAULT_TICK = 0.01 DEFAULT_WORKER_COUNT = 4 # as lower is as scheduler loop consume more cpu over single thread def __init__(self, *args, **kwargs): if kwargs is not None: if 'count' in kwargs: self._wcnt = kwargs['count'] else: self._wcnt = WorkerPool.DEFAULT_WORKER_COUNT if 'all_done' in kwargs: self._all_done = kwargs['all_done'] else: def ___pass(*args, **kwargs): pass self._all_done = ___pass if 'tick' in kwargs: self._tick = kwargs['tick'] self._wpol = list([]) # all workers self._wbsy = dict({}) # busy self._wfre = list([]) # free self._tque = list([]) # task queue self._tick = WorkerPool.DEFAULT_TICK # sleep time if all busy self._wrkr = SchedulerThread(0) # task schedule is worker self._task = Task() # taks for scheduler self._task.target = self._task_schedule self._task.args = tuple() self._task.kwargs = dict() self._ball_done = False def _on_worker_task_done(worker): """ default calback for scheduler when some worker is done with a task it is appended to free workers queue also marked free in busy dict """ self._wfre.append(worker) self._wbsy[worker._id] = False self.____f = _on_worker_task_done # pool initialization for i in xrange(1, self._wcnt + 1): if 'mp' in kwargs: w = WorkerThread(i, mp=kwargs['mp']) else: w = WorkerThread(i) w._pool = self # back reference w._worker_task_done = self.____f self._wpol.append(w) self._wfre.append(w) self._wbsy[i] = False self._wrkr._target = self._task_schedule self._wrkr._args = tuple() self._wrkr._kwargs = dict() def _task_schedule(self): """ task scheduler worker target function: check for tasks in queue schedule to free workers """ #print ">>>>>>>>> task schedule" #loop = True while not self._wrkr._bexit: #print "schedule loop" if self._tque: #print "have in que" task = self._tque.pop() if self._wfre: #print "have free workers" w = self._wfre.pop() success = w._set_task(task) if success: self._wbsy[w._id] = True else: #print "attach task to woker failed" self._wbsy[w._id] = False self._tque.append(task) sleep(self._tick) else: #print "thre is no free workers" sleep(self._tick) self._tque.append(task) self._wrkr._e.clear() self._wrkr._e.set() else: #print "noting in que" bsy_cnt = 0 if len(self._wfre) == len(self._wpol): #print "all done" self._ball_done = True self._all_done(self) break else: sleep(self._tick) self._wrkr._e.clear() self._wrkr._e.set() def _task_schedule_done(self, task_result): #print "task chedule done" pass def _append_task(self, task): self._tque.append(task) def _commit_tasks(self): self._wrkr._e.clear() self._wrkr._e.set() def _exit(self): for w in self._wpol: w._exit() self._wrkr._exit()
class WorkerPool(object): """ maintain asynchronous task distrubution over group ow working threads """ # time to sleep in scheduler when all workers are busy:bsy DEFAULT_TICK = 0.01 DEFAULT_WORKER_COUNT = 4 # as lower is as scheduler loop consume more cpu over single thread def __init__(self, *args, **kwargs): if kwargs is not None: if "count" in kwargs: self._wcnt = kwargs["count"] else: self._wcnt = WorkerPool.DEFAULT_WORKER_COUNT if "all_done" in kwargs: self._all_done = kwargs["all_done"] else: def ___pass(*args, **kwargs): pass self._all_done = ___pass if "tick" in kwargs: self._tick = kwargs["tick"] self._wpol = list([]) # all workers self._wbsy = dict({}) # busy self._wfre = list([]) # free self._tque = list([]) # task queue self._tick = WorkerPool.DEFAULT_TICK # sleep time if all busy self._wrkr = SchedulerThread(0) # task schedule is worker self._task = Task() # taks for scheduler self._task.target = self._task_schedule self._task.args = tuple() self._task.kwargs = dict() self._ball_done = False def _on_worker_task_done(worker): """ default calback for scheduler when some worker is done with a task it is appended to free workers queue also marked free in busy dict """ self._wfre.append(worker) self._wbsy[worker._id] = False self.____f = _on_worker_task_done # pool initialization for i in xrange(1, self._wcnt + 1): if "mp" in kwargs: w = WorkerThread(i, mp=kwargs["mp"]) else: w = WorkerThread(i) w._pool = self # back reference w._worker_task_done = self.____f self._wpol.append(w) self._wfre.append(w) self._wbsy[i] = False self._wrkr._target = self._task_schedule self._wrkr._args = tuple() self._wrkr._kwargs = dict() def _task_schedule(self): """ task scheduler worker target function: check for tasks in queue schedule to free workers """ # print ">>>>>>>>> task schedule" # loop = True while not self._wrkr._bexit: # print "schedule loop" if self._tque: # print "have in que" task = self._tque.pop() if self._wfre: # print "have free workers" w = self._wfre.pop() success = w._set_task(task) if success: self._wbsy[w._id] = True else: # print "attach task to woker failed" self._wbsy[w._id] = False self._tque.append(task) sleep(self._tick) else: # print "thre is no free workers" sleep(self._tick) self._tque.append(task) self._wrkr._e.clear() self._wrkr._e.set() else: # print "noting in que" bsy_cnt = 0 if len(self._wfre) == len(self._wpol): # print "all done" self._ball_done = True self._all_done(self) break else: sleep(self._tick) self._wrkr._e.clear() self._wrkr._e.set() def _task_schedule_done(self, task_result): # print "task chedule done" pass def _append_task(self, task): self._tque.append(task) def _commit_tasks(self): self._wrkr._e.clear() self._wrkr._e.set() def _exit(self): for w in self._wpol: w._exit() self._wrkr._exit()