def _handleTasks(taskqueue, inqueue, outqueue, pool): thread = threading.currentThread() put = inqueue._writer.send for taskseq, setLength in iter(taskqueue.get, None): i = -1 for i, task in enumerate(taskseq): if thread._state: debug('task handler found thread._state != RUN') break put(task) else: if setLength: debug('doing setLength()') setLength(i+1) continue break else: debug('task handler got sentinel') # tell result handler to finish when cache is empty outqueue.put(None) # tell workers there is no more work debug('task handler sending sentinel to workers') for p in pool: put(None) debug('task handler exiting')
def temp(self, *args, **kwds): debug('requesting creation of a shared %r object', typeid) token, exp = self._create(typeid, *args, **kwds) proxy = proxytype( token, manager=self, authkey=self._authkey, exposed=exp ) return proxy
def _handleTasks(taskqueue, inqueue, outqueue, pool): thread = threading.currentThread() put = inqueue._writer.send for taskseq, setLength in iter(taskqueue.get, None): i = -1 for i, task in enumerate(taskseq): if thread._state: debug('task handler found thread._state != RUN') break put(task) else: if setLength: debug('doing setLength()') setLength(i + 1) continue break else: debug('task handler got sentinel') # tell result handler to finish when cache is empty outqueue.put(None) # tell workers there is no more work debug('task handler sending sentinel to workers') for p in pool: put(None) debug('task handler exiting')
def cancelJoin(self): debug('Queue.cancelJoin()') self._joincancelled = True try: self._jointhread.cancel() except AttributeError: pass
def join(self): debug('joining pool') assert self._state in (CLOSE, TERMINATE) self._task_handler.join() self._result_handler.join() for p in self._pool: p.join()
def _finalizeClose(buffer, notempty): debug('telling queue thread to quit') notempty.acquire() try: buffer.append(_sentinel) notempty.notify() finally: notempty.release()
def _connect(self): debug('making connection to manager') name = currentProcess().getName() if threading.currentThread().getName() != 'MainThread': name += '|' + threading.currentThread().getName() connection = Client(self._token.address, authkey=self._authkey) dispatch(connection, None, 'acceptConnection', (name,)) self._tls.connection = connection
def _connect(self): debug('making connection to manager') name = currentProcess().getName() if threading.currentThread().getName() != 'MainThread': name += '|' + threading.currentThread().getName() connection = Client(self._token.address, authkey=self._authkey) dispatch(connection, None, 'acceptConnection', (name, )) self._tls.connection = connection
def __init__(self, kind, value): sl = self._semlock = _processing.SemLock(kind, value) debug('created semlock with handle %s' % sl.handle) self.__setstate__((sl.handle, sl.kind, sl.maxvalue)) if sys.platform != 'win32': def _afterFork(obj): obj._semlock._afterFork() _registerAfterFork(self, _afterFork)
def _afterFork(self): debug('Queue._afterFork()') self._notempty = threading.Condition(threading.Lock()) self._buffer = collections.deque() self._thread = None self._jointhread = None self._joincancelled = False self._closed = False self._close = None
def __setstate__(self, state): self._state = state if not hasattr(self, '_semlock'): self._semlock = _processing.SemLock._rebuild(*state) debug('recreated blocker with handle %r' % state[0]) self.acquire = self._semlock.acquire self.release = self._semlock.release self.__enter__ = self._semlock.__enter__ self.__exit__ = self._semlock.__exit__
def decref(self, c, ident): self.mutex.acquire() try: assert self.id_to_refcount[ident] >= 1 self.id_to_refcount[ident] -= 1 if self.id_to_refcount[ident] == 0: del self.id_to_obj[ident], self.id_to_refcount[ident] debug('disposing of obj with id %d', ident) finally: self.mutex.release()
def serveClient(self, connection): ''' Handle requests from the proxies in a particular process/thread ''' debug('starting server thread to service %r', threading.currentThread().getName()) recv = connection.recv send = connection.send id_to_obj = self.id_to_obj while not self.stop: try: methodname = obj = None request = recv() ident, methodname, args, kwds = request obj, exposed = id_to_obj[ident] if methodname not in exposed: raise AttributeError, ( 'method %r of %r object is not in exposed=%r' % (methodname, type(obj), exposed) ) function = getattr(obj, methodname) try: result = function(*args, **kwds) msg = ('#RETURN', result) except (SystemExit, KeyboardInterrupt): raise except Exception, e: msg = ('#ERROR', e) except AttributeError, e: if methodname is None: msg = ('#ERROR', RemoteError()) else: try: fallback_func = self.fallback_mapping[methodname] result = fallback_func( self, connection, ident, obj, *args, **kwds ) msg = ('#RETURN', result) except (SystemExit, KeyboardInterrupt): raise except Exception: msg = ('#ERROR', RemoteError()) except EOFError: debug('got EOF -- exiting thread serving %r', threading.currentThread().getName()) sys.exit(0)
def _handleResults(outqueue, cache): thread = threading.currentThread() get = outqueue._reader.recv for job, i, obj in iter(get, None): if thread._state: assert thread._state == TERMINATE debug('result handler found thread._state=TERMINATE') return try: cache[job]._set(i, obj) except KeyError: pass else: debug('result handler got sentinel') while cache and thread._state != TERMINATE: item = get() if item is None: debug('result handler ignoring extra sentinel') continue job, i, obj = item try: cache[job]._set(i, obj) except KeyError: pass debug('result handler exiting: len(cache)=%s, thread._state=%s', len(cache), thread._state)
def _feed(buffer, notempty, send, writelock, close): debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release nwait = notempty.wait bpopleft = buffer.popleft sentinel = _sentinel if sys.platform != 'win32': wacquire = writelock.acquire wrelease = writelock.release else: wacquire = None try: while 1: nacquire() try: if not buffer: nwait() finally: nrelease() try: while 1: obj = bpopleft() if obj is sentinel: debug('feeder thread got sentinel -- exiting') close() return if wacquire is None: send(obj) else: wacquire() try: send(obj) finally: wrelease() except IndexError: pass except Exception, e: # Since this runs in a daemon thread the objects it uses # may be become unusable while the process is cleaning up. # We ignore errors which happen after the process has # started to cleanup. if currentProcess()._exiting: subWarning('error in queue thread: %s', e) else: raise
def _decref(token, authkey, shutdown, tls, idset): idset.remove(token.id) # check whether manager is still alive manager_still_alive = shutdown is None or shutdown.stillActive() if manager_still_alive: # tell manager this process no longer cares about referent try: debug('DECREF %r', token.id) connection = Client(token.address, authkey=authkey) dispatch(connection, None, 'decref', (token.id,)) except (SystemExit, KeyboardInterrupt): raise except Exception, e: debug('... decref failed %s', e)
def _incref(self): connection = Client(self._token.address, authkey=self._authkey) dispatch(connection, None, 'incref', (self._id, )) debug('INCREF %r', self._token.id) assert self._id not in self._idset self._idset.add(self._id) shutdown = getattr(self._manager, 'shutdown', None) self._close = Finalize(self, BaseProxy._decref, args=(self._token, self._authkey, shutdown, self._tls, self._idset), exitpriority=10)
def _decref(token, authkey, shutdown, tls, idset): idset.remove(token.id) # check whether manager is still alive manager_still_alive = shutdown is None or shutdown.stillActive() if manager_still_alive: # tell manager this process no longer cares about referent try: debug('DECREF %r', token.id) connection = Client(token.address, authkey=authkey) dispatch(connection, None, 'decref', (token.id, )) except (SystemExit, KeyboardInterrupt): raise except Exception, e: debug('... decref failed %s', e)
def _incref(self): connection = Client(self._token.address, authkey=self._authkey) dispatch(connection, None, 'incref', (self._id,)) debug('INCREF %r', self._token.id) assert self._id not in self._idset self._idset.add(self._id) shutdown = getattr(self._manager, 'shutdown', None) self._close = Finalize( self, BaseProxy._decref, args=(self._token, self._authkey, shutdown, self._tls, self._idset), exitpriority=10 )
def _getListener(): global _listener if _listener is None: _lock.acquire() try: if _listener is None: from processing.connection import Listener debug('starting listener and thread for sending handles') _listener = Listener(authenticate=True) t = threading.Thread(target=_serve) t.setDaemon(True) t.start() finally: _lock.release() return _listener
def shutdown(self, c): ''' Shutdown this process ''' c.send(('#RETURN', None)) info('manager received shutdown message') # do some cleaning up _runFinalizers(0) for p in activeChildren(): debug('terminating a child process of manager') p.terminate() for p in activeChildren(): debug('terminating a child process of manager') p.join() _runFinalizers() info('manager exiting with exitcode 0') # now exit without waiting for other threads to finish exit(0)
def _callMethod(self, methodname, args=(), kwds={}): ''' Try to call a method of the referrent and return a copy of the result ''' try: conn = self._tls.connection except AttributeError: debug('thread %r does not own a connection', threading.currentThread().getName()) self._connect() conn = self._tls.connection conn.send((self._id, methodname, args, kwds)) kind, result = conn.recv() if kind == '#RETURN': return result elif kind == '#ERROR': raise result else: raise ValueError
def _startThread(self): debug('Queue._startThread()') # Start thread which transfers data from buffer to pipe self._buffer.clear() self._thread = threading.Thread(target=Queue._feed, args=(self._buffer, self._notempty, self._send, self._wlock, self._writer.close), name='QueueFeederThread') self._thread.setDaemon(True) debug('doing self._thread.start()') self._thread.start() debug('... done self._thread.start()') # On process exit we will wait for data to be flushed to pipe. # # However, if this process created the queue then all # processes which use the queue will be descendants of this # process. Therefore waiting for the queue to be flushed # is pointless once all the child processes have been joined. created_by_this_process = (self._opid == os.getpid()) if not self._joincancelled and not created_by_this_process: self._jointhread = Finalize(self._thread, Queue._finalizeJoin, [weakref.ref(self._thread)], exitpriority=-5) # Send sentinel to the thread queue object when garbage collected self._close = Finalize(self, Queue._finalizeClose, [self._buffer, self._notempty], exitpriority=10)
def create(self, c, typeid, *args, **kwds): ''' Create a new shared object and return its id ''' self.mutex.acquire() try: callable, exposed = self.registry[typeid] obj = callable(*args, **kwds) if exposed is None: exposed = publicMethods(obj) ident = id(obj) debug('have created %r object with id %r', typeid, ident) self.id_to_obj[ident] = (obj, set(exposed)) if ident not in self.id_to_refcount: self.id_to_refcount[ident] = None return ident, tuple(exposed) finally: self.mutex.release()
def _finalizeJoin(twr): debug('joining queue thread') thread = twr() if thread is not None: thread.join() debug('... queue thread joined') else: debug('... queue thread already dead')
def _startThread(self): debug('Queue._startThread()') # Start thread which transfers data from buffer to pipe self._buffer.clear() self._thread = threading.Thread( target=Queue._feed, args=(self._buffer, self._notempty, self._send, self._wlock, self._writer.close), name='QueueFeederThread' ) self._thread.setDaemon(True) debug('doing self._thread.start()') self._thread.start() debug('... done self._thread.start()') # On process exit we will wait for data to be flushed to pipe. # # However, if this process created the queue then all # processes which use the queue will be descendants of this # process. Therefore waiting for the queue to be flushed # is pointless once all the child processes have been joined. created_by_this_process = (self._opid == os.getpid()) if not self._joincancelled and not created_by_this_process: self._jointhread = Finalize( self._thread, Queue._finalizeJoin, [weakref.ref(self._thread)], exitpriority=-5 ) # Send sentinel to the thread queue object when garbage collected self._close = Finalize( self, Queue._finalizeClose, [self._buffer, self._notempty], exitpriority=10 )
# check whether manager is still alive manager_still_alive = shutdown is None or shutdown.stillActive() if manager_still_alive: # tell manager this process no longer cares about referent try: debug('DECREF %r', token.id) connection = Client(token.address, authkey=authkey) dispatch(connection, None, 'decref', (token.id,)) except (SystemExit, KeyboardInterrupt): raise except Exception, e: debug('... decref failed %s', e) else: debug('DECREF %r -- manager already shutdown', token.id) # check whether we can close this thread's connection because # the process owns no more references to objects for this manager if not idset and hasattr(tls, 'connection'): debug('thread %r has no more proxies so closing conn', threading.currentThread().getName()) tls.connection.close() del tls.connection def _afterFork(self): self._manager = None self._incref() def __reduce__(self): if hasattr(self, '_exposed'):
def joinThread(self): debug('Queue.joinThread()') assert self._closed if self._jointhread: self._jointhread()
def terminate(self): debug('terminating pool') self._state = TERMINATE self._terminate()
def close(self): debug('closing pool') self._state = CLOSE self._taskqueue.put(None)
def _terminatePool(taskqueue, inqueue, outqueue, cache, pool, task_handler, result_handler): debug('finalizing pool') if not result_handler.isAlive(): debug('result handler already finished -- no need to terminate') return cache = {} task_handler._state = TERMINATE result_handler._state = TERMINATE debug('sending sentinels') taskqueue.put(None) outqueue.put(None) debug('getting read lock on inqueue') inqueue._rlock.acquire() debug('terminating workers') for p in pool: p.terminate() if task_handler.isAlive(): debug('removing tasks from inqueue until task handler finished') while task_handler.isAlive() and inqueue._reader.poll(): inqueue._reader.recv() time.sleep(0) debug('joining result handler') result_handler.join() debug('joining task handler') task_handler.join() debug('joining pool workers') for p in pool: p.join() debug('closing connections') inqueue._reader.close() outqueue._reader.close() inqueue._writer.close() outqueue._writer.close()
# def worker(inqueue, outqueue, initializer=None, initargs=()): put = outqueue.put if initializer is not None: initializer(*initargs) for job, i, func, args, kwds in iter(inqueue.get, None): try: result = (True, func(*args, **kwds)) except Exception, e: result = (False, e) put((job, i, result)) debug('worker got sentinel -- exiting') # # Class representing a process pool # class Pool(object): ''' Class which supports an async version of the `apply()` builtin ''' def __init__(self, processes=None, initializer=None, initargs=()): self._inqueue = SimpleQueue() self._outqueue = SimpleQueue() self._taskqueue = Queue.Queue() self._cache = {} self._state = RUN
# check whether manager is still alive manager_still_alive = shutdown is None or shutdown.stillActive() if manager_still_alive: # tell manager this process no longer cares about referent try: debug('DECREF %r', token.id) connection = Client(token.address, authkey=authkey) dispatch(connection, None, 'decref', (token.id, )) except (SystemExit, KeyboardInterrupt): raise except Exception, e: debug('... decref failed %s', e) else: debug('DECREF %r -- manager already shutdown', token.id) # check whether we can close this thread's connection because # the process owns no more references to objects for this manager if not idset and hasattr(tls, 'connection'): debug('thread %r has no more proxies so closing conn', threading.currentThread().getName()) tls.connection.close() del tls.connection def _afterFork(self): self._manager = None self._incref() def __reduce__(self): if hasattr(self, '_exposed'):
def worker(inqueue, outqueue, initializer=None, initargs=()): put = outqueue.put if initializer is not None: initializer(*initargs) for job, i, func, args, kwds in iter(inqueue.get, None): try: result = (True, func(*args, **kwds)) except Exception, e: result = (False, e) put((job, i, result)) debug('worker got sentinel -- exiting') # # Class representing a process pool # class Pool(object): ''' Class which supports an async version of the `apply()` builtin ''' def __init__(self, processes=None, initializer=None, initargs=()): self._inqueue = SimpleQueue() self._outqueue = SimpleQueue() self._taskqueue = Queue.Queue()