def _handleTasks(taskqueue, inqueue, outqueue, pool):
thread = threading.currentThread()
put = inqueue._writer.send
for taskseq, setLength in iter(taskqueue.get, None):
i = -1
for i, task in enumerate(taskseq):
if thread._state:
debug('task handler found thread._state != RUN')
break
put(task)
else:
if setLength:
debug('doing setLength()')
setLength(i+1)
continue
break
else:
debug('task handler got sentinel')
# tell result handler to finish when cache is empty
outqueue.put(None)
# tell workers there is no more work
debug('task handler sending sentinel to workers')
for p in pool:
put(None)
debug('task handler exiting')
def temp(self, *args, **kwds):
debug('requesting creation of a shared %r object', typeid)
token, exp = self._create(typeid, *args, **kwds)
proxy = proxytype(
token, manager=self, authkey=self._authkey, exposed=exp
)
return proxy
def _handleTasks(taskqueue, inqueue, outqueue, pool):
thread = threading.currentThread()
put = inqueue._writer.send
for taskseq, setLength in iter(taskqueue.get, None):
i = -1
for i, task in enumerate(taskseq):
if thread._state:
debug('task handler found thread._state != RUN')
break
put(task)
else:
if setLength:
debug('doing setLength()')
setLength(i + 1)
continue
break
else:
debug('task handler got sentinel')
# tell result handler to finish when cache is empty
outqueue.put(None)
# tell workers there is no more work
debug('task handler sending sentinel to workers')
for p in pool:
put(None)
debug('task handler exiting')
def cancelJoin(self):
debug('Queue.cancelJoin()')
self._joincancelled = True
try:
self._jointhread.cancel()
except AttributeError:
pass
def join(self):
debug('joining pool')
assert self._state in (CLOSE, TERMINATE)
self._task_handler.join()
self._result_handler.join()
for p in self._pool:
p.join()
def cancelJoin(self):
debug('Queue.cancelJoin()')
self._joincancelled = True
try:
self._jointhread.cancel()
except AttributeError:
pass
def temp(self, *args, **kwds):
debug('requesting creation of a shared %r object', typeid)
token, exp = self._create(typeid, *args, **kwds)
proxy = proxytype(
token, manager=self, authkey=self._authkey, exposed=exp
)
return proxy
def join(self):
debug('joining pool')
assert self._state in (CLOSE, TERMINATE)
self._task_handler.join()
self._result_handler.join()
for p in self._pool:
p.join()
def _finalizeClose(buffer, notempty):
debug('telling queue thread to quit')
notempty.acquire()
try:
buffer.append(_sentinel)
notempty.notify()
finally:
notempty.release()
def _finalizeClose(buffer, notempty):
debug('telling queue thread to quit')
notempty.acquire()
try:
buffer.append(_sentinel)
notempty.notify()
finally:
notempty.release()
def _connect(self):
debug('making connection to manager')
name = currentProcess().getName()
if threading.currentThread().getName() != 'MainThread':
name += '|' + threading.currentThread().getName()
connection = Client(self._token.address, authkey=self._authkey)
dispatch(connection, None, 'acceptConnection', (name,))
self._tls.connection = connection
def _connect(self):
debug('making connection to manager')
name = currentProcess().getName()
if threading.currentThread().getName() != 'MainThread':
name += '|' + threading.currentThread().getName()
connection = Client(self._token.address, authkey=self._authkey)
dispatch(connection, None, 'acceptConnection', (name, ))
self._tls.connection = connection
def __init__(self, kind, value):
sl = self._semlock = _processing.SemLock(kind, value)
debug('created semlock with handle %s' % sl.handle)
self.__setstate__((sl.handle, sl.kind, sl.maxvalue))
if sys.platform != 'win32':
def _afterFork(obj):
obj._semlock._afterFork()
_registerAfterFork(self, _afterFork)
def _afterFork(self):
debug('Queue._afterFork()')
self._notempty = threading.Condition(threading.Lock())
self._buffer = collections.deque()
self._thread = None
self._jointhread = None
self._joincancelled = False
self._closed = False
self._close = None
def _afterFork(self):
debug('Queue._afterFork()')
self._notempty = threading.Condition(threading.Lock())
self._buffer = collections.deque()
self._thread = None
self._jointhread = None
self._joincancelled = False
self._closed = False
self._close = None
def __setstate__(self, state):
self._state = state
if not hasattr(self, '_semlock'):
self._semlock = _processing.SemLock._rebuild(*state)
debug('recreated blocker with handle %r' % state[0])
self.acquire = self._semlock.acquire
self.release = self._semlock.release
self.__enter__ = self._semlock.__enter__
self.__exit__ = self._semlock.__exit__
def decref(self, c, ident):
self.mutex.acquire()
try:
assert self.id_to_refcount[ident] >= 1
self.id_to_refcount[ident] -= 1
if self.id_to_refcount[ident] == 0:
del self.id_to_obj[ident], self.id_to_refcount[ident]
debug('disposing of obj with id %d', ident)
finally:
self.mutex.release()
def decref(self, c, ident):
self.mutex.acquire()
try:
assert self.id_to_refcount[ident] >= 1
self.id_to_refcount[ident] -= 1
if self.id_to_refcount[ident] == 0:
del self.id_to_obj[ident], self.id_to_refcount[ident]
debug('disposing of obj with id %d', ident)
finally:
self.mutex.release()
def serveClient(self, connection):
'''
Handle requests from the proxies in a particular process/thread
'''
debug('starting server thread to service %r',
threading.currentThread().getName())
recv = connection.recv
send = connection.send
id_to_obj = self.id_to_obj
while not self.stop:
try:
methodname = obj = None
request = recv()
ident, methodname, args, kwds = request
obj, exposed = id_to_obj[ident]
if methodname not in exposed:
raise AttributeError, (
'method %r of %r object is not in exposed=%r' %
(methodname, type(obj), exposed)
)
function = getattr(obj, methodname)
try:
result = function(*args, **kwds)
msg = ('#RETURN', result)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, e:
msg = ('#ERROR', e)
except AttributeError, e:
if methodname is None:
msg = ('#ERROR', RemoteError())
else:
try:
fallback_func = self.fallback_mapping[methodname]
result = fallback_func(
self, connection, ident, obj, *args, **kwds
)
msg = ('#RETURN', result)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
msg = ('#ERROR', RemoteError())
except EOFError:
debug('got EOF -- exiting thread serving %r',
threading.currentThread().getName())
sys.exit(0)
def serveClient(self, connection):
'''
Handle requests from the proxies in a particular process/thread
'''
debug('starting server thread to service %r',
threading.currentThread().getName())
recv = connection.recv
send = connection.send
id_to_obj = self.id_to_obj
while not self.stop:
try:
methodname = obj = None
request = recv()
ident, methodname, args, kwds = request
obj, exposed = id_to_obj[ident]
if methodname not in exposed:
raise AttributeError, (
'method %r of %r object is not in exposed=%r' %
(methodname, type(obj), exposed)
)
function = getattr(obj, methodname)
try:
result = function(*args, **kwds)
msg = ('#RETURN', result)
except (SystemExit, KeyboardInterrupt):
raise
except Exception, e:
msg = ('#ERROR', e)
except AttributeError, e:
if methodname is None:
msg = ('#ERROR', RemoteError())
else:
try:
fallback_func = self.fallback_mapping[methodname]
result = fallback_func(
self, connection, ident, obj, *args, **kwds
)
msg = ('#RETURN', result)
except (SystemExit, KeyboardInterrupt):
raise
except Exception:
msg = ('#ERROR', RemoteError())
except EOFError:
debug('got EOF -- exiting thread serving %r',
threading.currentThread().getName())
sys.exit(0)
def _handleResults(outqueue, cache):
thread = threading.currentThread()
get = outqueue._reader.recv
for job, i, obj in iter(get, None):
if thread._state:
assert thread._state == TERMINATE
debug('result handler found thread._state=TERMINATE')
return
try:
cache[job]._set(i, obj)
except KeyError:
pass
else:
debug('result handler got sentinel')
while cache and thread._state != TERMINATE:
item = get()
if item is None:
debug('result handler ignoring extra sentinel')
continue
job, i, obj = item
try:
cache[job]._set(i, obj)
except KeyError:
pass
debug('result handler exiting: len(cache)=%s, thread._state=%s',
len(cache), thread._state)
def _handleResults(outqueue, cache):
thread = threading.currentThread()
get = outqueue._reader.recv
for job, i, obj in iter(get, None):
if thread._state:
assert thread._state == TERMINATE
debug('result handler found thread._state=TERMINATE')
return
try:
cache[job]._set(i, obj)
except KeyError:
pass
else:
debug('result handler got sentinel')
while cache and thread._state != TERMINATE:
item = get()
if item is None:
debug('result handler ignoring extra sentinel')
continue
job, i, obj = item
try:
cache[job]._set(i, obj)
except KeyError:
pass
debug('result handler exiting: len(cache)=%s, thread._state=%s',
len(cache), thread._state)
def _feed(buffer, notempty, send, writelock, close):
debug('starting thread to feed data to pipe')
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
bpopleft = buffer.popleft
sentinel = _sentinel
if sys.platform != 'win32':
wacquire = writelock.acquire
wrelease = writelock.release
else:
wacquire = None
try:
while 1:
nacquire()
try:
if not buffer:
nwait()
finally:
nrelease()
try:
while 1:
obj = bpopleft()
if obj is sentinel:
debug('feeder thread got sentinel -- exiting')
close()
return
if wacquire is None:
send(obj)
else:
wacquire()
try:
send(obj)
finally:
wrelease()
except IndexError:
pass
except Exception, e:
# Since this runs in a daemon thread the objects it uses
# may be become unusable while the process is cleaning up.
# We ignore errors which happen after the process has
# started to cleanup.
if currentProcess()._exiting:
subWarning('error in queue thread: %s', e)
else:
raise
def _feed(buffer, notempty, send, writelock, close):
debug('starting thread to feed data to pipe')
nacquire = notempty.acquire
nrelease = notempty.release
nwait = notempty.wait
bpopleft = buffer.popleft
sentinel = _sentinel
if sys.platform != 'win32':
wacquire = writelock.acquire
wrelease = writelock.release
else:
wacquire = None
try:
while 1:
nacquire()
try:
if not buffer:
nwait()
finally:
nrelease()
try:
while 1:
obj = bpopleft()
if obj is sentinel:
debug('feeder thread got sentinel -- exiting')
close()
return
if wacquire is None:
send(obj)
else:
wacquire()
try:
send(obj)
finally:
wrelease()
except IndexError:
pass
except Exception, e:
# Since this runs in a daemon thread the objects it uses
# may be become unusable while the process is cleaning up.
# We ignore errors which happen after the process has
# started to cleanup.
if currentProcess()._exiting:
subWarning('error in queue thread: %s', e)
else:
raise
def _decref(token, authkey, shutdown, tls, idset):
idset.remove(token.id)
# check whether manager is still alive
manager_still_alive = shutdown is None or shutdown.stillActive()
if manager_still_alive:
# tell manager this process no longer cares about referent
try:
debug('DECREF %r', token.id)
connection = Client(token.address, authkey=authkey)
dispatch(connection, None, 'decref', (token.id,))
except (SystemExit, KeyboardInterrupt):
raise
except Exception, e:
debug('... decref failed %s', e)
def _incref(self):
connection = Client(self._token.address, authkey=self._authkey)
dispatch(connection, None, 'incref', (self._id, ))
debug('INCREF %r', self._token.id)
assert self._id not in self._idset
self._idset.add(self._id)
shutdown = getattr(self._manager, 'shutdown', None)
self._close = Finalize(self,
BaseProxy._decref,
args=(self._token, self._authkey, shutdown,
self._tls, self._idset),
exitpriority=10)
def _decref(token, authkey, shutdown, tls, idset):
idset.remove(token.id)
# check whether manager is still alive
manager_still_alive = shutdown is None or shutdown.stillActive()
if manager_still_alive:
# tell manager this process no longer cares about referent
try:
debug('DECREF %r', token.id)
connection = Client(token.address, authkey=authkey)
dispatch(connection, None, 'decref', (token.id, ))
except (SystemExit, KeyboardInterrupt):
raise
except Exception, e:
debug('... decref failed %s', e)
def _incref(self):
connection = Client(self._token.address, authkey=self._authkey)
dispatch(connection, None, 'incref', (self._id,))
debug('INCREF %r', self._token.id)
assert self._id not in self._idset
self._idset.add(self._id)
shutdown = getattr(self._manager, 'shutdown', None)
self._close = Finalize(
self, BaseProxy._decref,
args=(self._token, self._authkey, shutdown,
self._tls, self._idset),
exitpriority=10
)
def _getListener():
global _listener
if _listener is None:
_lock.acquire()
try:
if _listener is None:
from processing.connection import Listener
debug('starting listener and thread for sending handles')
_listener = Listener(authenticate=True)
t = threading.Thread(target=_serve)
t.setDaemon(True)
t.start()
finally:
_lock.release()
return _listener
def _getListener():
global _listener
if _listener is None:
_lock.acquire()
try:
if _listener is None:
from processing.connection import Listener
debug('starting listener and thread for sending handles')
_listener = Listener(authenticate=True)
t = threading.Thread(target=_serve)
t.setDaemon(True)
t.start()
finally:
_lock.release()
return _listener
def shutdown(self, c):
'''
Shutdown this process
'''
c.send(('#RETURN', None))
info('manager received shutdown message')
# do some cleaning up
_runFinalizers(0)
for p in activeChildren():
debug('terminating a child process of manager')
p.terminate()
for p in activeChildren():
debug('terminating a child process of manager')
p.join()
_runFinalizers()
info('manager exiting with exitcode 0')
# now exit without waiting for other threads to finish
exit(0)
def shutdown(self, c):
'''
Shutdown this process
'''
c.send(('#RETURN', None))
info('manager received shutdown message')
# do some cleaning up
_runFinalizers(0)
for p in activeChildren():
debug('terminating a child process of manager')
p.terminate()
for p in activeChildren():
debug('terminating a child process of manager')
p.join()
_runFinalizers()
info('manager exiting with exitcode 0')
# now exit without waiting for other threads to finish
exit(0)
def _callMethod(self, methodname, args=(), kwds={}):
'''
Try to call a method of the referrent and return a copy of the result
'''
try:
conn = self._tls.connection
except AttributeError:
debug('thread %r does not own a connection',
threading.currentThread().getName())
self._connect()
conn = self._tls.connection
conn.send((self._id, methodname, args, kwds))
kind, result = conn.recv()
if kind == '#RETURN':
return result
elif kind == '#ERROR':
raise result
else:
raise ValueError
def _callMethod(self, methodname, args=(), kwds={}):
'''
Try to call a method of the referrent and return a copy of the result
'''
try:
conn = self._tls.connection
except AttributeError:
debug('thread %r does not own a connection',
threading.currentThread().getName())
self._connect()
conn = self._tls.connection
conn.send((self._id, methodname, args, kwds))
kind, result = conn.recv()
if kind == '#RETURN':
return result
elif kind == '#ERROR':
raise result
else:
raise ValueError
def _startThread(self):
debug('Queue._startThread()')
# Start thread which transfers data from buffer to pipe
self._buffer.clear()
self._thread = threading.Thread(target=Queue._feed,
args=(self._buffer, self._notempty,
self._send, self._wlock,
self._writer.close),
name='QueueFeederThread')
self._thread.setDaemon(True)
debug('doing self._thread.start()')
self._thread.start()
debug('... done self._thread.start()')
# On process exit we will wait for data to be flushed to pipe.
#
# However, if this process created the queue then all
# processes which use the queue will be descendants of this
# process. Therefore waiting for the queue to be flushed
# is pointless once all the child processes have been joined.
created_by_this_process = (self._opid == os.getpid())
if not self._joincancelled and not created_by_this_process:
self._jointhread = Finalize(self._thread,
Queue._finalizeJoin,
[weakref.ref(self._thread)],
exitpriority=-5)
# Send sentinel to the thread queue object when garbage collected
self._close = Finalize(self,
Queue._finalizeClose,
[self._buffer, self._notempty],
exitpriority=10)
def create(self, c, typeid, *args, **kwds):
'''
Create a new shared object and return its id
'''
self.mutex.acquire()
try:
callable, exposed = self.registry[typeid]
obj = callable(*args, **kwds)
if exposed is None:
exposed = publicMethods(obj)
ident = id(obj)
debug('have created %r object with id %r', typeid, ident)
self.id_to_obj[ident] = (obj, set(exposed))
if ident not in self.id_to_refcount:
self.id_to_refcount[ident] = None
return ident, tuple(exposed)
finally:
self.mutex.release()
def create(self, c, typeid, *args, **kwds):
'''
Create a new shared object and return its id
'''
self.mutex.acquire()
try:
callable, exposed = self.registry[typeid]
obj = callable(*args, **kwds)
if exposed is None:
exposed = publicMethods(obj)
ident = id(obj)
debug('have created %r object with id %r', typeid, ident)
self.id_to_obj[ident] = (obj, set(exposed))
if ident not in self.id_to_refcount:
self.id_to_refcount[ident] = None
return ident, tuple(exposed)
finally:
self.mutex.release()
def _finalizeJoin(twr):
debug('joining queue thread')
thread = twr()
if thread is not None:
thread.join()
debug('... queue thread joined')
else:
debug('... queue thread already dead')
def _finalizeJoin(twr):
debug('joining queue thread')
thread = twr()
if thread is not None:
thread.join()
debug('... queue thread joined')
else:
debug('... queue thread already dead')
def _startThread(self):
debug('Queue._startThread()')
# Start thread which transfers data from buffer to pipe
self._buffer.clear()
self._thread = threading.Thread(
target=Queue._feed,
args=(self._buffer, self._notempty, self._send,
self._wlock, self._writer.close),
name='QueueFeederThread'
)
self._thread.setDaemon(True)
debug('doing self._thread.start()')
self._thread.start()
debug('... done self._thread.start()')
# On process exit we will wait for data to be flushed to pipe.
#
# However, if this process created the queue then all
# processes which use the queue will be descendants of this
# process. Therefore waiting for the queue to be flushed
# is pointless once all the child processes have been joined.
created_by_this_process = (self._opid == os.getpid())
if not self._joincancelled and not created_by_this_process:
self._jointhread = Finalize(
self._thread, Queue._finalizeJoin,
[weakref.ref(self._thread)],
exitpriority=-5
)
# Send sentinel to the thread queue object when garbage collected
self._close = Finalize(
self, Queue._finalizeClose,
[self._buffer, self._notempty],
exitpriority=10
)
# check whether manager is still alive
manager_still_alive = shutdown is None or shutdown.stillActive()
if manager_still_alive:
# tell manager this process no longer cares about referent
try:
debug('DECREF %r', token.id)
connection = Client(token.address, authkey=authkey)
dispatch(connection, None, 'decref', (token.id,))
except (SystemExit, KeyboardInterrupt):
raise
except Exception, e:
debug('... decref failed %s', e)
else:
debug('DECREF %r -- manager already shutdown',
token.id)
# check whether we can close this thread's connection because
# the process owns no more references to objects for this manager
if not idset and hasattr(tls, 'connection'):
debug('thread %r has no more proxies so closing conn',
threading.currentThread().getName())
tls.connection.close()
del tls.connection
def _afterFork(self):
self._manager = None
self._incref()
def __reduce__(self):
if hasattr(self, '_exposed'):
def joinThread(self):
debug('Queue.joinThread()')
assert self._closed
if self._jointhread:
self._jointhread()
def terminate(self):
debug('terminating pool')
self._state = TERMINATE
self._terminate()
def close(self):
debug('closing pool')
self._state = CLOSE
self._taskqueue.put(None)
def terminate(self):
debug('terminating pool')
self._state = TERMINATE
self._terminate()
def _terminatePool(taskqueue, inqueue, outqueue, cache, pool, task_handler,
result_handler):
debug('finalizing pool')
if not result_handler.isAlive():
debug('result handler already finished -- no need to terminate')
return
cache = {}
task_handler._state = TERMINATE
result_handler._state = TERMINATE
debug('sending sentinels')
taskqueue.put(None)
outqueue.put(None)
debug('getting read lock on inqueue')
inqueue._rlock.acquire()
debug('terminating workers')
for p in pool:
p.terminate()
if task_handler.isAlive():
debug('removing tasks from inqueue until task handler finished')
while task_handler.isAlive() and inqueue._reader.poll():
inqueue._reader.recv()
time.sleep(0)
debug('joining result handler')
result_handler.join()
debug('joining task handler')
task_handler.join()
debug('joining pool workers')
for p in pool:
p.join()
debug('closing connections')
inqueue._reader.close()
outqueue._reader.close()
inqueue._writer.close()
outqueue._writer.close()
#
def worker(inqueue, outqueue, initializer=None, initargs=()):
put = outqueue.put
if initializer is not None:
initializer(*initargs)
for job, i, func, args, kwds in iter(inqueue.get, None):
try:
result = (True, func(*args, **kwds))
except Exception, e:
result = (False, e)
put((job, i, result))
debug('worker got sentinel -- exiting')
#
# Class representing a process pool
#
class Pool(object):
'''
Class which supports an async version of the `apply()` builtin
'''
def __init__(self, processes=None, initializer=None, initargs=()):
self._inqueue = SimpleQueue()
self._outqueue = SimpleQueue()
self._taskqueue = Queue.Queue()
self._cache = {}
self._state = RUN
# check whether manager is still alive
manager_still_alive = shutdown is None or shutdown.stillActive()
if manager_still_alive:
# tell manager this process no longer cares about referent
try:
debug('DECREF %r', token.id)
connection = Client(token.address, authkey=authkey)
dispatch(connection, None, 'decref', (token.id, ))
except (SystemExit, KeyboardInterrupt):
raise
except Exception, e:
debug('... decref failed %s', e)
else:
debug('DECREF %r -- manager already shutdown', token.id)
# check whether we can close this thread's connection because
# the process owns no more references to objects for this manager
if not idset and hasattr(tls, 'connection'):
debug('thread %r has no more proxies so closing conn',
threading.currentThread().getName())
tls.connection.close()
del tls.connection
def _afterFork(self):
self._manager = None
self._incref()
def __reduce__(self):
if hasattr(self, '_exposed'):
def close(self):
debug('closing pool')
self._state = CLOSE
self._taskqueue.put(None)
def worker(inqueue, outqueue, initializer=None, initargs=()):
put = outqueue.put
if initializer is not None:
initializer(*initargs)
for job, i, func, args, kwds in iter(inqueue.get, None):
try:
result = (True, func(*args, **kwds))
except Exception, e:
result = (False, e)
put((job, i, result))
debug('worker got sentinel -- exiting')
#
# Class representing a process pool
#
class Pool(object):
'''
Class which supports an async version of the `apply()` builtin
'''
def __init__(self, processes=None, initializer=None, initargs=()):
self._inqueue = SimpleQueue()
self._outqueue = SimpleQueue()
self._taskqueue = Queue.Queue()
def _terminatePool(taskqueue, inqueue, outqueue, cache, pool,
task_handler, result_handler):
debug('finalizing pool')
if not result_handler.isAlive():
debug('result handler already finished -- no need to terminate')
return
cache = {}
task_handler._state = TERMINATE
result_handler._state = TERMINATE
debug('sending sentinels')
taskqueue.put(None)
outqueue.put(None)
debug('getting read lock on inqueue')
inqueue._rlock.acquire()
debug('terminating workers')
for p in pool:
p.terminate()
if task_handler.isAlive():
debug('removing tasks from inqueue until task handler finished')
while task_handler.isAlive() and inqueue._reader.poll():
inqueue._reader.recv()
time.sleep(0)
debug('joining result handler')
result_handler.join()
debug('joining task handler')
task_handler.join()
debug('joining pool workers')
for p in pool:
p.join()
debug('closing connections')
inqueue._reader.close()
outqueue._reader.close()
inqueue._writer.close()
outqueue._writer.close()
def joinThread(self):
debug('Queue.joinThread()')
assert self._closed
if self._jointhread:
self._jointhread()
