def _start_thread(self):
util.debug('Queue._start_thread()')
# Start thread which transfers data from buffer to pipe
self._buffer.clear()
self._thread = threading.Thread(
target=Queue._feed,
args=(self._buffer, self._notempty, self._send_bytes, self._wlock,
self._writer.close, self._reducers, self._ignore_epipe,
self._on_queue_feeder_error, self._sem),
name='QueueFeederThread')
self._thread.daemon = True
util.debug('doing self._thread.start()')
self._thread.start()
util.debug('... done self._thread.start()')
# On process exit we will wait for data to be flushed to pipe.
#
# However, if this process created the queue then all
# processes which use the queue will be descendants of this
# process. Therefore waiting for the queue to be flushed
# is pointless once all the child processes have been joined.
created_by_this_process = (self._opid == os.getpid())
if not self._joincancelled and not created_by_this_process:
self._jointhread = util.Finalize(self._thread,
Queue._finalize_join,
[weakref.ref(self._thread)],
exitpriority=-5)
# Send sentinel to the thread queue object when garbage collected
self._close = util.Finalize(self,
Queue._finalize_close,
[self._buffer, self._notempty],
exitpriority=10)
def __init__(self, kind, value, maxvalue):
name = 'loky'
unlink_now = sys.platform == 'win32' or name == 'fork'
for i in range(100):
try:
self._semlock = SemLockC(kind, value, maxvalue,
SemLock._make_name(), unlink_now)
except FileExistsError:
pass
else:
break
else:
raise FileExistsError('cannot find name for semaphore')
self._make_methods()
if sys.platform != 'win32':
def _after_fork(obj):
obj._semlock._after_fork()
util.register_after_fork(self, _after_fork)
if self._semlock.name is not None:
# We only get here if we are on Unix with forking
# disabled. When the object is garbage collected or the
# process shuts down we unlink the semaphore name
from .semaphore_tracker import register
if sys.version_info < (3, 4):
register(self._semlock.name)
util.Finalize(self,
SemLock._cleanup, (self._semlock.name, ),
exitpriority=0)
def from_address(cls, address, authkey):
"""
Return manager given an address.
address: (ip_addr, port) or string referring to pipe.
Address to connect to.
authkey: string
Authorization key.
"""
manager = cls(address, authkey)
conn = connection.Client(address, authkey=authkey)
try:
managers.dispatch(conn, None, 'dummy')
finally:
conn.close()
manager._state.value = managers.State.STARTED
manager._name = 'Host-%s:%s' % manager.address
manager.shutdown = util.Finalize(manager,
HostManager._finalize_host,
args=(manager._address,
manager._authkey,
manager._name),
exitpriority=-10)
return manager
def main():
# get data from parent over stdin
data = pickle.load(sys.stdin)
sys.stdin.close()
# set some stuff
_logger.setLevel(data['dist_log_level'])
forking.prepare(data)
# create server for a `HostManager` object
server = managers.Server(HostManager._registry, ('', 0), data['authkey'])
current_process()._server = server
# report server address and number of cpus back to parent
conn = connection.Client(data['parent_address'], authkey=data['authkey'])
conn.send((data['index'], server.address, slot_count))
conn.close()
# set name etc
current_process().set_name('Host-%s:%s' % server.address)
util._run_after_forkers()
# register a cleanup function
def cleanup(directory):
debug('removing directory %s', directory)
shutil.rmtree(directory)
debug('shutting down host manager')
util.Finalize(None, cleanup, args=[data['dir']], exitpriority=0)
# start host manager
debug('remote host manager starting in %s', data['dir'])
server.serve_forever()
def start(self, initializer=None, initargs=()):
"""
Spawn a server process for this manager object
"""
if initializer is not None and not hasattr(initializer, '__call__'):
raise TypeError('initializer must be a callable')
reader, writer = connection.Pipe(duplex=False)
self._process = Process(target=type(self)._run_server,
args=(self._registry, self._address,
self._authkey, self._serializer, writer,
initializer, initargs))
ident = ':'.join((str(i) for i in self._process._identity))
self._process.name = type(self).__name__ + '-' + ident
self._process.start()
writer.close()
self._address = reader.recv()
reader.close()
self._state.value = State.STARTED
self.shutdown = util.Finalize(self,
type(self)._finalize_manager,
args=(self._process, self._address,
self._authkey, self._state,
self._Client),
exitpriority=0)
return
def __init__(self, kind, value, maxvalue):
# unlink_now is only used on win32 or when we are using fork.
unlink_now = False
for i in range(100):
try:
self._semlock = _SemLock(
kind, value, maxvalue, SemLock._make_name(),
unlink_now)
except FileExistsError: # pragma: no cover
pass
else:
break
else: # pragma: no cover
raise FileExistsError('cannot find name for semaphore')
util.debug('created semlock with handle %s and name "%s"'
% (self._semlock.handle, self._semlock.name))
self._make_methods()
def _after_fork(obj):
obj._semlock._after_fork()
util.register_after_fork(self, _after_fork)
# When the object is garbage collected or the
# process shuts down we unlink the semaphore name
resource_tracker.register(self._semlock.name, "semlock")
util.Finalize(self, SemLock._cleanup, (self._semlock.name,),
exitpriority=0)
def __init__(self, process_obj):
cmd = ' '.join('"%s"' % x for x in get_command_line())
prep_data = get_preparation_data(process_obj._name)
# create pipe for communication with child
rfd, wfd = os.pipe()
# get handle for read end of the pipe and make it inheritable
rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
os.close(rfd)
with open(wfd, 'wb', closefd=True) as to_child:
# start process
try:
hp, ht, pid, tid = _winapi.CreateProcess(
_python_exe, cmd + (' %s' % rhandle), None, None, 1, 0,
None, None, None)
_winapi.CloseHandle(ht)
finally:
close(rhandle)
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel, ))
# send information to child
Popen._tls.process_handle = int(hp)
try:
dump(prep_data, to_child, HIGHEST_PROTOCOL)
dump(process_obj, to_child, HIGHEST_PROTOCOL)
finally:
del Popen._tls.process_handle
def start(self, initializer=None, initargs=()):
"""Spawn a server process for this manager object."""
assert self._state.value == State.INITIAL
logger.debug("start manager %s", self)
if initializer is not None and not callable(initializer):
raise TypeError('initializer must be a callable')
# pipe over which we will retrieve address of server
reader, writer = fiber.queues.Pipe(duplex=False)
# spawn process which runs a server
self._process = fiber.Process(
target=type(self)._run_server,
args=(self._registry, self._address, self._authkey,
self._serializer, writer, initializer, initargs),
)
ident = ':'.join(str(i) for i in self._process._identity)
self._process.name = type(self).__name__ + '-' + ident
self._process.start()
# get address of server
writer.close()
self._address = reader.recv()
reader.close()
# register a finalizer
self._state.value = State.STARTED
self.shutdown = mp_util.Finalize(self,
type(self)._finalize_manager,
args=(self._process, self._address,
self._authkey, self._state,
self._Client),
exitpriority=0)
def start(self):
'''
Spawn a server process for this manager object
'''
assert self._state.value == State.INITIAL
# pipe over which we will retrieve address of server
reader, writer = connection.Pipe(duplex=False)
# spawn process which runs a server
self._process = Process(
target=type(self)._run_server,
args=(self._registry, self._address, self._authkey,
self._serializer, writer),
)
ident = ':'.join(str(i) for i in self._process._identity)
self._process.name = type(self).__name__ + '-' + ident
self._process.start()
# get address of server
writer.close()
self._address = reader.recv()
reader.close()
# register a finalizer
self._state.value = State.STARTED
self.shutdown = util.Finalize(
self, type(self)._finalize_manager,
args=(self._process, self._address, self._authkey,
self._state, self._Client),
exitpriority=0
)
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(
process_obj._name, getattr(process_obj, "init_main_module", True))
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rfd, wfd = os.pipe()
rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
os.close(rfd)
cmd = get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
try:
with open(wfd, 'wb') as to_child:
# start process
try:
# This flag allows to pass inheritable handles from the
# parent to the child process in a python2-3 compatible way
# (see
# https://github.com/tomMoral/loky/pull/204#discussion_r290719629
# for more detail). When support for Python 2 is dropped,
# the cleaner multiprocessing.reduction.steal_handle should
# be used instead.
inherit = True
hp, ht, pid, tid = _winapi.CreateProcess(
spawn.get_executable(), cmd,
None, None, inherit, 0,
None, None, None)
_winapi.CloseHandle(ht)
except BaseException as e:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
# send information to child
set_spawning_popen(self)
if sys.version_info[:2] < (3, 4):
Popen._tls.process_handle = int(hp)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
if sys.version_info[:2] < (3, 4):
del Popen._tls.process_handle
except IOError as exc:
# IOError 22 happens when the launched subprocess terminated before
# wfd.close is called. Thus we can safely ignore it.
if exc.errno != 22:
raise
util.debug("While starting {}, ignored a IOError 22"
.format(process_obj._name))
def __init__(self, size): # 对应c语言类型需要占用的字节数
if size < 0:
raise ValueError("Size {0:n} out of range".format(size))
if sys.maxsize <= size:
raise OverflowError("Size {0:n} too large".format(size))
block = BufferWrapper._heap.malloc(size)
self._state = (block, size)
util.Finalize(self, BufferWrapper._heap.free, args=(block, ))
def _incref(self):
conn = self._Client(self._token.address, authkey=self._authkey)
dispatch(conn, None, 'incref', (self._id,))
util.debug('INCREF %r', self._token.id)
self._idset.add(self._id)
state = self._manager and self._manager._state
self._close = util.Finalize(self, BaseProxy._decref, args=(
self._token, self._authkey, state,
self._tls, self._idset, self._Client), exitpriority=10)
return
def _launch(self, process_obj):
tracker_fd = resource_tracker._resource_tracker.getfd()
fp = BytesIO()
set_spawning_popen(self)
try:
prep_data = spawn.get_preparation_data(
process_obj._name,
getattr(process_obj, "init_main_module", True))
reduction.dump(prep_data, fp)
reduction.dump(process_obj, fp)
finally:
set_spawning_popen(None)
try:
parent_r, child_w = os.pipe()
child_r, parent_w = os.pipe()
# for fd in self._fds:
# _mk_inheritable(fd)
cmd_python = [sys.executable]
cmd_python += ['-m', self.__module__]
cmd_python += ['--process-name', str(process_obj.name)]
cmd_python += [
'--pipe',
str(reduction._mk_inheritable(child_r))
]
reduction._mk_inheritable(child_w)
reduction._mk_inheritable(tracker_fd)
self._fds.extend([child_r, child_w, tracker_fd])
if sys.version_info >= (3, 8) and os.name == 'posix':
mp_tracker_fd = prep_data['mp_tracker_args']['fd']
self.duplicate_for_child(mp_tracker_fd)
from .fork_exec import fork_exec
pid = fork_exec(cmd_python, self._fds, env=process_obj.env)
util.debug("launched python with pid {} and cmd:\n{}".format(
pid, cmd_python))
self.sentinel = parent_r
method = 'getbuffer'
if not hasattr(fp, method):
method = 'getvalue'
with os.fdopen(parent_w, 'wb') as f:
f.write(getattr(fp, method)())
self.pid = pid
finally:
if parent_r is not None:
util.Finalize(self, os.close, (parent_r, ))
for fd in (child_r, child_w):
if fd is not None:
os.close(fd)
def from_address(cls, address, authkey):
manager = cls(address, authkey)
managers.transact(address, authkey, 'dummy')
manager._state.value = managers.State.STARTED
manager._name = 'Host-%s:%s' % manager.address
manager.shutdown = util.Finalize(
manager, HostManager._finalize_host,
args=(manager._address, manager._authkey, manager._name),
exitpriority=-10
)
return manager
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(
process_obj._name, process_obj.init_main_module)
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rhandle, wfd = _winapi.CreatePipe(None, 0)
if sys.version_info[:2] > (3, 3):
wfd = msvcrt.open_osfhandle(wfd, 0)
cmd = spawn.get_command_line(parent_pid=os.getpid(),
pipe_handle=rhandle)
cmd = ' '.join('"%s"' % x for x in cmd)
try:
with open(wfd, 'wb') as to_child:
# start process
try:
inherit = sys.version_info[:2] < (3, 4)
hp, ht, pid, tid = _winapi.CreateProcess(
spawn.get_executable(), cmd,
None, None, inherit, 0,
None, None, None)
_winapi.CloseHandle(ht)
except:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel,))
# send information to child
set_spawning_popen(self)
if sys.version_info[:2] < (3, 4):
Popen._tls.process_handle = int(hp)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
if sys.version_info[:2] < (3, 4):
del Popen._tls.process_handle
except IOError as exc:
# IOError 22 happens when the launched subprocess terminated before
# wfd.close is called. Thus we can safely ignore it.
if exc.errno != 22:
raise
util.debug("While starting {}, ignored a IOError 22"
.format(process_obj._name))
def _launch(self, process_obj):
tracker_fd = semaphore_tracker._semaphore_tracker.getfd()
fp = BytesIO()
set_spawning_popen(self)
try:
prep_data = spawn.get_preparation_data(
process_obj._name, process_obj.init_main_module)
reduction.dump(prep_data, fp)
reduction.dump(process_obj, fp)
finally:
set_spawning_popen(None)
try:
parent_r, child_w = os.pipe()
child_r, parent_w = os.pipe()
# for fd in self._fds:
# _mk_inheritable(fd)
cmd_python = [sys.executable]
cmd_python += ['-m', self.__module__]
cmd_python += ['--name-process', str(process_obj.name)]
cmd_python += [
'--pipe',
str(reduction._mk_inheritable(child_r))
]
reduction._mk_inheritable(child_w)
if tracker_fd is not None:
cmd_python += [
'--semaphore',
str(reduction._mk_inheritable(tracker_fd))
]
self._fds.extend([child_r, child_w, tracker_fd])
util.debug("launch python with cmd:\n%s" % cmd_python)
from .fork_exec import fork_exec
pid = fork_exec(cmd_python, self._fds)
self.sentinel = parent_r
method = 'getbuffer'
if not hasattr(fp, method):
method = 'getvalue'
with os.fdopen(parent_w, 'wb') as f:
f.write(getattr(fp, method)())
self.pid = pid
finally:
if parent_r is not None:
util.Finalize(self, os.close, (parent_r, ))
for fd in (child_r, child_w):
if fd is not None:
os.close(fd)
def get_reusable_workers(n_jobs=4, hostfile=None):
global _worker_comm, _n_workers
if _worker_comm is None:
_n_workers = n_jobs
_worker_comm = _spawn_workers(n_jobs, hostfile)
util.Finalize(None, shutdown_reusable_workers, exitpriority=20)
else:
if _n_workers != n_jobs:
warnings.warn("You should not require different size")
shutdown_reusable_workers()
return get_reusable_workers(n_jobs=n_jobs, hostfile=hostfile)
return _worker_comm
def __init__(self, size, fd=-1):
self.size = size
self.fd = fd
if fd == -1:
'''
User-callable function to create and return a unique temporary file.
The return value is a pair (fd, name) where fd is the file descriptor returned by os.open, and name is the filename.
If 'prefix' is not None, the file name will begin with that prefix,otherwise a default prefix is used.
If 'dir' is not None, the file will be created in that directory,otherwise a default directory is used.
If 'text' is specified and true, the file is opened in text mode. Else (the default) the file is opened in binary mode. On some operating systems, this makes no difference.
The file is readable and writable only by the creating user ID.The file descriptor is not inherited by children of this process.
'''
self.fd, name = tempfile.mkstemp(prefix=f'pym-{os.getpid()}-')
os.unlink(name) # 删除临时文件
util.Finalize(
self, os.close,
(self.fd, )) # os.close: Close a file descriptor,这里没看懂
os.ftruncate(
self.fd, size
) # Truncate a file, specified by file descriptor, to a specific length.
self.buffer = mmap.mmap(self.fd, self.size)
def __init__(self, process_obj):
sys.stdout.flush()
sys.stderr.flush()
self.returncode = None
r, w = os.pipe()
self.sentinel = r
self.pid = os.fork()
if self.pid == 0:
os.close(r)
if 'random' in sys.modules:
import random
random.seed()
code = process_obj._bootstrap()
os._exit(code)
# `w` will be closed when the child exits, at which point `r`
# will become ready for reading (using e.g. select()).
os.close(w)
util.Finalize(self, os.close, (r, ))
def from_address(cls, address, authkey, host):
"""
Return manager given an address.
address: (ip_addr, port) or string referring to pipe.
Address to connect to.
authkey: string
Authorization key.
host: :class:`Host`
Host we're managing.
"""
if host.tunnel_outgoing:
_LOGGER.debug('Client setting up tunnel for %s:%s', host.hostname,
address[1])
address, cleanup = setup_tunnel(host.hostname,
address[1],
identity=host.identity_filename)
else:
cleanup = None
manager = cls(address, authkey)
_LOGGER.debug('Client connecting to server at %s' % (address, ))
conn = connection.Client(address, authkey=authkey)
try:
managers.dispatch(conn, None, 'dummy')
finally:
conn.close()
manager._state.value = managers.State.STARTED
manager._name = 'Host-%s:%s' % manager.address
manager.shutdown = util.Finalize(manager,
HostManager._finalize_host,
args=(manager._address,
manager._authkey, cleanup,
host.reverse_cleanup),
exitpriority=-10)
return manager
def __init__(self, process_obj):
prep_data = spawn.get_preparation_data(
process_obj._name, getattr(process_obj, "init_main_module", True))
# read end of pipe will be "stolen" by the child process
# -- see spawn_main() in spawn.py.
rfd, wfd = os.pipe()
rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
os.close(rfd)
cmd = get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle)
cmd = ' '.join(f'"{x}"' for x in cmd)
python_exe = spawn.get_executable()
# copy the environment variables to set in the child process
child_env = os.environ.copy()
child_env.update(process_obj.env)
# bpo-35797: When running in a venv, we bypass the redirect
# executor and launch our base Python.
if WINENV and _path_eq(python_exe, sys.executable):
python_exe = sys._base_executable
child_env["__PYVENV_LAUNCHER__"] = sys.executable
try:
with open(wfd, 'wb') as to_child:
# start process
try:
# This flag allows to pass inheritable handles from the
# parent to the child process in a python2-3 compatible way
# (see
# https://github.com/tomMoral/loky/pull/204#discussion_r290719629
# for more detail). When support for Python 2 is dropped,
# the cleaner multiprocessing.reduction.steal_handle should
# be used instead.
inherit = True
hp, ht, pid, tid = _winapi.CreateProcess(
python_exe, cmd, None, None, inherit, 0, child_env,
None, None)
_winapi.CloseHandle(ht)
except BaseException:
_winapi.CloseHandle(rhandle)
raise
# set attributes of self
self.pid = pid
self.returncode = None
self._handle = hp
self.sentinel = int(hp)
util.Finalize(self, _winapi.CloseHandle, (self.sentinel, ))
# send information to child
set_spawning_popen(self)
try:
reduction.dump(prep_data, to_child)
reduction.dump(process_obj, to_child)
finally:
set_spawning_popen(None)
except IOError as exc:
# IOError 22 happens when the launched subprocess terminated before
# wfd.close is called. Thus we can safely ignore it.
if exc.errno != 22:
raise
util.debug(
f"While starting {process_obj._name}, ignored a IOError 22")
def main(): #pragma no cover
"""
Code which runs a host manager.
Expects configuration data from parent on `stdin`.
Replies with address and optionally public key.
The environment variable ``OPENMDAO_KEEPDIRS`` can be used to avoid
removal of the temporary directory used here.
"""
sys.stdout = open('stdout', 'w')
sys.stderr = open('stderr', 'w')
# util.log_to_stderr(logging.DEBUG)
# Avoid root possibly masking us.
logging.getLogger().setLevel(logging.DEBUG)
import platform
hostname = platform.node()
pid = os.getpid()
ident = '(%s:%d)' % (hostname, pid)
print '%s main startup' % ident
sys.stdout.flush()
# Get data from parent over stdin.
data = cPickle.load(sys.stdin)
sys.stdin.close()
print '%s data received' % ident
authkey = data['authkey']
allow_shell = data['allow_shell']
allowed_users = data['allowed_users']
print '%s using %s authentication' % (ident, keytype(authkey))
if allowed_users is None:
print '%s allowed_users: ANY' % ident
else:
print '%s allowed_users: %s' % (ident, sorted(allowed_users.keys()))
if allow_shell:
print '%s ALLOWING SHELL ACCESS' % ident
sys.stdout.flush()
log_level = data['dist_log_level']
os.environ['OPENMDAO_KEEPDIRS'] = data['keep_dirs']
exc = None
server = None
try:
# Update HostManager registry.
dct = data['registry']
print '%s registry:' % ident
for name in dct.keys():
module = dct[name]
print ' %s: %s' % (name, module)
mod = __import__(module, fromlist=name)
cls = getattr(mod, name)
register(cls, HostManager)
# Set some stuff.
print '%s preparing to fork, log level %d' % (ident, log_level)
sys.stdout.flush()
util.get_logger().setLevel(log_level)
forking.prepare(data)
# Create Server for a HostManager object.
name = '%d[%d]' % (data['index'], pid)
logging.getLogger(name).setLevel(log_level)
server = OpenMDAO_Server(HostManager._registry, (hostname, 0),
authkey,
'pickle',
name=name,
allowed_users=allowed_users,
allowed_hosts=[data['parent_address'][0]])
except Exception as exc:
print '%s caught exception: %s' % (ident, exc)
# Report server address and public key back to parent.
print '%s connecting to parent at %s' % (ident, data['parent_address'])
sys.stdout.flush()
conn = connection.Client(data['parent_address'], authkey=authkey)
if exc:
conn.send((data['index'], None, str(exc)))
else:
conn.send((data['index'], server.address, server.public_key_text))
conn.close()
if exc:
print '%s exiting' % ident
sys.exit(1)
# Set name etc.
current_process()._server = server
current_process()._name = 'Host-%s:%s' % server.address
current_process().authkey = authkey
logging.getLogger(current_process()._name).setLevel(log_level)
util._run_after_forkers()
# Register a cleanup function.
def cleanup(directory):
keep_dirs = int(os.environ.get('OPENMDAO_KEEPDIRS', '0'))
if not keep_dirs and os.path.exists(directory):
print '%s removing directory %s' % (ident, directory)
shutil.rmtree(directory)
print '%s shutting down host manager' % ident
util.Finalize(None, cleanup, args=[data['dir']], exitpriority=0)
# Start host manager.
print '%s remote host manager starting in %s' % (ident, data['dir'])
sys.stdout.flush()
server.serve_forever()
def __init__(self,
processes=None,
initializer=None,
initargs=(),
maxtasksperchild=None,
context=None):
# Attributes initialized early to make sure that they exist in
# __del__() if __init__() raises an exception
self._pool = []
self._state = INIT
class LifoQueueManager(BaseManager):
pass
LifoQueueManager.register('LifoQueue', queue.LifoQueue)
self._manager = LifoQueueManager()
self._manager.start()
self._ctx = context or get_context()
self._setup_queues()
self._taskqueue = queue.LifoQueue()
# The _change_notifier queue exist to wake up self._handle_workers()
# when the cache (self._cache) is empty or when there is a change in
# the _state variable of the thread that runs _handle_workers.
self._change_notifier = self._ctx.SimpleQueue()
self._cache = _PoolCache(notifier=self._change_notifier)
self._maxtasksperchild = maxtasksperchild
self._initializer = initializer
self._initargs = initargs
if processes is None:
processes = os.cpu_count() or 1
if processes < 1:
raise ValueError("Number of processes must be at least 1")
if initializer is not None and not callable(initializer):
raise TypeError('initializer must be a callable')
self._processes = processes
try:
self._repopulate_pool()
except Exception:
for p in self._pool:
if p.exitcode is None:
p.terminate()
for p in self._pool:
p.join()
raise
sentinels = self._get_sentinels()
self._worker_handler = threading.Thread(
target=Pool._handle_workers,
args=(self._cache, self._taskqueue, self._ctx, self.Process,
self._processes, self._pool, self._inqueue, self._outqueue,
self._initializer, self._initargs, self._maxtasksperchild,
self._wrap_exception, sentinels, self._change_notifier,
self))
self._worker_handler.daemon = True
self._worker_handler._state = RUN
self._worker_handler.start()
self._task_handler = threading.Thread(
target=Pool._handle_tasks,
args=(self._taskqueue, self._quick_put, self._outqueue, self._pool,
self._cache))
self._task_handler.daemon = True
self._task_handler._state = RUN
self._task_handler.start()
self._result_handler = threading.Thread(target=Pool._handle_results,
args=(self._outqueue,
self._quick_get,
self._cache))
self._result_handler.daemon = True
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = util.Finalize(
self,
self._terminate_pool,
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
self._change_notifier, self._worker_handler,
self._task_handler, self._result_handler, self._cache),
exitpriority=15)
self._state = RUN
def finalize(obj, func, *args, **kwargs):
return mp_util.Finalize(obj,
func,
args=args,
kwargs=kwargs,
exitpriority=0)