def __init__(self, setup):
BuiltinCore.__init__(self, setup)
if setup['children'] is None:
setup['children'] = multiprocessing.cpu_count()
#: A dict of child name -> one end of the
#: :class:`multiprocessing.Pipe` object used to communicate
#: with that child. (The child is given the other end of the
#: Pipe.)
self.pipes = dict()
#: A queue that keeps track of which children are available to
#: render a configuration. A child is popped from the queue
#: when it starts to render a config, then it's pushed back on
#: when it's done. This lets us use a blocking call to
#: :func:`Queue.Queue.get` when waiting for an available
#: child.
self.available_children = Queue(maxsize=self.setup['children'])
# sigh. multiprocessing was added in py2.6, which is when the
# camelCase methods for threading objects were deprecated in
# favor of the Pythonic under_score methods. So
# multiprocessing.Event *only* has is_set(), while
# threading.Event has *both* isSet() and is_set(). In order
# to make the core work with Python 2.4+, and with both
# multiprocessing and threading Event objects, we just
# monkeypatch self.terminate to have isSet().
self.terminate = DualEvent(threading_event=self.terminate)
def __init__(self, setup):
BuiltinCore.__init__(self, setup)
if setup['children'] is None:
setup['children'] = multiprocessing.cpu_count()
#: A dict of child name -> one end of the
#: :class:`multiprocessing.Pipe` object used to communicate
#: with that child. (The child is given the other end of the
#: Pipe.)
self.pipes = dict()
#: A queue that keeps track of which children are available to
#: render a configuration. A child is popped from the queue
#: when it starts to render a config, then it's pushed back on
#: when it's done. This lets us use a blocking call to
#: :func:`Queue.Queue.get` when waiting for an available
#: child.
self.available_children = Queue(maxsize=self.setup['children'])
# sigh. multiprocessing was added in py2.6, which is when the
# camelCase methods for threading objects were deprecated in
# favor of the Pythonic under_score methods. So
# multiprocessing.Event *only* has is_set(), while
# threading.Event has *both* isSet() and is_set(). In order
# to make the core work with Python 2.4+, and with both
# multiprocessing and threading Event objects, we just
# monkeypatch self.terminate to have isSet().
self.terminate = DualEvent(threading_event=self.terminate)
def shutdown(self):
BuiltinCore.shutdown(self)
for child in multiprocessing.active_children():
self.logger.debug("Shutting down child %s" % child.name)
child.join(self.shutdown_timeout)
if child.is_alive():
self.logger.error("Waited %s seconds to shut down %s, "
"terminating" % (self.shutdown_timeout,
child.name))
child.terminate()
else:
self.logger.debug("Child %s shut down" % child.name)
self.logger.debug("All children shut down")
def shutdown(self):
BuiltinCore.shutdown(self)
for child in multiprocessing.active_children():
self.logger.debug("Shutting down child %s" % child.name)
child.join(self.shutdown_timeout)
if child.is_alive():
self.logger.error("Waited %s seconds to shut down %s, "
"terminating" %
(self.shutdown_timeout, child.name))
child.terminate()
else:
self.logger.debug("Child %s shut down" % child.name)
self.logger.debug("All children shut down")
def _run(self):
for cnum in range(self.setup['children']):
name = "Child-%s" % cnum
(mainpipe, childpipe) = multiprocessing.Pipe()
self.pipes[name] = mainpipe
self.logger.debug("Starting child %s" % name)
childcore = ChildCore(self.setup, childpipe, self.terminate)
child = multiprocessing.Process(target=childcore.run, name=name)
child.start()
self.logger.debug("Child %s started with PID %s" % (name,
child.pid))
self.available_children.put(name)
return BuiltinCore._run(self)
def _run(self):
for cnum in range(self.setup['children']):
name = "Child-%s" % cnum
(mainpipe, childpipe) = multiprocessing.Pipe()
self.pipes[name] = mainpipe
self.logger.debug("Starting child %s" % name)
childcore = ChildCore(self.setup, childpipe, self.terminate)
child = multiprocessing.Process(target=childcore.run, name=name)
child.start()
self.logger.debug("Child %s started with PID %s" %
(name, child.pid))
self.available_children.put(name)
return BuiltinCore._run(self)