class Core(BuiltinCore):
""" A multiprocessing core that delegates building the actual
client configurations to
:class:`Bcfg2.Server.MultiprocessingCore.ChildCore` objects. The
parent process doesn't build any children itself; all calls to
:func:`GetConfig` are delegated to children. All other calls are
handled by the parent process. """
#: How long to wait for a child process to shut down cleanly
#: before it is terminated.
shutdown_timeout = 10.0
def __init__(self, setup):
BuiltinCore.__init__(self, setup)
if setup['children'] is None:
setup['children'] = multiprocessing.cpu_count()
#: A dict of child name -> one end of the
#: :class:`multiprocessing.Pipe` object used to communicate
#: with that child. (The child is given the other end of the
#: Pipe.)
self.pipes = dict()
#: A queue that keeps track of which children are available to
#: render a configuration. A child is popped from the queue
#: when it starts to render a config, then it's pushed back on
#: when it's done. This lets us use a blocking call to
#: :func:`Queue.Queue.get` when waiting for an available
#: child.
self.available_children = Queue(maxsize=self.setup['children'])
# sigh. multiprocessing was added in py2.6, which is when the
# camelCase methods for threading objects were deprecated in
# favor of the Pythonic under_score methods. So
# multiprocessing.Event *only* has is_set(), while
# threading.Event has *both* isSet() and is_set(). In order
# to make the core work with Python 2.4+, and with both
# multiprocessing and threading Event objects, we just
# monkeypatch self.terminate to have isSet().
self.terminate = DualEvent(threading_event=self.terminate)
def _run(self):
for cnum in range(self.setup['children']):
name = "Child-%s" % cnum
(mainpipe, childpipe) = multiprocessing.Pipe()
self.pipes[name] = mainpipe
self.logger.debug("Starting child %s" % name)
childcore = ChildCore(self.setup, childpipe, self.terminate)
child = multiprocessing.Process(target=childcore.run, name=name)
child.start()
self.logger.debug("Child %s started with PID %s" % (name,
child.pid))
self.available_children.put(name)
return BuiltinCore._run(self)
def shutdown(self):
BuiltinCore.shutdown(self)
for child in multiprocessing.active_children():
self.logger.debug("Shutting down child %s" % child.name)
child.join(self.shutdown_timeout)
if child.is_alive():
self.logger.error("Waited %s seconds to shut down %s, "
"terminating" % (self.shutdown_timeout,
child.name))
child.terminate()
else:
self.logger.debug("Child %s shut down" % child.name)
self.logger.debug("All children shut down")
@exposed
def GetConfig(self, address):
client = self.resolve_client(address)[0]
childname = self.available_children.get()
self.logger.debug("Building configuration on child %s" % childname)
pipe = self.pipes[childname]
pipe.send(client)
config = pipe.recv()
self.available_children.put_nowait(childname)
return config
class Snapshots(Bcfg2.Server.Plugin.Statistics):
name = 'Snapshots'
deprecated = True
def __init__(self, core, datastore):
Bcfg2.Server.Plugin.Statistics.__init__(self, core, datastore)
self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile)
self.work_queue = Queue()
self.loader = threading.Thread(target=self.load_snapshot)
self.loader.start()
def load_snapshot(self):
while self.running:
try:
(metadata, data) = self.work_queue.get(block=True, timeout=5)
except:
continue
self.statistics_from_old_stats(metadata, data)
def process_statistics(self, metadata, data):
return self.work_queue.put((metadata, data))
def statistics_from_old_stats(self, metadata, xdata):
# entries are name -> (modified, correct, start, desired, end)
# not sure we can get all of this from old format stats
t1 = time.time()
entries = dict([('Package', dict()), ('Service', dict()),
('Path', dict())])
extra = dict([('Package', dict()), ('Service', dict()),
('Path', dict())])
bad = []
state = xdata.find('.//Statistics')
correct = state.get('state') == 'clean'
revision = u_str(state.get('revision', '-1'))
for entry in state.find('.//Bad'):
data = [False, False, u_str(entry.get('name'))] \
+ build_snap_ent(entry)
if entry.tag in ftypes:
etag = 'Path'
else:
etag = entry.tag
entries[etag][entry.get('name')] = data
for entry in state.find('.//Modified'):
if entry.tag in ftypes:
etag = 'Path'
else:
etag = entry.tag
if entry.get('name') in entries[etag]:
data = [True, False, u_str(entry.get('name'))] + \
build_snap_ent(entry)
else:
data = [True, False, u_str(entry.get('name'))] + \
build_snap_ent(entry)
for entry in state.find('.//Extra'):
if entry.tag in datafields:
data = build_snap_ent(entry)[1]
ename = u_str(entry.get('name'))
data['name'] = ename
extra[entry.tag][ename] = data
else:
print("extra", entry.tag, entry.get('name'))
t2 = time.time()
snap = Snapshot.from_data(self.session, correct, revision, metadata,
entries, extra)
self.session.add(snap)
self.session.commit()
t3 = time.time()
logger.info("Snapshot storage took %fs" % (t3 - t2))
return True
class Snapshots(Bcfg2.Server.Plugin.Statistics):
name = 'Snapshots'
deprecated = True
def __init__(self, core, datastore):
Bcfg2.Server.Plugin.Statistics.__init__(self, core, datastore)
self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile)
self.work_queue = Queue()
self.loader = threading.Thread(target=self.load_snapshot)
def start_threads(self):
self.loader.start()
def load_snapshot(self):
while self.running:
try:
(metadata, data) = self.work_queue.get(block=True, timeout=5)
except:
continue
self.statistics_from_old_stats(metadata, data)
def process_statistics(self, metadata, data):
return self.work_queue.put((metadata, data))
def statistics_from_old_stats(self, metadata, xdata):
# entries are name -> (modified, correct, start, desired, end)
# not sure we can get all of this from old format stats
t1 = time.time()
entries = dict([('Package', dict()),
('Service', dict()), ('Path', dict())])
extra = dict([('Package', dict()), ('Service', dict()),
('Path', dict())])
bad = []
state = xdata.find('.//Statistics')
correct = state.get('state') == 'clean'
revision = u_str(state.get('revision', '-1'))
for entry in state.find('.//Bad'):
data = [False, False, u_str(entry.get('name'))] \
+ build_snap_ent(entry)
if entry.tag in ftypes:
etag = 'Path'
else:
etag = entry.tag
entries[etag][entry.get('name')] = data
for entry in state.find('.//Modified'):
if entry.tag in ftypes:
etag = 'Path'
else:
etag = entry.tag
if entry.get('name') in entries[etag]:
data = [True, False, u_str(entry.get('name'))] + \
build_snap_ent(entry)
else:
data = [True, False, u_str(entry.get('name'))] + \
build_snap_ent(entry)
for entry in state.find('.//Extra'):
if entry.tag in datafields:
data = build_snap_ent(entry)[1]
ename = u_str(entry.get('name'))
data['name'] = ename
extra[entry.tag][ename] = data
else:
print("extra", entry.tag, entry.get('name'))
t2 = time.time()
snap = Snapshot.from_data(self.session, correct, revision,
metadata, entries, extra)
self.session.add(snap)
self.session.commit()
t3 = time.time()
logger.info("Snapshot storage took %fs" % (t3 - t2))
return True
class Core(BuiltinCore):
""" A multiprocessing core that delegates building the actual
client configurations to
:class:`Bcfg2.Server.MultiprocessingCore.ChildCore` objects. The
parent process doesn't build any children itself; all calls to
:func:`GetConfig` are delegated to children. All other calls are
handled by the parent process. """
#: How long to wait for a child process to shut down cleanly
#: before it is terminated.
shutdown_timeout = 10.0
def __init__(self, setup):
BuiltinCore.__init__(self, setup)
if setup['children'] is None:
setup['children'] = multiprocessing.cpu_count()
#: A dict of child name -> one end of the
#: :class:`multiprocessing.Pipe` object used to communicate
#: with that child. (The child is given the other end of the
#: Pipe.)
self.pipes = dict()
#: A queue that keeps track of which children are available to
#: render a configuration. A child is popped from the queue
#: when it starts to render a config, then it's pushed back on
#: when it's done. This lets us use a blocking call to
#: :func:`Queue.Queue.get` when waiting for an available
#: child.
self.available_children = Queue(maxsize=self.setup['children'])
# sigh. multiprocessing was added in py2.6, which is when the
# camelCase methods for threading objects were deprecated in
# favor of the Pythonic under_score methods. So
# multiprocessing.Event *only* has is_set(), while
# threading.Event has *both* isSet() and is_set(). In order
# to make the core work with Python 2.4+, and with both
# multiprocessing and threading Event objects, we just
# monkeypatch self.terminate to have isSet().
self.terminate = DualEvent(threading_event=self.terminate)
def _run(self):
for cnum in range(self.setup['children']):
name = "Child-%s" % cnum
(mainpipe, childpipe) = multiprocessing.Pipe()
self.pipes[name] = mainpipe
self.logger.debug("Starting child %s" % name)
childcore = ChildCore(self.setup, childpipe, self.terminate)
child = multiprocessing.Process(target=childcore.run, name=name)
child.start()
self.logger.debug("Child %s started with PID %s" %
(name, child.pid))
self.available_children.put(name)
return BuiltinCore._run(self)
def shutdown(self):
BuiltinCore.shutdown(self)
for child in multiprocessing.active_children():
self.logger.debug("Shutting down child %s" % child.name)
child.join(self.shutdown_timeout)
if child.is_alive():
self.logger.error("Waited %s seconds to shut down %s, "
"terminating" %
(self.shutdown_timeout, child.name))
child.terminate()
else:
self.logger.debug("Child %s shut down" % child.name)
self.logger.debug("All children shut down")
@exposed
def GetConfig(self, address):
client = self.resolve_client(address)[0]
childname = self.available_children.get()
self.logger.debug("Building configuration on child %s" % childname)
pipe = self.pipes[childname]
pipe.send(client)
config = pipe.recv()
self.available_children.put_nowait(childname)
return config
class RPCQueue(Bcfg2.Server.Plugin.Debuggable):
""" An implementation of a :class:`multiprocessing.Queue` designed
for several additional use patterns:
* Random-access reads, based on a key that identifies the data;
* Publish-subscribe, where a datum is sent to all hosts.
The subscribers can deal with this as a normal Queue with no
special handling.
"""
poll_wait = 3.0
def __init__(self):
Bcfg2.Server.Plugin.Debuggable.__init__(self)
self._terminate = threading.Event()
self._queues = dict()
self._available_listeners = Queue()
self._blocking_listeners = []
def add_subscriber(self, name):
""" Add a subscriber to the queue. This returns the
:class:`multiprocessing.Queue` object that the subscriber
should read from. """
self._queues[name] = multiprocessing.Queue()
return self._queues[name]
def publish(self, method, args=None, kwargs=None):
""" Publish an RPC call to the queue for consumption by all
subscribers. """
for queue in self._queues.values():
queue.put((None, (method, args or [], kwargs or dict())))
def rpc(self, dest, method, args=None, kwargs=None):
""" Make an RPC call to the named subscriber, expecting a
response. This opens a
:class:`multiprocessing.connection.Listener` and passes the
Listener address to the child as part of the RPC call, so that
the child can connect to the Listener to submit its results.
Listeners are reused when possible to minimize overhead.
"""
try:
listener = self._available_listeners.get_nowait()
self.logger.debug("Reusing existing RPC listener at %s" %
listener.address)
except Empty:
listener = Listener()
self.logger.debug("Created new RPC listener at %s" %
listener.address)
self._blocking_listeners.append(listener)
try:
self._queues[dest].put((listener.address,
(method, args or [], kwargs or dict())))
conn = listener.accept()
self._blocking_listeners.remove(listener)
try:
while not self._terminate.is_set():
if conn.poll(self.poll_wait):
return conn.recv()
finally:
conn.close()
finally:
self._available_listeners.put(listener)
def close(self):
""" Close queues and connections. """
self._terminate.set()
self.logger.debug("Closing RPC queues")
for name, queue in self._queues.items():
self.logger.debug("Closing RPC queue to %s" % name)
queue.close()
# close any listeners that are waiting for connections
self.logger.debug("Closing RPC connections")
for listener in self._blocking_listeners:
self.logger.debug("Closing RPC connection at %s" %
listener.address)
listener.close()
self.logger.debug("Closing RPC listeners")
try:
while True:
listener = self._available_listeners.get_nowait()
self.logger.debug("Closing RPC listener at %s" %
listener.address)
listener.close()
except Empty:
pass
