def clear(self, key):
'''
Clear any state (progress flags and results) for key.
'''
with self._lock:
with catch(KeyError):
del self._done[key]
with catch(KeyError):
del self._results[key]
def _stop_tasks(self):
# cancel any pending io work
for task in self._iotasks:
with catch(RuntimeError):
task.cancel()
self._iotasks.clear()
# close the servers
if self.server:
with catch(RuntimeError):
self.server.cancel()
def remove_blocks(self, name, from_peers=False):
'''
Remove blocks being served.
:param name: Name of the blocks to be removed.
:param from_peers: If True, the blocks under name will be removed from
other peer nodes as well.
'''
with catch(KeyError):
del self.cache[name]
with catch(KeyError):
del self._available_events[name]
if from_peers:
for peer in self.worker.peers.filter():
peer._remove_blocks(name)
def run_workers():
signal.signal(signal.SIGUSR1, dump_threads)
argparser = argparse.ArgumentParser(parents=[many_argparser])
conf = bndl.conf
def_worker_count = conf.get(
'bndl.compute.worker_count') or os.cpu_count() or 1
argparser.add_argument('process_count',
nargs='?',
type=int,
default=def_worker_count,
metavar='worker count',
help='The number of workers to start (defaults'
' to %s).' % def_worker_count)
args = argparser.parse_args()
# reconstruct the arguments for the worker
# parse_known_args doesn't take out the worker_count positional argument correctly
worker_args = []
if args.listen_addresses:
worker_args += ['--listen-addresses'] + args.listen_addresses
if args.seeds:
worker_args += ['--seeds'] + args.seeds
superv = WorkerSupervisor.from_args(args, worker_args)
superv.start()
try:
superv.wait()
except KeyboardInterrupt:
with catch(log_level=logging.WARNING):
superv.stop()
def _stop_tasks(self):
# stop watching
if self._watchdog:
with catch(RuntimeError, log_level=logging.WARNING):
self._watchdog.stop()
self._watchdog = None
# cancel any pending io work
for task in self._iotasks:
with catch(RuntimeError, log_level=logging.WARNING):
task.cancel()
self._iotasks.clear()
# close the servers
for server in self.servers.values():
if server:
with catch(RuntimeError, log_level=logging.WARNING):
server.close()
def _peer_connected(self, peer):
with (yield from self._peer_table_lock):
if self.name == peer.name:
# don't allow connection loops
logger.debug('self connect attempt of %s', peer.name)
yield from peer.disconnect(reason='self connect')
return
known_peer = self.peers.get(peer.name)
if not known_peer:
for address in peer.addresses:
matches = self.peers.filter(address=address,
connected=None)
if matches:
known_peer = matches[0]
if known_peer.is_connected:
logger.warning(
'Peers %s and %s (of %s) share addresses (%s) and are both connected',
known_peer.name, peer.name, self.name,
list(known_peer.addresses) +
list(peer.addresses))
break
if known_peer:
if known_peer.is_connected and peer is not known_peer:
# perform a 'tie brake' between the two connections with the peer
# this is to prevent situations where two nodes 'call each other'
# at the same time
if known_peer < peer:
# existing connection wins
logger.debug('already connected with %s, closing %s',
peer.name, known_peer.conn)
yield from peer.disconnect(
reason='already connected, old connection wins')
return
else:
# new connection wins
logger.debug('already connected with %s, closing %s',
peer.name, known_peer.conn)
yield from known_peer.disconnect(
reason='already connected, new connection wins')
if known_peer.name != peer.name:
with catch(KeyError):
del self.peers[known_peer.name]
self.peers[peer.name] = peer
# notify others of the new peer
if peer.cluster:
task = self.loop.create_task(self._notifiy_peers(peer))
self._iotasks.add(task)
task.add_done_callback(self._iotasks.discard)
return True
def disconnect(self, reason='', active=True):
logger.log(
logging.INFO if active and self.is_connected else logging.DEBUG,
'%s (local) disconnected from %s (remote) with reason: %s (%s disconnect)',
self.local.name, self.name, reason,
'active' if active and self.is_connected else 'passive')
# possibly notify the other end
if active and self.is_connected:
with catch():
yield from self.send(Disconnect(reason=reason), drain=True)
# close the io tasks and the server
self._stop_tasks()
# close the connection
if self.conn:
with catch():
yield from self.conn.close()
# clear the fields
self.server = None
self.conn = None
def serve(self, reader, writer, init=False):
try:
conn = Connection(self.loop, reader, writer)
peer = self.PeerNode(self.loop, self)
connect = peer._connect if init else \
peer._connected
yield from connect(conn)
except GeneratorExit:
conn.close()
except Exception:
with catch():
conn.close()
logger.exception('unable to accept connection from %s',
conn.peername())
def create_ctx(config=None):
from bndl.util import dash
if config is None:
config = bndl.conf
listen_addresses = config.get('bndl.net.listen_addresses')
seeds = config.get('bndl.net.seeds')
worker_count = config.get('bndl.compute.worker_count')
supervisor = None
def stop():
# signal the aio loop can stop and everything can be torn down
if supervisor:
supervisor.stop()
try:
supervisor.wait()
except TimeoutExpired:
pass
stop_nodes([driver])
dash.stop()
driver = Driver(addresses=listen_addresses, seeds=seeds)
start_nodes([driver])
try:
if not seeds and worker_count is None:
worker_count = os.cpu_count()
if worker_count:
args = ['--seeds'] + list((seeds or driver.addresses))
if listen_addresses:
args += ['--listen-addresses'] + listen_addresses
supervisor = WorkerSupervisor(args, worker_count)
supervisor.start()
ctx = ComputeContext(driver, config=config)
dash.run(driver, ctx)
ctx.add_listener(lambda obj: stop()
if obj is ctx and ctx.stopped else None)
atexit.register(stop)
atexit.register(ctx.stop)
return ctx
except Exception:
with catch(log_level=logging.WARNING):
stop()
raise
def main():
signal.signal(signal.SIGUSR1, dump_threads)
try:
args = argparser.parse_args()
config = bndl.conf
if args.listen_addresses:
config['bndl.net.listen_addresses'] = args.listen_addresses
if args.seeds:
config['bndl.net.seeds'] = args.seeds
config['bndl.compute.worker_count'] = 0
if args.worker_count is not None:
config['bndl.compute.worker_count'] = args.worker_count
config['bndl.run.numactl'] = args.numactl
config['bndl.run.pincore'] = args.pincore
config['bndl.run.jemalloc'] = args.jemalloc
config.update(*args.conf)
ctx = create_ctx(config)
ns = dict(ctx=ctx)
if config['bndl.net.seeds'] or config['bndl.compute.worker_count'] != 0:
print('Connecting with workers ...', end='\r')
worker_count = ctx.await_workers(args.worker_count)
node_count = len(
groupby(identity, [
tuple(sorted(worker.ip_addresses()))
for worker in ctx.workers
]))
header = HEADER + '\nConnected with %r workers on %r nodes.' % (
worker_count, node_count)
else:
header = HEADER
try:
import IPython
IPython.embed(header=header, user_ns=ns)
except ImportError:
import code
code.interact(header, local=ns)
finally:
with catch():
ctx.stop()
def node(self):
if self._node is None:
with catch(RuntimeError):
self._node = current_worker()
return self._node
def _remove_blocks(self, peer, name):
with catch(KeyError):
del self.cache[name]
with catch(KeyError):
del self._available_events[name]