def f():
c = Center('127.0.0.1')
c.listen(8007)
w = Worker(c.ip, c.port, ip='127.0.0.1')
yield w._start()
assert isinstance(w.port, int)
assert w.port > 1024
def test_thread():
c = Center('127.0.0.1', 8000, start=True, block=False)
assert c.loop.is_running()
while not hasattr(c, 'server'):
sleep(0.01)
c.close()
assert not c.loop.is_running()
def f():
c = Center(ip='127.0.0.1')
c.listen(0)
x = Worker(c.ip, c.port, ip='127.0.0.1')
y = Worker(c.ip, c.port, ip='127.0.0.1')
z = Worker(c.ip, c.port, ip='127.0.0.1')
x.data['a'] = 1
y.data['a'] = 2
yield [x._start(), y._start(), z._start()]
zz = rpc(ip=z.ip, port=z.port)
yield zz.compute(function=dumps(inc),
args=dumps(('a', )),
who_has={'a': [x.address]},
key='b')
assert z.data['b'] == 2
yield zz.compute(function=dumps(inc),
args=dumps(('a', )),
who_has={'a': [y.address]},
key='c')
assert z.data['c'] == 3
yield [x._close(), y._close(), z._close()]
zz.close_streams()
def f():
c = Center(ip='127.0.0.1')
c.listen(0)
x = Worker(c.ip, c.port, ip='127.0.0.1')
y = Worker(c.ip, c.port, ip='127.0.0.1')
z = Worker(c.ip, c.port, ip='127.0.0.1')
x.data['a'] = 1
y.data['a'] = 2
yield [x._start(), y._start(), z._start()]
zz = rpc(ip=z.ip, port=z.port)
yield zz.compute(function=dumps(inc),
args=dumps(('a',)),
who_has={'a': [x.address]},
key='b')
assert z.data['b'] == 2
yield zz.compute(function=dumps(inc),
args=dumps(('a',)),
who_has={'a': [y.address]},
key='c')
assert z.data['c'] == 3
yield [x._close(), y._close(), z._close()]
zz.close_streams()
def cluster(**kwargs):
loop = asyncio.new_event_loop()
c = Center('127.0.0.1', 8100, loop=loop)
a = Worker('127.0.0.1', 8101, c.ip, c.port, loop=loop, **kwargs)
b = Worker('127.0.0.1', 8102, c.ip, c.port, loop=loop, **kwargs)
kill_q = Queue()
@asyncio.coroutine
def stop():
while kill_q.empty():
yield from asyncio.sleep(0.01, loop=loop)
kill_q.get()
cor = asyncio.gather(c.go(), a.go(), b.go(), loop=loop)
cor2 = asyncio.wait([stop(), cor],
loop=loop,
return_when=asyncio.FIRST_COMPLETED)
thread, loop = spawn_loop(cor, loop)
try:
yield c, a, b
finally:
if a.status != 'closed':
a.close()
if b.status != 'closed':
b.close()
c.close()
kill_q.put(b'')
def f():
c = Center('127.0.0.1')
c.listen(8007)
w = Worker(c.ip, c.port, ip='127.0.0.1')
yield w._start()
assert isinstance(w.port, int)
assert w.port > 1024
def g():
c = Center('127.0.0.1', 8017)
c.listen(c.port)
a = Worker('127.0.0.1', 8018, c.ip, c.port, ncores=1)
yield a._start()
b = Worker('127.0.0.1', 8019, c.ip, c.port, ncores=1)
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
yield f(c, a, b)
finally:
with ignoring(Exception):
yield a._close()
with ignoring(Exception):
yield b._close()
c.stop()
def f():
c = Center(ip="127.0.0.1")
c.listen(0)
x = Worker(c.ip, c.port, ip="127.0.0.1")
y = Worker(c.ip, c.port, ip="127.0.0.1")
z = Worker(c.ip, c.port, ip="127.0.0.1")
x.data["a"] = 1
y.data["a"] = 2
yield [x._start(), y._start(), z._start()]
zz = rpc(ip=z.ip, port=z.port)
yield zz.compute(function=inc, args=("a",), needed=["a"], who_has={"a": {x.address}}, key="b")
assert z.data["b"] == 2
yield zz.compute(function=inc, args=("a",), needed=["a"], who_has={"a": {y.address}}, key="c")
assert z.data["c"] == 3
yield [x._close(), y._close(), z._close()]
zz.close_streams()
def test_metadata(loop):
c = Center('127.0.0.1', 8006)
c.listen(8006)
@gen.coroutine
def f():
stream = yield TCPClient().connect('127.0.0.1', 8006)
cc = rpc(stream)
response = yield cc.register(address='alice', ncores=4)
assert 'alice' in c.has_what
assert c.ncores['alice'] == 4
response = yield cc.add_keys(address='alice', keys=['x', 'y'])
assert response == b'OK'
response = yield cc.register(address='bob', ncores=4)
response = yield cc.add_keys(address='bob', keys=['y', 'z'])
assert response == b'OK'
response = yield cc.who_has(keys=['x', 'y'])
assert response == {'x': set(['alice']), 'y': set(['alice', 'bob'])}
response = yield cc.remove_keys(address='bob', keys=['y'])
assert response == b'OK'
response = yield cc.has_what(keys=['alice', 'bob'])
assert response == {'alice': set(['x', 'y']), 'bob': set(['z'])}
response = yield cc.ncores()
assert response == {'alice': 4, 'bob': 4}
response = yield cc.ncores(addresses=['alice', 'charlie'])
assert response == {'alice': 4, 'charlie': None}
response = yield cc.unregister(address='alice', close=True)
assert response == b'OK'
assert 'alice' not in c.has_what
assert 'alice' not in c.ncores
c.stop()
IOLoop.current().run_sync(f)
def test_metadata():
c = Center('127.0.0.1', 8006)
c.listen(8006)
@gen.coroutine
def f():
stream = yield TCPClient().connect('127.0.0.1', 8006)
cc = rpc(stream)
response = yield cc.register(address='alice', ncores=4)
assert 'alice' in c.has_what
assert c.ncores['alice'] == 4
response = yield cc.add_keys(address='alice', keys=['x', 'y'])
assert response == b'OK'
response = yield cc.register(address='bob', ncores=4)
response = yield cc.add_keys(address='bob', keys=['y', 'z'])
assert response == b'OK'
response = yield cc.who_has(keys=['x', 'y'])
assert response == {'x': set(['alice']), 'y': set(['alice', 'bob'])}
response = yield cc.remove_keys(address='bob', keys=['y'])
assert response == b'OK'
response = yield cc.has_what(keys=['alice', 'bob'])
assert response == {'alice': set(['x', 'y']), 'bob': set(['z'])}
response = yield cc.ncores()
assert response == {'alice': 4, 'bob': 4}
response = yield cc.unregister(address='alice', close=True)
assert response == b'OK'
assert 'alice' not in c.has_what
assert 'alice' not in c.ncores
c.stop()
IOLoop.current().run_sync(f)
def test_metadata():
c = Center('127.0.0.1', 8006, loop=loop)
@asyncio.coroutine
def f():
reader, writer = yield from connect('127.0.0.1', 8006, loop=loop)
cc = rpc(reader, writer)
response = yield from cc.register(address='alice', ncores=4)
assert 'alice' in c.has_what
assert c.ncores['alice'] == 4
response = yield from cc.add_keys(address='alice', keys=['x', 'y'])
assert response == b'OK'
response = yield from cc.register(address='bob', ncores=4)
response = yield from cc.add_keys(address='bob', keys=['y', 'z'])
assert response == b'OK'
response = yield from cc.who_has(keys=['x', 'y'])
assert response == {'x': set(['alice']), 'y': set(['alice', 'bob'])}
response = yield from cc.remove_keys(address='bob', keys=['y'])
assert response == b'OK'
response = yield from cc.has_what(keys=['alice', 'bob'])
assert response == {'alice': set(['x', 'y']), 'bob': set(['z'])}
response = yield from cc.ncores()
assert response == {'alice': 4, 'bob': 4}
response = yield from cc.unregister(address='alice', close=True)
assert response == b'OK'
assert 'alice' not in c.has_what
assert 'alice' not in c.ncores
yield from c._close()
loop.run_until_complete(asyncio.gather(c.go(), f()))
def test_metadata():
c = Center('127.0.0.1')
c.listen(8006)
cc = rpc(ip='127.0.0.1', port=8006)
response = yield cc.register(address=alice, ncores=4)
assert alice in c.has_what
assert c.ncores[alice] == 4
response = yield cc.add_keys(address=alice, keys=['x', 'y'])
assert response == 'OK'
response = yield cc.register(address=bob, ncores=4)
response = yield cc.add_keys(address=bob, keys=['y', 'z'])
assert response == 'OK'
response = yield cc.who_has(keys=['x', 'y'])
assert valmap(set, response) == {
'x': {alice},
'y': {alice, bob}}
response = yield cc.remove_keys(address=bob, keys=['y'])
assert response == 'OK'
response = yield cc.has_what(keys=[alice, bob])
assert valmap(set, response) == {alice: {'x', 'y'}, bob: {'z'}}
response = yield cc.ncores()
assert response == {alice: 4, bob: 4}
response = yield cc.ncores(addresses=[alice, charlie])
assert response == {alice: 4, charlie: None}
response = yield cc.unregister(address=alice, close=True)
assert response == 'OK'
assert alice not in c.has_what
assert alice not in c.ncores
yield c.terminate()
def test_map_locality():
c = Center('127.0.0.1', 8017, loop=loop)
a = Worker('127.0.0.1', 8018, c.ip, c.port, loop=loop, ncores=4)
b = Worker('127.0.0.1', 8019, c.ip, c.port, loop=loop, ncores=4)
p = Pool(c.ip, c.port, loop=loop, start=False)
@asyncio.coroutine
def f():
while len(c.ncores) < 2:
yield from asyncio.sleep(0.01, loop=loop)
yield from p._sync_center()
results = yield from p._map(lambda x: x * 1000, list(range(20)))
assert p.has_what[(a.ip, a.port)].issuperset(a.data)
assert p.has_what[(b.ip, b.port)].issuperset(b.data)
s = {(a.ip, a.port), (b.ip, b.port)}
assert all(p.who_has[result.key].issubset(s) for result in results)
results2 = yield from p._map(lambda x: -x, results)
aval = set(a.data.values())
bval = set(b.data.values())
try:
assert sum(-v in aval for v in aval) > 0.8 * len(aval)
assert sum(-v in bval for v in bval) > 0.8 * len(bval)
finally:
yield from p._close_connections()
yield from a._close()
yield from b._close()
yield from c._close()
loop.run_until_complete(asyncio.gather(c.go(), a.go(), b.go(), f()))
def test_metadata():
c = Center('127.0.0.1')
c.listen(8006)
stream = yield TCPClient().connect('127.0.0.1', 8006)
cc = rpc(stream)
response = yield cc.register(address='alice', ncores=4)
assert 'alice' in c.has_what
assert c.ncores['alice'] == 4
response = yield cc.add_keys(address='alice', keys=['x', 'y'])
assert response == b'OK'
response = yield cc.register(address='bob', ncores=4)
response = yield cc.add_keys(address='bob', keys=['y', 'z'])
assert response == b'OK'
response = yield cc.who_has(keys=['x', 'y'])
assert response == {'x': set(['alice']), 'y': set(['alice', 'bob'])}
response = yield cc.remove_keys(address='bob', keys=['y'])
assert response == b'OK'
response = yield cc.has_what(keys=['alice', 'bob'])
assert response == {'alice': set(['x', 'y']), 'bob': set(['z'])}
response = yield cc.ncores()
assert response == {'alice': 4, 'bob': 4}
response = yield cc.ncores(addresses=['alice', 'charlie'])
assert response == {'alice': 4, 'charlie': None}
response = yield cc.unregister(address='alice', close=True)
assert response == b'OK'
assert 'alice' not in c.has_what
assert 'alice' not in c.ncores
yield c.terminate()
def test_delete_data():
from distributed import Worker
c = Center('127.0.0.1', 8037, loop=loop)
a = Worker('127.0.0.1', 8038, c.ip, c.port, loop=loop)
@asyncio.coroutine
def f():
while len(c.ncores) < 1:
yield from asyncio.sleep(0.01, loop=loop)
yield from rpc(a.ip, a.port).update_data(data={'x': 1, 'y': 2})
assert a.data == {'x': 1, 'y': 2}
yield from rpc(c.ip, c.port).add_keys(address=(a.ip, a.port),
keys=['x', 'y'])
yield from rpc(c.ip, c.port).delete_data(keys=['x'])
assert a.data == {'y': 2}
assert not c.who_has['x']
assert list(c.has_what[(a.ip, a.port)]) == ['y']
yield from a._close()
yield from c._close()
loop.run_until_complete(asyncio.gather(c.go(), a.go(), f()))
def g():
c = Center('127.0.0.1', 8017)
c.listen(c.port)
a = Worker('127.0.0.1', 8018, c.ip, c.port, ncores=1)
yield a._start()
b = Worker('127.0.0.1', 8019, c.ip, c.port, ncores=1)
yield b._start()
while len(c.ncores) < 2:
yield gen.sleep(0.01)
try:
yield f(c, a, b)
finally:
with ignoring(Exception):
yield a._close()
with ignoring(Exception):
yield b._close()
c.stop()
def test_pool():
c = Center('127.0.0.1', 8017, loop=loop)
a = Worker('127.0.0.1', 8018, c.ip, c.port, loop=loop, ncores=1)
b = Worker('127.0.0.1', 8019, c.ip, c.port, loop=loop, ncores=1)
p = Pool(c.ip, c.port, loop=loop, start=False)
@asyncio.coroutine
def f():
yield from p._sync_center()
computation = yield from p._apply_async(add, [1, 2])
assert computation.status == b'running'
assert set(p.available_cores.values()) == set([0, 1])
x = yield from computation._get()
assert computation.status == x.status == b'success'
assert list(p.available_cores.values()) == [1, 1]
result = yield from x._get()
assert result == 3
computation = yield from p._apply_async(add, [x, 10])
y = yield from computation._get()
result = yield from y._get()
assert result == 13
assert set((len(a.data), len(b.data))) == set((0, 2))
x = yield from p._apply_async(add, [1, 2])
y = yield from p._apply_async(add, [1, 2])
assert list(p.available_cores.values()) == [0, 0]
xx = yield from x._get()
yield from xx._get()
assert set(p.available_cores.values()) == set([0, 1])
yy = yield from y._get()
yield from yy._get()
assert list(p.available_cores.values()) == [1, 1]
seq = yield from p._map(lambda x: x * 100, [1, 2, 3])
result = yield from seq[0]._get(False)
assert result == 100
result = yield from seq[1]._get(False)
assert result == 200
result = yield from seq[2]._get(True)
assert result == 300
# Handle errors gracefully
results = yield from p._map(lambda x: 3 / x, [0, 1, 2, 3])
assert all(isinstance(result, RemoteData) for result in results)
try:
yield from results[0]._get()
assert False
except ZeroDivisionError:
pass
yield from p._close_connections()
yield from a._close()
yield from b._close()
yield from c._close()
loop.run_until_complete(asyncio.gather(c.go(), a.go(), b.go(), f()))
