def test_adaptive_local_cluster_multi_workers():
loop = IOLoop.current()
cluster = LocalCluster(0, scheduler_port=0, silence_logs=False, nanny=False,
diagnostics_port=None, loop=loop, start=False)
cluster.scheduler.allowed_failures = 1000
alc = Adaptive(cluster.scheduler, cluster, interval=100)
c = Client(cluster, start=False, loop=loop)
yield c._start()
futures = c.map(slowinc, range(100), delay=0.01)
start = time()
while not cluster.scheduler.worker_info:
yield gen.sleep(0.01)
assert time() < start + 15
yield c._gather(futures)
del futures
start = time()
while cluster.workers:
yield gen.sleep(0.01)
assert time() < start + 5
assert not cluster.workers
yield gen.sleep(0.2)
assert not cluster.workers
futures = c.map(slowinc, range(100), delay=0.01)
yield c._gather(futures)
yield c._shutdown()
yield cluster._close()
def test_adaptive_local_cluster_multi_workers():
loop = IOLoop.current()
cluster = LocalCluster(0,
scheduler_port=0,
silence_logs=False,
processes=False,
diagnostics_port=None,
loop=loop,
start=False)
cluster.scheduler.allowed_failures = 1000
alc = Adaptive(cluster.scheduler, cluster, interval=100)
c = Client(cluster, start=False, loop=loop)
yield c._start()
futures = c.map(slowinc, range(100), delay=0.01)
start = time()
while not cluster.scheduler.worker_info:
yield gen.sleep(0.01)
assert time() < start + 15
yield c._gather(futures)
del futures
start = time()
while cluster.workers:
yield gen.sleep(0.01)
assert time() < start + 5
assert not cluster.workers
assert not cluster.scheduler.workers
yield gen.sleep(0.2)
assert not cluster.workers
assert not cluster.scheduler.workers
futures = c.map(slowinc, range(100), delay=0.01)
yield c._gather(futures)
yield c._shutdown()
yield cluster._close()
def test_adaptive_local_cluster_multi_workers():
loop = IOLoop.current()
cluster = LocalCluster(0, scheduler_port=0, silence_logs=False, nanny=False,
diagnostic_port=None, loop=loop, start=False)
alc = Adaptive(cluster.scheduler, cluster, interval=100)
c = Client(cluster, start=False, loop=loop)
yield c._start()
for i in range(20):
futures = c.map(slowinc, range(100), delay=0.01)
yield c._gather(futures)
del futures
yield gen.sleep(0.1)
yield c._shutdown()
yield cluster._close()
def test_adaptive_local_cluster_multi_workers():
loop = IOLoop.current()
cluster = LocalCluster(0,
scheduler_port=0,
silence_logs=False,
nanny=False,
diagnostic_port=None,
loop=loop,
start=False)
alc = Adaptive(cluster.scheduler, cluster, interval=100)
c = Client(cluster, start=False, loop=loop)
yield c._start()
for i in range(20):
futures = c.map(slowinc, range(100), delay=0.01)
yield c._gather(futures)
del futures
yield gen.sleep(0.1)
yield c._shutdown()
yield cluster._close()
def test_publish_roundtrip(s, a, b):
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
data = yield c._scatter([0, 1, 2])
yield c._publish_dataset(data=data)
assert 'published-data' in s.who_wants[data[0].key]
result = yield f._get_dataset(name='data')
assert len(result) == len(data)
out = yield f._gather(result)
assert out == [0, 1, 2]
with pytest.raises(KeyError) as exc_info:
result = yield f._get_dataset(name='nonexistent')
assert "not found" in str(exc_info.value)
assert "nonexistent" in str(exc_info.value)
yield c._shutdown()
yield f._shutdown()
def test_publish_roundtrip(s, a, b):
c = Client((s.ip, s.port), start=False)
yield c._start()
f = Client((s.ip, s.port), start=False)
yield f._start()
data = yield c._scatter([0, 1, 2])
yield c._publish_dataset(data=data)
assert 'published-data' in s.who_wants[data[0].key]
result = yield f._get_dataset(name='data')
assert len(result) == len(data)
out = yield f._gather(result)
assert out == [0, 1, 2]
with pytest.raises(KeyError) as exc_info:
result = yield f._get_dataset(name='nonexistent')
assert "not found" in str(exc_info.value)
assert "nonexistent" in str(exc_info.value)
yield c._shutdown()
yield f._shutdown()
