def test_cleanup():
c = LocalCluster(2, scheduler_port=0, silence_logs=False,
diagnostic_port=None)
port = c.scheduler.port
c.close()
c2 = LocalCluster(2, scheduler_port=port, silence_logs=False,
diagnostic_port=None)
c.close()
def test_scale_up_and_down():
loop = IOLoop.current()
cluster = LocalCluster(0, scheduler_port=0, nanny=False, silence_logs=False,
diagnostics_port=None, loop=loop, start=False)
c = Client(cluster, start=False, loop=loop)
yield c._start()
assert not cluster.workers
yield cluster.scale_up(2)
assert len(cluster.workers) == 2
assert len(cluster.scheduler.ncores) == 2
addr = cluster.workers[0].address
yield cluster.scale_down([addr])
assert len(cluster.workers) == 1
assert addr not in cluster.scheduler.ncores
yield c._shutdown()
yield cluster._close()
def test_only_local_access(loop):
with LocalCluster(
0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
) as c:
sync(loop, assert_can_connect_locally_4, c.scheduler.port)
def test_protocol_inproc(loop):
with LocalCluster(protocol="inproc://", loop=loop, processes=False) as cluster:
assert cluster.scheduler.address.startswith("inproc://")
def test_protocol_ip(loop):
with LocalCluster(host="tcp://127.0.0.2",
loop=loop,
n_workers=0,
processes=False) as cluster:
assert cluster.scheduler.address.startswith("tcp://127.0.0.2")
def test_dont_select_closed_worker():
# Make sure distributed does not try to reuse a client from a
# closed cluster (https://github.com/dask/distributed/issues/2840).
with clean(threads=False):
cluster = LocalCluster(n_workers=0)
c = Client(cluster)
cluster.scale(2)
assert c == get_client()
c.close()
cluster.close()
cluster2 = LocalCluster(n_workers=0)
c2 = Client(cluster2)
cluster2.scale(2)
current_client = get_client()
assert c2 == current_client
cluster2.close()
c2.close()
async def test_defaults(cleanup):
async with LocalCluster(
scheduler_port=0, silence_logs=False, dashboard_address=None, asynchronous=True
) as c:
assert sum(w.nthreads for w in c.workers.values()) == CPU_COUNT
assert all(isinstance(w, Nanny) for w in c.workers.values())
async def test_async_with():
async with LocalCluster(processes=False, asynchronous=True) as cluster:
w = cluster.workers
assert w
assert not w
def test_protocol_ip(loop):
with LocalCluster(ip='tcp://127.0.0.2', loop=loop,
processes=False) as cluster:
assert cluster.scheduler.address.startswith('tcp://127.0.0.2')
def test_memory_limit_none():
with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
processes=False, diagnostics_port=None, memory_limit=None) as c:
w = c.workers[0]
assert type(w.data) is dict
assert w.memory_limit is None
def test_remote_access(loop):
with LocalCluster(scheduler_port=0, silence_logs=False,
diagnostics_port=None, ip='', loop=loop) as c:
sync(loop, assert_can_connect_from_everywhere_4_6, c.scheduler.port)
def test_memory(loop):
with LocalCluster(scheduler_port=0, processes=False, silence_logs=False,
diagnostics_port=None, loop=loop) as cluster:
assert sum(w.memory_limit for w in cluster.workers) < TOTAL_MEMORY * 0.8
def test_worker_params():
with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
diagnostics_port=None, memory_limit=500) as c:
assert [w.memory_limit for w in c.workers] == [500] * 2
def test_Client_with_local(loop):
with LocalCluster(1, scheduler_port=0, silence_logs=False,
diagnostics_port=None, loop=loop) as c:
with Client(c, loop=loop) as e:
assert len(e.ncores()) == len(c.workers)
assert c.scheduler_address in repr(c)
def test_death_timeout_raises(loop):
with pytest.raises(gen.TimeoutError):
with LocalCluster(scheduler_port=0, silence_logs=False,
death_timeout=1e-10, diagnostics_port=None,
loop=loop) as cluster:
pass
def test_protocol_tcp(loop):
with LocalCluster(
protocol="tcp", loop=loop, n_workers=0, processes=False
) as cluster:
assert cluster.scheduler.address.startswith("tcp://")
def test_only_local_access(loop):
with LocalCluster(scheduler_port=0,
silence_logs=False,
diagnostics_port=None,
loop=loop) as c:
sync(loop, assert_can_connect_locally_4, c.scheduler.port)
def test_protocol_tcp(loop):
with LocalCluster(protocol='tcp', loop=loop, processes=False) as cluster:
assert cluster.scheduler.address.startswith('tcp://')
async def test_simple():
async with LocalCluster(protocol="ucx", asynchronous=True) as cluster:
async with Client(cluster, asynchronous=True) as client:
assert cluster.scheduler_address.startswith("ucx://")
assert await client.submit(lambda x: x + 1, 10) == 11
