def main(ctx, scheduler, scheduler_port, hostnames, hostfile, nthreads, nprocs,
ssh_username, ssh_port, ssh_private_key, log_directory):
try:
hostnames = list(hostnames)
if hostfile:
with open(hostfile) as f:
hosts = f.read().split()
hostnames.extend(hosts)
if not scheduler:
scheduler = hostnames[0]
except IndexError:
print(ctx.get_help())
exit(1)
c = SSHCluster(scheduler, scheduler_port, hostnames, nthreads, nprocs,
ssh_username, ssh_port, ssh_private_key, log_directory)
import distributed
print('\n---------------------------------------------------------------')
print(' Dask.distributed v{version}\n'.format(version=distributed.__version__))
print('Worker nodes:'.format(n=len(hostnames)))
for i, host in enumerate(hostnames):
print(' {num}: {host}'.format(num = i, host = host))
print('\nscheduler node: {addr}:{port}'.format(addr=scheduler, port=scheduler_port))
print('---------------------------------------------------------------\n\n')
# Monitor the output of remote processes. This blocks until the user issues a KeyboardInterrupt.
c.monitor_remote_processes()
# Close down the remote processes and exit.
print("\n[ dask-ssh ]: Shutting down remote processes (this may take a moment).")
c.shutdown()
print("[ dask-ssh ]: Remote processes have been terminated. Exiting.")
def test_defer_to_old(loop):
with pytest.warns(Warning):
with SSHCluster(
scheduler_addr="127.0.0.1",
scheduler_port=7437,
worker_addrs=["127.0.0.1", "127.0.0.1"],
) as c:
from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
assert isinstance(c, OldSSHCluster)
async def test_remote_python_as_dict():
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=[dict(known_hosts=None)] * 3,
asynchronous=True,
scheduler_options={"idle_timeout": "5s"},
worker_options={"death_timeout": "5s"},
remote_python=[sys.executable] * 3,
) as cluster:
assert cluster.workers[0].remote_python == sys.executable
async def test_ssh_n_workers_with_nprocs_is_an_error():
with pytest.raises(ValueError, match="Both nprocs and n_workers"):
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=dict(known_hosts=None),
asynchronous=True,
scheduler_options={},
worker_options={"n_workers": 2, "nprocs": 2},
) as cluster:
assert not cluster
async def test_list_of_connect_options_raises():
with pytest.raises(RuntimeError):
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=[dict(known_hosts=None)] * 4, # Mismatch in length 4 != 3
asynchronous=True,
scheduler_options={"idle_timeout": "5s"},
worker_options={"death_timeout": "5s"},
) as _:
pass
async def test_ssh_cluster_raises_if_asyncssh_not_installed(monkeypatch, cleanup):
monkeypatch.setitem(sys.modules, "asyncssh", None)
with pytest.raises(ImportError, match="SSHCluster requires the `asyncssh` package"):
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=[dict(known_hosts=None)] * 3,
asynchronous=True,
scheduler_options={"idle_timeout": "5s"},
worker_options={"death_timeout": "5s"},
) as cluster:
assert not cluster
async def test_ssh_nprocs_renamed_to_n_workers():
with pytest.warns(FutureWarning, match="renamed to n_workers"):
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=dict(known_hosts=None),
asynchronous=True,
scheduler_options={"idle_timeout": "5s"},
worker_options={"death_timeout": "5s", "nprocs": 2},
) as cluster:
assert len(cluster.workers) == 2
async with Client(cluster, asynchronous=True) as client:
await client.wait_for_workers(4)
def main(ctx, scheduler, scheduler_port, hostnames, hostfile, nthreads, nprocs,
ssh_username, ssh_port, ssh_private_key, nohost, log_directory, remote_python):
try:
hostnames = list(hostnames)
if hostfile:
with open(hostfile) as f:
hosts = f.read().split()
hostnames.extend(hosts)
if not scheduler:
scheduler = hostnames[0]
except IndexError:
print(ctx.get_help())
exit(1)
c = SSHCluster(scheduler, scheduler_port, hostnames, nthreads, nprocs,
ssh_username, ssh_port, ssh_private_key, nohost, log_directory, remote_python)
import distributed
print('\n---------------------------------------------------------------')
print(' Dask.distributed v{version}\n'.format(version=distributed.__version__))
print('Worker nodes:'.format(n=len(hostnames)))
for i, host in enumerate(hostnames):
print(' {num}: {host}'.format(num=i, host=host))
print('\nscheduler node: {addr}:{port}'.format(addr=scheduler, port=scheduler_port))
print('---------------------------------------------------------------\n\n')
# Monitor the output of remote processes. This blocks until the user issues a KeyboardInterrupt.
c.monitor_remote_processes()
# Close down the remote processes and exit.
print("\n[ dask-ssh ]: Shutting down remote processes (this may take a moment).")
c.shutdown()
print("[ dask-ssh ]: Remote processes have been terminated. Exiting.")
async def test_basic():
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=dict(known_hosts=None),
asynchronous=True,
scheduler_options={"idle_timeout": "5s"},
worker_options={"death_timeout": "5s"},
) as cluster:
assert len(cluster.workers) == 2
async with Client(cluster, asynchronous=True) as client:
result = await client.submit(lambda x: x + 1, 10)
assert result == 11
assert not cluster._supports_scaling
assert "SSH" in repr(cluster)
async def test_unimplemented_options():
with pytest.raises(Exception):
async with SSHCluster(
["127.0.0.1"] * 3,
connect_kwargs=dict(known_hosts=None),
asynchronous=True,
worker_kwargs={
"nthreads": 2,
"memory_limit": "2 GiB",
"death_timeout": "5s",
"unimplemented_option": 2,
},
scheduler_kwargs={"idle_timeout": "5s"},
) as cluster:
assert cluster
async def test_config_inherited_by_subprocess(loop):
def f(x):
return dask.config.get("foo") + 1
with dask.config.set(foo=100):
async with SSHCluster(
["127.0.0.1"] * 2,
connect_options=dict(known_hosts=None),
asynchronous=True,
scheduler_options={"idle_timeout": "5s"},
worker_options={"death_timeout": "5s"},
) as cluster:
async with Client(cluster, asynchronous=True) as client:
result = await client.submit(f, 1)
assert result == 101
async def test_list_of_remote_python_raises():
with pytest.raises(RuntimeError):
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=[dict(known_hosts=None)] * 3,
asynchronous=True,
scheduler_options={
"port": 0,
"idle_timeout": "5s"
},
worker_options={"death_timeout": "5s"},
remote_python=[sys.executable] *
4, # Mismatch in length 4 != 3
) as _:
pass
def test_cluster(loop):
with SSHCluster(scheduler_addr='127.0.0.1',
scheduler_port=7437,
worker_addrs=['127.0.0.1', '127.0.0.1']) as c:
with Client(c, loop=loop) as e:
start = time()
while len(e.ncores()) != 2:
sleep(0.01)
assert time() < start + 5
c.add_worker('127.0.0.1')
start = time()
while len(e.ncores()) != 3:
sleep(0.01)
assert time() < start + 5
async def test_keywords():
async with SSHCluster(
["127.0.0.1"] * 3,
connect_options=dict(known_hosts=None),
asynchronous=True,
worker_options={
"nthreads": 2,
"memory_limit": "2 GiB",
"death_timeout": "5s",
},
scheduler_options={"idle_timeout": "10s"},
) as cluster:
async with Client(cluster, asynchronous=True) as client:
assert (await client.run_on_scheduler(
lambda dask_scheduler: dask_scheduler.idle_timeout)) == 10
d = client.scheduler_info()["workers"]
assert all(v["nthreads"] == 2 for v in d.values())
async def test_nprocs_attribute_is_deprecated():
async with SSHCluster(
["127.0.0.1"] * 2,
connect_options=dict(known_hosts=None),
asynchronous=True,
scheduler_options={"idle_timeout": "5s"},
worker_options={"death_timeout": "5s"},
) as cluster:
assert len(cluster.workers) == 1
worker = cluster.workers[0]
assert worker.n_workers == 1
with pytest.warns(FutureWarning, match="renamed to n_workers"):
assert worker.nprocs == 1
with pytest.warns(FutureWarning, match="renamed to n_workers"):
worker.nprocs = 3
assert worker.n_workers == 3
def test_ssh_hosts_None():
with pytest.raises(ValueError):
SSHCluster(hosts=None)
def test_ssh_hosts_empty_list():
with pytest.raises(ValueError):
SSHCluster(hosts=[])
