def deploy_jupyter(nodes: Nodes):
ps_jupyter = "ps -u $USER | grep jupyter ; exit 0"
node = nodes[0]
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
assert nodes.running()
local_port = get_free_local_port()
deployment = node.deploy_notebook(local_port=local_port)
with cancel_on_exit(deployment):
print(deployment)
assert str(deployment) == repr(deployment)
assert deployment.local_port == local_port
ps_jupyter_lines = node.run(ps_jupyter).splitlines()
pprint(ps_jupyter_lines)
assert len(ps_jupyter_lines) == 1
check_local_http_connection(port=local_port)
yield node
retry(lambda: check_no_output(node=node, command=ps_jupyter),
retries=5 * get_testing_process_count(),
seconds_between_retries=1)
def test_able_to_sync_jupyter():
user = USER_47
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(set_up_key_location(user))
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(clear_deployment_sync_data(user))
cluster = show_cluster(name=TEST_CLUSTER)
nodes = cluster.allocate_nodes()
stack.enter_context(cancel_on_exit(nodes))
node = nodes[0]
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
local_port = get_free_local_port()
jupyter = node.deploy_notebook(local_port=local_port)
stack.enter_context(cancel_on_exit(jupyter))
deployments = cluster.pull_deployments()
assert not deployments.jupyter_deployments
cluster.push_deployment(deployment=jupyter)
deployments = cluster.pull_deployments()
print(deployments)
assert len(deployments.jupyter_deployments) == 1
jupyter_2 = deployments.jupyter_deployments[0]
try:
assert jupyter.local_port != jupyter_2.local_port
check_local_http_connection(port=jupyter.local_port)
check_local_http_connection(port=jupyter_2.local_port)
finally:
jupyter_2.cancel_local()
def test_cancelled_jupyter_allocation_is_discarded_on_pull():
user = USER_48
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(set_up_key_location(user))
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
stack.enter_context(clear_deployment_sync_data(user))
cluster = show_cluster(name=TEST_CLUSTER)
nodes = cluster.allocate_nodes()
stack.enter_context(cancel_on_exit(nodes))
node = nodes[0]
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
local_port = get_free_local_port()
jupyter = node.deploy_notebook(local_port=local_port)
try:
deployments = cluster.pull_deployments()
assert not deployments.jupyter_deployments
cluster.push_deployment(deployment=jupyter)
jupyter.cancel()
jupyter = None
deployments = cluster.pull_deployments()
assert not deployments.jupyter_deployments
finally:
if jupyter is not None:
jupyter.cancel()
def test_node_tunnel_fall_back_when_local_port_taken():
"""Checks that a tunnel will fall back to a random port if local port is
taken."""
user = USER_53
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(set_up_key_location(user))
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
cluster = show_cluster(name=TEST_CLUSTER)
nodes = cluster.allocate_nodes(nodes=1,
cores=1,
memory_per_node=MiB(100),
walltime=Walltime(minutes=30))
stack.enter_context(cancel_on_exit(nodes))
node = nodes[0]
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
there = get_free_remote_port(node=node)
here = get_free_local_port()
tunnel_1 = node.tunnel(there=there, here=here)
stack.enter_context(close_tunnel_on_exit(tunnel_1))
assert tunnel_1.here == here
tunnel_2 = node.tunnel(there=there, here=here)
stack.enter_context(close_tunnel_on_exit(tunnel_2))
assert tunnel_2.here != here
def test_tunnel_single_hop():
user = USER_3
bindings = [
Binding("", get_free_local_port()),
Binding("127.0.0.1", PORT_1)
]
run_tunnel_test_for_bindings(user=user, bindings=bindings)
def test_tunnel_two_hops():
user = USER_59
bindings = [
Binding("", get_free_local_port()),
Binding("127.0.0.1", 22),
Binding("127.0.0.1", PORT_2)
]
run_tunnel_test_for_bindings(user=user, bindings=bindings)
def test_tunnel_multiple_hops():
user = USER_60
bindings = [
Binding("", get_free_local_port()),
Binding("c1", 22),
Binding("c2", 8022),
Binding("c3", 8023),
Binding("127.0.0.1", PORT_3)
]
run_tunnel_test_for_bindings(user=user, bindings=bindings)
def test_node_tunnel_fall_back_when_local_port_free_but_fails():
"""Checks that a tunnel will fall back to a random port if local port is
is initially free, but tunnel cannot be created anyway (e.g. another
process binds to it at the last moment)."""
user = USER_54
with ExitStack() as stack:
stack.enter_context(disable_pytest_stdin())
stack.enter_context(set_up_key_location(user))
stack.enter_context(reset_environment(user))
stack.enter_context(set_password(get_test_user_password(user)))
cluster = show_cluster(name=TEST_CLUSTER)
nodes = cluster.allocate_nodes(nodes=1,
cores=1,
memory_per_node=MiB(100),
walltime=Walltime(minutes=30))
stack.enter_context(cancel_on_exit(nodes))
node = nodes[0]
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
there = get_free_remote_port(node=node)
here = get_free_local_port()
real_build_tunnel = idact.detail.nodes.node_impl.build_tunnel
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tries = [0]
def fake_build_tunnel(*args, **kwargs) -> TunnelInternal:
tries[0] += 1
if tries[0] == 1:
raise RuntimeError("Fake failure.")
if tries[0] != 2:
assert False
return real_build_tunnel(*args, **kwargs)
try:
idact.detail.nodes.node_impl.build_tunnel = fake_build_tunnel
tunnel = node.tunnel(there=there, here=here)
stack.enter_context(close_tunnel_on_exit(tunnel))
assert tries[0] == 2
assert tunnel.here != here
finally:
idact.detail.nodes.node_impl.build_tunnel = real_build_tunnel
sock.close()
def run_tunnel_stress_test(stack: ExitStack, user: str, nodes: Nodes):
node = nodes[0]
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
there = get_free_remote_port(node=nodes[0])
here = get_free_local_port()
try:
for _ in range(5):
start_stress_cpu(user=user, timeout=10)
tunnel = node.tunnel(there=there, here=here)
stack.enter_context(close_tunnel_on_exit(tunnel))
finally:
stop_stress_cpu(user=user)
assert tunnel.here == here
assert tunnel.there == there
def run_tunnel_test(user: str, nodes: Nodes):
node = nodes[0]
nodes.wait(timeout=SLURM_WAIT_TIMEOUT)
assert nodes.running()
with ExitStack() as stack:
stack.enter_context(cancel_on_exit(nodes))
there = get_free_remote_port(node=nodes[0])
here = get_free_local_port()
server = start_dummy_server_thread(user=user, server_port=there)
stack.enter_context(join_on_exit(server))
tunnel = node.tunnel(there=there, here=here)
stack.enter_context(close_tunnel_on_exit(tunnel))
print(tunnel)
assert str(tunnel) == repr(tunnel)
assert tunnel.here == here
assert tunnel.there == there
def access_dummy_server():
return requests.get(
"http://127.0.0.1:{local_port}".format(local_port=here))
request = retry(access_dummy_server,
retries=5 * get_testing_process_count(),
seconds_between_retries=2)
assert "text/html" in request.headers['Content-type']
ssh_tunnel = node.tunnel_ssh()
stack.enter_context(close_tunnel_on_exit(ssh_tunnel))
assert str(ssh_tunnel) == repr(ssh_tunnel)
assert str(ssh_tunnel).startswith("ssh ")
assert user in str(ssh_tunnel)
assert str(ssh_tunnel.here) in str(ssh_tunnel)
assert ssh_tunnel.there == node.port
assert not nodes.running()
with pytest.raises(RuntimeError):
nodes.wait()
with pytest.raises(RuntimeError):
node.tunnel(there=there, here=here)