def test_remote_child_function():
for ip_address in ip_addresses:
pyc = PythonChildProcess(ip_address=ip_address,command=partial(remote_test_func, a=1, b=2))
(stdin, stdout, stderr) = pyc.execute_child_process()
stderr_out =stderr.readlines()
stderr_out = [line.strip() for line in stderr_out if "pydev debugger" not in line]
stdout_out = stdout.readlines()
def test_remote_child_function():
for ip_address in ip_addresses:
pyc = PythonChildProcess(ip_address=ip_address,command=partial(remote_test_func, a=1, b=2))
(stdin, stdout, stderr) = pyc.execute_child_process()
stderr_out =stderr.readlines()
stderr_out = [line.strip() for line in stderr_out if "pydev debugger" not in line]
stdout_out = stdout.readlines()
def test_simple_process():
for ip_address in ip_addresses:
nanny = Nanny()
command = "python %s --callback=%s"%(get_test_functions_path(ip_address),"success_function")
pyc = PythonChildProcess(name="Process", ip_address=ip_address,command=command)
nanny.register_child_process(pyc,)
with captured_output() as (out,err):
nanny.execute_all_child_processes()
assert "%s: Success"%(pyc.get_name()) == out.getvalue().strip()
def test_simple_pcp_list():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=success_function"]
pyc = PythonChildProcess(ip_address=ip_address,command=command)
(stdin, stdout, stderr) = pyc.execute_child_process()
stderr_out = stderr.readlines()
stderr_out = [line.strip() for line in stderr_out if "pydev debugger" not in line]
stdout_out = stdout.readlines()
assert len("".join(stderr_out)) == 0, "Stderr not empty. Received: %s"%stderr_out
assert len(stdout_out) == 1 and stdout_out[0].strip() == "Success", "Stdout not as expected. Received: %s"%stdout_out
def test_simple_pcp_list():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=success_function"]
pyc = PythonChildProcess(ip_address=ip_address,command=command)
(stdin, stdout, stderr) = pyc.execute_child_process()
stderr_out = stderr.readlines()
stderr_out = [line.strip() for line in stderr_out if "pydev debugger" not in line]
stdout_out = stdout.readlines()
assert len("".join(stderr_out)) == 0, "Stderr not empty. Received: %s"%stderr_out
assert len(stdout_out) == 1 and stdout_out[0].strip() == "Success", "Stdout not as expected. Received: %s"%stdout_out
def test_remote_graphics():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=remote_graphics"]
cp = PythonChildProcess(ip_address=ip_address,command=command,take_care_of_deconstruct=True)
i, stdout, stderr = cp.execute_child_process()
time.sleep(1)
stderr_out = stderr.readlines()
stdout_out = stdout.readlines()
# assert len(stderr_out) == 0, "Stderr not empty. Received: %s"%stderr_out
assert stdout_out[-1].strip() == "Success", "Stdout not as expected. Received: %s"%stdout_out
def test_remote_graphics():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=remote_graphics"]
cp = PythonChildProcess(ip_address=ip_address,command=command,take_care_of_deconstruct=True)
i, stdout, stderr = cp.execute_child_process()
time.sleep(1)
stderr_out = stderr.readlines()
stdout_out = stdout.readlines()
# assert len(stderr_out) == 0, "Stderr not empty. Received: %s"%stderr_out
assert stdout_out[-1].strip() == "Success", "Stdout not as expected. Received: %s"%stdout_out
def test_simple_process():
for ip_address in ip_addresses:
nanny = Nanny()
command = "python %s --callback=%s" % (
get_test_functions_path(ip_address), "success_function")
pyc = PythonChildProcess(name="Process",
ip_address=ip_address,
command=command)
nanny.register_child_process(pyc, )
with captured_output() as (out, err):
nanny.execute_all_child_processes()
assert "%s: Success" % (pyc.get_name()) == out.getvalue().strip()
def test_kill_process_gently():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=sleep_function"]
cp = PythonChildProcess(ip_address, command)
stdin , stdout, stderr = cp.execute_child_process()
time.sleep(1)
cp.kill()
time.sleep(1)
stdout_out = stdout.readlines()
assert stdout_out[-1].strip() == "Interrupted"
assert not cp.is_alive(), "Process is still alive, killing it did not work"
def test_several_simple_processes(N):
for ip_address in ip_addresses:
nanny = Nanny()
command = "python %s --callback=%s"%(get_test_functions_path(ip_address),"success_function")
for i in range(N):
pyc = PythonChildProcess(name="Process%i"%i, ip_address=ip_address,command=command)
nanny.register_child_process(pyc,)
with captured_output() as (out,err):
nanny.execute_all_child_processes()
out_value = out.getvalue().strip()
for pyc in nanny.managed_child_processes.values():
assert "%s: Success"%(pyc.get_name()) in out_value
def test_iter_print():
for ip_address in ip_addresses:
nanny = Nanny()
command = ["python","-u", get_test_functions_path(ip_address), "--callback=iter_print"]
pyc = PythonChildProcess(name="P1",ip_address=ip_address,command=command)
nanny.register_child_process(pyc)
with captured_output() as (out, err):
nanny.execute_all_child_processes(time_out=1)
if pyc.is_local():
assert str(out.getvalue().strip()) == "\n".join(["P1: %i"%i for i in [0,2,4,6,8]])
assert str(err.getvalue().strip()) == "\n".join(["P1: %i"%i for i in [1,3,5,7,9]])
else:
assert "\r\n".join(["P1: %i" % i for i in range(10)]) == str(out.getvalue().strip())
def test_several_simple_processes(N):
for ip_address in ip_addresses:
nanny = Nanny()
command = "python %s --callback=%s" % (
get_test_functions_path(ip_address), "success_function")
for i in range(N):
pyc = PythonChildProcess(name="Process%i" % i,
ip_address=ip_address,
command=command)
nanny.register_child_process(pyc, )
with captured_output() as (out, err):
nanny.execute_all_child_processes()
out_value = out.getvalue().strip()
for pyc in nanny.managed_child_processes.values():
assert "%s: Success" % (pyc.get_name()) in out_value
def test_kill_process_strongly():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=hanging_sleep_function"]
cp = PythonChildProcess(ip_address, command)
stdin , stdout, stderr = cp.execute_child_process()
time.sleep(1)
cp.kill()
time.sleep(1)
assert cp.is_alive(), "Process terminated too soon. Check remote_test_functions.py implementation!"
cp.kill(signal.SIGTERM)
time.sleep(1)
assert not cp.is_alive(), "Process is still alive, killing it did not work"
def test_process_termination():
for ip_address in ip_addresses:
nanny = Nanny()
command = "python %s --callback=%s" % (
get_test_functions_path(ip_address), "count_low")
pyc = PythonChildProcess(name="Process1",
ip_address=ip_address,
command=command)
nanny.register_child_process(pyc, )
command = "python %s --callback=%s" % (
get_test_functions_path(ip_address), "count_high")
pyc = PythonChildProcess(name="Process2",
ip_address=ip_address,
command=command)
nanny.register_child_process(pyc, )
with captured_output() as (out, err):
nanny.execute_all_child_processes(time_out=1)
check_text = "Child Process Process2 at %s did not terminate 1 seconds after the first process in cluster terminated. Terminating now." % ip_address
assert check_text in out.getvalue() or check_text in err.getvalue()
def test_kill_process_strongly():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=hanging_sleep_function"]
cp = PythonChildProcess(ip_address, command)
stdin , stdout, stderr = cp.execute_child_process()
time.sleep(1)
cp.kill()
time.sleep(1)
assert cp.is_alive(), "Process terminated too soon. Check remote_test_functions.py implementation!"
cp.kill(signal.SIGKILL)
time.sleep(1)
assert not cp.is_alive(), "Process is still alive, killing it did not work"
def test_output_monitor():
for ip_address in ip_addresses:
nanny = Nanny()
command = "python %s --callback=%s" % (
get_test_functions_path(ip_address), "short_sleep")
pyc = PythonChildProcess(name="Process1",
ip_address=ip_address,
command=command)
nanny.register_child_process(pyc, monitor_if_stuck_timeout=5)
command = "python %s --callback=%s" % (
get_test_functions_path(ip_address), "count_high")
pyc = PythonChildProcess(name="Process2",
ip_address=ip_address,
command=command)
nanny.register_child_process(pyc, monitor_if_stuck_timeout=3)
with captured_output() as (out, err):
nanny.execute_all_child_processes(time_out=1)
check_text1 = "Timeout occurred after 0.1 min, process Process1 stuck"
check_text = "Child Process Process2 at %s did not terminate 1 seconds after the first process in cluster terminated. Terminating now." % ip_address
assert check_text in out.getvalue() or check_text in err.getvalue()
assert check_text1 in out.getvalue() or check_text1 in err.getvalue()
def test_iter_print():
for ip_address in ip_addresses:
nanny = Nanny()
command = [
"python", "-u",
get_test_functions_path(ip_address), "--callback=iter_print"
]
pyc = PythonChildProcess(name="P1",
ip_address=ip_address,
command=command)
nanny.register_child_process(pyc)
with captured_output() as (out, err):
nanny.execute_all_child_processes(time_out=1)
if pyc.is_local():
assert str(out.getvalue().strip()) == "\n".join(
["P1: %i" % i for i in [0, 2, 4, 6, 8]])
assert str(err.getvalue().strip()) == "\n".join(
["P1: %i" % i for i in [1, 3, 5, 7, 9]])
else:
assert "\r\n".join(["P1: %i" % i for i in range(10)
]) == str(out.getvalue().strip())
def test_kill_process_gently():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=sleep_function"]
cp = PythonChildProcess(ip_address, command)
stdin , stdout, stderr = cp.execute_child_process()
time.sleep(1)
cp.kill()
time.sleep(1)
stdout_out = stdout.readlines()
assert stdout_out[-1].strip() == "Interrupted"
assert not cp.is_alive(), "Process is still alive, killing it did not work"
def test_interrupt_process_gently():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=count_high"]
cp = PythonChildProcess(ip_address, command)
stdin , stdout, stderr = cp.execute_child_process()
time.sleep(5)
cp.kill()
time.sleep(1)
stdout_out = stdout.readlines()
stderr_out = stderr.readlines()
if cp.is_local():
assert stderr_out[-1].strip() == "KeyboardInterrupt"
else:
assert stdout_out[-1].strip() == "KeyboardInterrupt"
assert not cp.is_alive(), "Process is still alive, killing it did not work"
def test_interrupt_process_gently():
for ip_address in ip_addresses:
command = ["python", get_test_functions_path(ip_address), "--callback=count_high"]
cp = PythonChildProcess(ip_address, command)
stdin , stdout, stderr = cp.execute_child_process()
time.sleep(5)
cp.kill()
time.sleep(1)
stdout_out = stdout.readlines()
stderr_out = stderr.readlines()
if cp.is_local():
assert stderr_out[-1].strip() == "KeyboardInterrupt"
else:
assert stdout_out[-1].strip() == "KeyboardInterrupt"
assert not cp.is_alive(), "Process is still alive, killing it did not work"
def set_up_plotting_server():
"""
Sets up the plotting server.
"""
print("Setting up Plotting Server")
# First we generate the system call that starts the server
# TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine
file_to_execute = os.path.join(os.path.dirname(__file__), 'plotting_server.py')
file_to_execute = file_to_execute.replace(os.path.expanduser("~"),"~",1)
plotting_server_address = get_plotting_server_address()
if plotting_server_address == "":
plotting_server_address = "127.0.0.1"
if plotting_server_address in get_local_ips():
command = ["python", "-u", file_to_execute]
else:
check_config_file(plotting_server_address) # Make sure all things are set
check_ssh_connection(plotting_server_address) # Make sure the SSH-connection works
command =["export DISPLAY=:0.0;", "python","-u", file_to_execute]
# TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set.
# With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously.
global _nanny
_nanny = Nanny()
cp = PythonChildProcess(ip_address=plotting_server_address, command=command, name="Plotting_Server",set_up_port_for_structured_back_communication=True)
_nanny.register_child_process(cp,monitor_for_termination=False,monitor_if_stuck_timeout=None,)
_nanny.execute_all_child_processes(blocking=False)
back_comm_queue = cp.get_queue_from_cp()
try:
is_debug_mode = getattr(sys, 'gettrace', None)
timeout = None if is_debug_mode() else 10
server_message = back_comm_queue.get(block=True,timeout=timeout)
except Queue.Empty:
print("The Plotting Server did not respond for 10 seconds. It probably crashed")
sys.exit(1)
try:
port = int(server_message.dbplot_message)
except ValueError:
print("There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}".format(str_port))
sys.exit(0)
# In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh:
# Also, we have the ssh session open already, so why not reuse it.
if plotting_server_address not in get_local_ips():
ssh_conn = cp.get_ssh_connection()
# Needs to be in a thread since the call blocks forever.
# Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example)
# will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two
# processes terminates, the ssh-tunnel is closed for the other process as well.
t3 = threading.Thread(target = forward_tunnel, kwargs={"local_port":port, "remote_host":plotting_server_address, "remote_port":port,"ssh_conn":ssh_conn})
t3.setDaemon(True)
t3.start()
# Now attempt to connect to the plotting server
server_address = ("localhost", port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(tuple(server_address))
except:
raise
# Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons:
# 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded.
# 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist.
# (There shouldn't be any, but, you know, principle)
global _to_subprocess_queue
global _id_queue
_to_subprocess_queue = Queue.Queue()
_id_queue = Queue.Queue()
t1 = threading.Thread(target=push_to_server, args=(_to_subprocess_queue, sock))
t1.setDaemon(True)
t1.start()
# if blocking:
t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock))
t2.setDaemon(True)
t2.start()
def set_up_plotting_server():
"""
Sets up the plotting server.
"""
print("Setting up Plotting Server")
# First we generate the system call that starts the server
# TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine
file_to_execute = os.path.join(os.path.dirname(__file__), 'plotting_server.py')
file_to_execute = file_to_execute.replace(os.path.expanduser("~"),"~",1)
plotting_server_address = get_plotting_server_address()
if plotting_server_address == "":
plotting_server_address = "127.0.0.1"
if plotting_server_address in get_local_ips():
command = ["python", "-u", file_to_execute]
else:
check_config_file(plotting_server_address) # Make sure all things are set
check_ssh_connection(plotting_server_address) # Make sure the SSH-connection works
command =["export DISPLAY=:0.0;", "python","-u", file_to_execute]
# TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set.
# With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously.
global _nanny
_nanny = Nanny()
cp = PythonChildProcess(ip_address=plotting_server_address, command=command, name="Plotting_Server",set_up_port_for_structured_back_communication=True)
_nanny.register_child_process(cp,monitor_for_termination=False,monitor_if_stuck_timeout=None,)
_nanny.execute_all_child_processes(blocking=False)
back_comm_queue = cp.get_queue_from_cp()
try:
is_debug_mode = getattr(sys, 'gettrace', None)
timeout = None if is_debug_mode() else 10
server_message = back_comm_queue.get(block=True,timeout=timeout)
except Queue.Empty:
print("The Plotting Server did not respond for 10 seconds. It probably crashed")
sys.exit(1)
try:
port = int(server_message.dbplot_message)
except ValueError:
print("There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}".format(server_message.dbplot_message))
sys.exit(0)
# In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh:
# Also, we have the ssh session open already, so why not reuse it.
if plotting_server_address not in get_local_ips():
ssh_conn = cp.get_ssh_connection()
# Needs to be in a thread since the call blocks forever.
# Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example)
# will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two
# processes terminates, the ssh-tunnel is closed for the other process as well.
t3 = threading.Thread(target = forward_tunnel, kwargs={"local_port":port, "remote_host":plotting_server_address, "remote_port":port,"ssh_conn":ssh_conn})
t3.setDaemon(True)
t3.start()
# Now attempt to connect to the plotting server
server_address = ("localhost", port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(tuple(server_address))
except:
raise
# Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons:
# 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded.
# 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist.
# (There shouldn't be any, but, you know, principle)
global _to_subprocess_queue
global _id_queue
_to_subprocess_queue = Queue.Queue()
_id_queue = Queue.Queue()
t1 = threading.Thread(target=push_to_server, args=(_to_subprocess_queue, sock))
t1.setDaemon(True)
t1.start()
# if blocking:
t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock))
t2.setDaemon(True)
t2.start()
