def check_if_port_is_free(ip_address, port):
'''
This checks if the remote server is able to accept requests at the given port.
:param connections: A list ["ip_address:port"] of strings
:param force_close: In case the port is in use, force close the program that is occupying it.
:return:
'''
try:
port = int(port)
except ValueError:
print ("Please provide a valid port for address %s. Received %s instead" %(ip_address, port))
raise
if ip_address in get_local_ips():
import socket
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((ip_address,port))
except:
raise
finally:
s.close()
else:
check_ssh_connection(ip_address)
ssh_connect = get_ssh_connection(ip_address=ip_address)
check_port_function = 'python -c "import socket; s=socket.socket(socket.AF_INET, socket.SOCK_STREAM);s.bind((\'%s\',%i));s.close()"'%(ip_address,port)
stdin , stdout, stderr = ssh_connect.exec_command(check_port_function)
err = stderr.read()
assert not err, "The remote address %s cannot allocate port %i. The following error was raised: \n %s" % (ip_address, port,err.strip().split("\n")[-1])
def __init__(self,
function,
ip_address,
set_up_port_for_structured_back_communication=True,
**kwargs):
if ip_address == 'localhost':
ip_address = "127.0.0.1"
pickled_function = pickle_dumps_without_main_refs(function)
encoded_pickled_function = base64.b64encode(pickled_function)
remote_run_script_path = inspect.getfile(remote_function_run_script)
if remote_run_script_path.endswith('pyc'):
remote_run_script_path = remote_run_script_path[:-1]
self.return_value_queue, return_port = listen_on_port(7000)
all_local_ips = get_local_ips()
return_address = "127.0.0.1" if ip_address in all_local_ips else all_local_ips[
-1]
command = [
sys.executable, '-u', remote_run_script_path,
encoded_pickled_function, return_address,
str(return_port)
]
super(RemotePythonProcess,
self).__init__(ip_address=ip_address,
command=command,
set_up_port_for_structured_back_communication=
set_up_port_for_structured_back_communication,
**kwargs)
def __init__(self,
command,
ip_address='localhost',
name=None,
take_care_of_deconstruct=False,
set_up_port_for_structured_back_communication=False):
'''
Creates a ChildProcess
:param ip_address: The command will be executed at this ip_address
:param command: the command to execute.
:param name: optional name. If not set, will be process_i, with i a global counter
:param take_care_of_deconstruct: If set to True, deconstruct() is registered at exit
:param port_for_structured_back_communication: Needs to be implemented according to the properties of the child process (see PythonChildProcess)
:return:
'''
if name is None:
name = "process_%s" % ChildProcess.counter
if ip_address == 'localhost':
ip_address = '127.0.0.1'
ChildProcess.counter += 1
self.name = name
self.ip_address = ip_address
self.local_process = self.ip_address in get_local_ips()
self.set_up_port_for_structured_back_communication = set_up_port_for_structured_back_communication
self.queue_from_cp = None
self.command = command
self.id = uuid.uuid4()
self.channel = None
self.cp_started = False
self.take_care_of_deconstruct = take_care_of_deconstruct
if self.take_care_of_deconstruct:
atexit.register(self.deconstruct)
def __init__(self, command, ip_address = 'localhost', name=None, take_care_of_deconstruct=False, set_up_port_for_structured_back_communication=False):
'''
Creates a ChildProcess
:param ip_address: The command will be executed at this ip_address
:param command: the command to execute.
:param name: optional name. If not set, will be process_i, with i a global counter
:param take_care_of_deconstruct: If set to True, deconstruct() is registered at exit
:param port_for_structured_back_communication: Needs to be implemented according to the properties of the child process (see PythonChildProcess)
:return:
'''
if name is None:
name = "process_%s"%ChildProcess.counter
if ip_address=='localhost':
ip_address='127.0.0.1'
ChildProcess.counter += 1
self.name = name
self.ip_address = ip_address
self.local_process = self.ip_address in get_local_ips()
self.set_up_port_for_structured_back_communication = set_up_port_for_structured_back_communication
self.queue_from_cp = None
self.command = command
self.id = uuid.uuid4()
self.channel = None
self.cp_started = False
self.take_care_of_deconstruct = take_care_of_deconstruct
if self.take_care_of_deconstruct:
atexit.register(self.deconstruct)
def __init__(self, ip_address, command, name=None, take_care_of_deconstruct=False):
'''
Creates a ChildProcess
:param ip_address: The command will be executed at this ip_address
:param command: the command to execute. If it is a python command and remote, will substitute the virtualenv
:param name: optional name. If not set, will be process_i, with i a clobal counter
:param take_care_of_deconstruct: If set to True, deconstruct() is registered at exit
:return:
'''
if name is None:
name = "process_%s"%ChildProcess.counter
ChildProcess.counter += 1
self.name = name
self.ip_address = ip_address
self.local_process = self.ip_address in get_local_ips()
# command = command.replace(unichr(34),unichr(39))
self.command = command
self.id = uuid.uuid4()
self.channel = None
self.cp_started = False
self.take_care_of_deconstruct = take_care_of_deconstruct
if self.take_care_of_deconstruct:
atexit.register(self.deconstruct)
def __init__(self,
function,
ip_address,
set_up_port_for_structured_back_communication=True,
slurm_kwargs={},
slurm_command="srun",
**kwargs):
'''
:param function:
:param ip_address:
:param set_up_port_for_structured_back_communication:
:param slurm_kwargs:
:param kwargs:
'''
assert ip_address in get_local_ips(
), "At the moment, we want you to start a slurm process only from localhost"
assert slurm_command in [
"srun"
], "At the moment, we only support 'srun' for execution of slurm"
super(SlurmPythonProcess,
self).__init__(function, ip_address,
set_up_port_for_structured_back_communication,
**kwargs)
self.slurm_kwargs = slurm_kwargs
def get_test_functions_path(ip_address):
if ip_address in get_local_ips():
return os.path.join(os.path.dirname(__file__),
"remote_test_functions.py")
else:
remote_path = get_remote_artemis_path(ip_address)
return os.path.join(remote_path, "remote/remote_test_functions.py")
def __init__(self, function, ip_address, set_up_port_for_structured_back_communication=True, slurm_kwargs={}, slurm_command="srun", **kwargs):
'''
:param function:
:param ip_address:
:param set_up_port_for_structured_back_communication:
:param slurm_kwargs:
:param kwargs:
'''
assert ip_address in get_local_ips(), "At the moment, we want you to start a slurm process only from localhost"
assert slurm_command in ["srun"], "At the moment, we only support 'srun' for execution of slurm"
super(SlurmPythonProcess,self).__init__(function, ip_address, set_up_port_for_structured_back_communication, **kwargs)
self.slurm_kwargs = slurm_kwargs
def test_check_if_port_is_free():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
with raises(AssertionError):
check_if_port_is_free(ip_address, 80)
else:
sock, port = get_socket(ip_address, 7005)
# port is now not free:
with raises(SocketServer.socket.error):
check_if_port_is_free(ip_address, port)
sock.close()
# now no error to be expected
check_if_port_is_free(ip_address, port)
def test_check_if_port_is_free():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
with raises(AssertionError):
check_if_port_is_free(ip_address,80)
else:
sock, port = get_socket(ip_address,7005)
# port is now not free:
with raises(SocketServer.socket.error):
check_if_port_is_free(ip_address, port)
sock.close()
# now no error to be expected
check_if_port_is_free(ip_address,port)
def __init__(self, function, ip_address, set_up_port_for_structured_back_communication=True, **kwargs):
if ip_address=='localhost':
ip_address="127.0.0.1"
pickled_function = pickle_dumps_without_main_refs(function)
encoded_pickled_function = base64.b64encode(pickled_function)
remote_run_script_path = inspect.getfile(remote_function_run_script)
if remote_run_script_path.endswith('pyc'):
remote_run_script_path = remote_run_script_path[:-1]
self.return_value_queue, return_port = listen_on_port(7000)
all_local_ips = get_local_ips()
return_address = "127.0.0.1" if ip_address in all_local_ips else all_local_ips[-1]
command = [sys.executable, '-u', remote_run_script_path, encoded_pickled_function, return_address, str(return_port)]
super(RemotePythonProcess, self).__init__(ip_address=ip_address, command=command, set_up_port_for_structured_back_communication=set_up_port_for_structured_back_communication, **kwargs)
def prepare_command(self,command):
'''
All the stuff that I need to prepare for the command to definitely work
:param command:
:return:
'''
if self.set_up_port_for_structured_back_communication:
self.queue_from_cp, port = listen_on_port()
if self.is_local():
address = "127.0.0.1"
else:
address = get_local_ips()[-1]
if self.is_local():
home_dir = os.path.expanduser("~")
else:
_,_,stdout,_ = self._run_command("echo $$; exec echo ~")
home_dir = stdout.read().strip()
if type(command) == list:
if self.set_up_port_for_structured_back_communication:
command.append("--port=%i"%port)
command.append("--address=%s"%address)
if not self.local_process:
command = [c.replace("python", self.get_extended_command(get_artemis_config_value(section=self.get_ip(), option="python", default_generator=lambda: sys.executable)), 1) if c.startswith("python") else c for c in command]
command = [s.replace("~",home_dir) for s in command]
command = " ".join([c for c in command])
else:
command = [c.strip("'") for c in command]
command = [c.replace("python", sys.executable, 1) if c.startswith("python") else c for c in command]
command = [s.replace("~",home_dir) for s in command]
elif isinstance(command, string_types) and command.startswith("python"):
if self.set_up_port_for_structured_back_communication:
command += " --port=%i "%port
command += "--address=%s"%address
if not self.local_process:
command = command.replace("python", self.get_extended_command(get_artemis_config_value(section=self.get_ip(), option="python",default_generator=lambda: sys.executable)), 1)
else:
command = command.replace("python", sys.executable)
command = command.replace("~",home_dir)
else:
raise NotImplementedError()
return command
def test_remote_graphics():
for ip_address in ip_addresses:
# command = 'export DISPLAY=:0.0; python -c "from matplotlib import pyplot as plt;import time; plt.figure();plt.show();time.sleep(10)"'
if ip_address not in get_local_ips():
command = [
"export DISPLAY=:0.0;", "python", "-u", "-c",
"from matplotlib import pyplot as plt;import time; plt.figure();plt.show();time.sleep(10)"
]
else:
command = [
"python", "-u", "-c",
"from matplotlib import pyplot as plt;import time; plt.figure();plt.show();time.sleep(10)"
]
cp = ChildProcess(ip_address=ip_address, command=command)
i, o, e = cp.execute_child_process()
time.sleep(5)
cp.deconstruct(message=signal.SIGTERM)
print(o.read())
print(e.read())
def test_kill_process_gently():
for ip_address in ip_addresses:
if ip_address in get_local_ips():
command = "python %s" % os.path.join(os.path.dirname(__file__),
"bogus_test_functions.py")
else:
remote_path = get_remote_artemis_path()
command = "python %s" % os.path.join(
remote_path, "remote/bogus_test_functions.py")
cp = ChildProcess(ip_address, command)
stdin, stdout, stderr = cp.execute_child_process()
# stdin , stdout, stderr = ssh_conn.exec_command(command)
pid = cp.get_pid()
print("Pid: %s" % pid)
#stdout
t1 = ParamikoPrintThread(source_pipe=stdout,
target_pipe=sys.stdout,
prefix="stdout: ")
t1.start()
# stderr
t2 = ParamikoPrintThread(source_pipe=stderr,
target_pipe=sys.stderr,
prefix="stderr: ")
t2.start()
print("Waiting 4 seconds")
time.sleep(4)
print("Killing Process %s:" % (pid))
cp.kill()
print("Waiting 4 seconds")
time.sleep(4)
if cp.is_alive():
print("Process still alive")
else:
print("Process dead")
print("Terminating")
def execute_command(ip_address, command, blocking=True):
'''
This method spawns a child-process (either locally or remote, depending on the ip_address). Then it executes the given command and handles communication.
If blocking is True, then this call does not return before the child process returns. It then prints stdout to console, followed by stderr.
If blocking is False, then this call returns immediately and asynchronously forwards stdout and stderr to the console in separate threads.
If ip_address is local, then the command will be split using shlex.split() and must be formatted accordingly. The subprocess call is executed with shell=False
for all the right reasons.
:param ip_address:
:param command: String. command to execute
:param blocking:
:return:
'''
#TODO: Reformat this to work with the Nanny. Let it manage the communication
if ip_address in get_local_ips():
sub = subprocess.Popen(shlex.split(command),stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
stdout_pipe = sub.stdout
stderr_pipe = sub.stderr
else:
ssh_conn = get_ssh_connection(ip_address)
transport = ssh_conn.get_transport()
channel = transport.open_session()
channel.exec_command(command)#
bufsize=-1
stdout_pipe = channel.makefile('r', bufsize)
stderr_pipe = channel.makefile_stderr('r',bufsize)
#stdout
t1 = ParamikoPrintThread(source_pipe=stdout_pipe, target_pipe=sys.stdout)
t1.start()
# stderr
t2 = ParamikoPrintThread(source_pipe=stderr_pipe, target_pipe=sys.stderr)
t2.start()
if blocking:
t1.join()
t2.join()
def prepare_command(self, command):
'''
All the stuff that I need to prepare for the command to definitely work
:param command:
:return:
'''
if self.set_up_port_for_structured_back_communication:
self.queue_from_cp, port = listen_on_port()
if self.is_local():
address = "127.0.0.1"
else:
address = get_local_ips()[-1]
if self.is_local():
home_dir = os.path.expanduser("~")
else:
_, _, stdout, _ = self._run_command("echo $$; exec echo ~")
home_dir = stdout.read().strip()
if type(command) == list:
if self.set_up_port_for_structured_back_communication:
command.append("--port=%i" % port)
command.append("--address=%s" % address)
if not self.local_process:
command = [
c.replace(
"python",
self.get_extended_command(
get_artemis_config_value(
section=self.get_ip(),
option="python",
default_generator=lambda: sys.executable)), 1)
if c.startswith("python") else c for c in command
]
command = [s.replace("~", home_dir) for s in command]
command = " ".join([c for c in command])
else:
command = [c.strip("'") for c in command]
command = [
c.replace("python", sys.executable, 1)
if c.startswith("python") else c for c in command
]
command = [s.replace("~", home_dir) for s in command]
elif isinstance(command,
string_types) and command.startswith("python"):
if self.set_up_port_for_structured_back_communication:
command += " --port=%i " % port
command += "--address=%s" % address
if not self.local_process:
command = command.replace(
"python",
self.get_extended_command(
get_artemis_config_value(
section=self.get_ip(),
option="python",
default_generator=lambda: sys.executable)), 1)
else:
command = command.replace("python", sys.executable)
command = command.replace("~", home_dir)
else:
raise NotImplementedError()
return command
import SocketServer
from artemis.config import get_artemis_config_value
from pytest import raises
from artemis.remote.utils import get_local_ips, check_ssh_connection, check_if_port_is_free, get_socket
ip_addresses = [
get_artemis_config_value(section="tests",
option="remote_server",
default_generator=lambda: "127.0.0.1")
]
# ip_addresses=["127.0.0.1"]
if ip_addresses[0] not in get_local_ips():
ip_addresses.append("127.0.0.1")
def test_check_ssh_connections():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
check_ssh_connection(ip_address)
def test_check_if_port_is_free():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
with raises(AssertionError):
check_if_port_is_free(ip_address, 80)
else:
sock, port = get_socket(ip_address, 7005)
# port is now not free:
def get_test_functions_path(ip_address):
if ip_address in get_local_ips():
return os.path.join(os.path.dirname(__file__), "remote_test_functions.py")
else:
remote_path = get_remote_artemis_path(ip_address)
return os.path.join(remote_path, "remote/remote_test_functions.py")
import signal
import sys
import time
import os
from artemis.config import get_artemis_config_value
from artemis.remote.child_processes import PythonChildProcess, RemotePythonProcess
from artemis.remote.remote_execution import ParamikoPrintThread, execute_command
from artemis.remote.utils import get_local_ips, get_remote_artemis_path
from functools import partial
ip_addresses = [get_artemis_config_value(section="tests", option="remote_server",default_generator=lambda: "127.0.0.1")]
# ip_addresses=["127.0.0.1"]
if ip_addresses[0] not in get_local_ips():
ip_addresses.append("127.0.0.1")
def get_test_functions_path(ip_address):
if ip_address in get_local_ips():
return os.path.join(os.path.dirname(__file__), "remote_test_functions.py")
else:
remote_path = get_remote_artemis_path(ip_address)
return os.path.join(remote_path, "remote/remote_test_functions.py")
def test_simple_pcp():
for ip_address in ip_addresses:
command = "python %s --callback=%s"%(get_test_functions_path(ip_address),"success_function")
pyc = PythonChildProcess(ip_address=ip_address,command=command)
(stdin, stdout, stderr) = pyc.execute_child_process()
stderr_out =stderr.readlines()
def test_check_ssh_connections():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
check_ssh_connection(ip_address)
def test_check_ssh_connections():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
check_ssh_connection(ip_address)
import pytest
from artemis.plotting.plotting_backend import get_plotting_server_address
from artemis.remote.utils import get_local_ips
from artemis.remote.virtualenv import check_diff_local_remote_virtualenv, get_remote_installed_packages
ip_address = get_plotting_server_address()
is_local = ip_address in get_local_ips()
@pytest.mark.skipif(ip_address="" or is_local, reason="No sense for local ip")
def test_check_diff_local_remote_virtualenv():
# original_virtual_env_value = get_config_value(".artemisrc", ip_address, "python")
# import ConfigParser
# import os
#
# Config = ConfigParser.ConfigParser()
# Config.read(os.path.expanduser("~/.artemisrc"))
# Config.set(section=ip_address,option="python",value="~/virtualenvs/test_env/bin/python")
# with open(os.path.expanduser("~/.artemisrc"), 'wb') as configfile:
# Config.write(configfile)
check_diff_local_remote_virtualenv(ip_address,
auto_install=False,
auto_upgrade=False,
ignore_warnings=True)
# Config.set(section=ip_address,option="python",value=original_virtual_env_value)
# with open(os.path.expanduser("~/.artemisrc"), 'wb') as configfile:
# Config.write(configfile)
def set_up_plotting_server():
"""
Sets up the plotting server.
"""
print("Setting up Plotting Server")
# First we generate the system call that starts the server
# TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine
file_to_execute = os.path.join(os.path.dirname(__file__), 'plotting_server.py')
file_to_execute = file_to_execute.replace(os.path.expanduser("~"),"~",1)
plotting_server_address = get_plotting_server_address()
if plotting_server_address == "":
plotting_server_address = "127.0.0.1"
if plotting_server_address in get_local_ips():
command = ["python", "-u", file_to_execute]
else:
check_config_file(plotting_server_address) # Make sure all things are set
check_ssh_connection(plotting_server_address) # Make sure the SSH-connection works
command =["export DISPLAY=:0.0;", "python","-u", file_to_execute]
# TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set.
# With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously.
global _nanny
_nanny = Nanny()
cp = PythonChildProcess(ip_address=plotting_server_address, command=command, name="Plotting_Server",set_up_port_for_structured_back_communication=True)
_nanny.register_child_process(cp,monitor_for_termination=False,monitor_if_stuck_timeout=None,)
_nanny.execute_all_child_processes(blocking=False)
back_comm_queue = cp.get_queue_from_cp()
try:
is_debug_mode = getattr(sys, 'gettrace', None)
timeout = None if is_debug_mode() else 10
server_message = back_comm_queue.get(block=True,timeout=timeout)
except Queue.Empty:
print("The Plotting Server did not respond for 10 seconds. It probably crashed")
sys.exit(1)
try:
port = int(server_message.dbplot_message)
except ValueError:
print("There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}".format(str_port))
sys.exit(0)
# In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh:
# Also, we have the ssh session open already, so why not reuse it.
if plotting_server_address not in get_local_ips():
ssh_conn = cp.get_ssh_connection()
# Needs to be in a thread since the call blocks forever.
# Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example)
# will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two
# processes terminates, the ssh-tunnel is closed for the other process as well.
t3 = threading.Thread(target = forward_tunnel, kwargs={"local_port":port, "remote_host":plotting_server_address, "remote_port":port,"ssh_conn":ssh_conn})
t3.setDaemon(True)
t3.start()
# Now attempt to connect to the plotting server
server_address = ("localhost", port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(tuple(server_address))
except:
raise
# Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons:
# 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded.
# 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist.
# (There shouldn't be any, but, you know, principle)
global _to_subprocess_queue
global _id_queue
_to_subprocess_queue = Queue.Queue()
_id_queue = Queue.Queue()
t1 = threading.Thread(target=push_to_server, args=(_to_subprocess_queue, sock))
t1.setDaemon(True)
t1.start()
# if blocking:
t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock))
t2.setDaemon(True)
t2.start()
def set_up_plotting_server():
"""
Sets up the plotting server.
"""
print("Setting up Plotting Server")
# First we generate the system call that starts the server
# TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine
file_to_execute = os.path.join(os.path.dirname(__file__), 'plotting_server.py')
file_to_execute = file_to_execute.replace(os.path.expanduser("~"),"~",1)
plotting_server_address = get_plotting_server_address()
if plotting_server_address == "":
plotting_server_address = "127.0.0.1"
if plotting_server_address in get_local_ips():
command = ["python", "-u", file_to_execute]
else:
check_config_file(plotting_server_address) # Make sure all things are set
check_ssh_connection(plotting_server_address) # Make sure the SSH-connection works
command =["export DISPLAY=:0.0;", "python","-u", file_to_execute]
# TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set.
# With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously.
global _nanny
_nanny = Nanny()
cp = PythonChildProcess(ip_address=plotting_server_address, command=command, name="Plotting_Server",set_up_port_for_structured_back_communication=True)
_nanny.register_child_process(cp,monitor_for_termination=False,monitor_if_stuck_timeout=None,)
_nanny.execute_all_child_processes(blocking=False)
back_comm_queue = cp.get_queue_from_cp()
try:
is_debug_mode = getattr(sys, 'gettrace', None)
timeout = None if is_debug_mode() else 10
server_message = back_comm_queue.get(block=True,timeout=timeout)
except Queue.Empty:
print("The Plotting Server did not respond for 10 seconds. It probably crashed")
sys.exit(1)
try:
port = int(server_message.dbplot_message)
except ValueError:
print("There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}".format(server_message.dbplot_message))
sys.exit(0)
# In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh:
# Also, we have the ssh session open already, so why not reuse it.
if plotting_server_address not in get_local_ips():
ssh_conn = cp.get_ssh_connection()
# Needs to be in a thread since the call blocks forever.
# Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example)
# will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two
# processes terminates, the ssh-tunnel is closed for the other process as well.
t3 = threading.Thread(target = forward_tunnel, kwargs={"local_port":port, "remote_host":plotting_server_address, "remote_port":port,"ssh_conn":ssh_conn})
t3.setDaemon(True)
t3.start()
# Now attempt to connect to the plotting server
server_address = ("localhost", port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(tuple(server_address))
except:
raise
# Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons:
# 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded.
# 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist.
# (There shouldn't be any, but, you know, principle)
global _to_subprocess_queue
global _id_queue
_to_subprocess_queue = Queue.Queue()
_id_queue = Queue.Queue()
t1 = threading.Thread(target=push_to_server, args=(_to_subprocess_queue, sock))
t1.setDaemon(True)
t1.start()
# if blocking:
t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock))
t2.setDaemon(True)
t2.start()
import os
import pytest
import shutil
from artemis.config import get_artemis_config_value
from artemis.fileman.config_files import get_config_value
from artemis.fileman.local_dir import get_artemis_data_path
from artemis.plotting.matplotlib_backend import get_plotting_server_address
from artemis.remote.file_system import rsync, simple_rsync, check_config_file
from artemis.remote.utils import get_local_ips
ip_address = get_plotting_server_address()
is_local = ip_address in get_local_ips()
@pytest.mark.skipif(is_local, reason ="No sense for local ip")
def test_rsync():
options = ["-r"]
username = get_artemis_config_value(section=ip_address, option="username")
from_path = get_artemis_data_path(relative_path="tmp/tests/", make_local_dir=True)
with open(os.path.join(from_path, "test1"), "wb"):
pass
with open(os.path.join(from_path, "test2"), "wb"):
pass
to_path = "%s@%s:/home/%s/temp/"%(username, ip_address, username)
assert rsync(options, from_path, to_path)
shutil.rmtree(from_path)
def set_up_plotting_server():
"""
Sets up the plotting server.
"""
print("Setting up Plotting Server")
# First we generate the system call that starts the server
# TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine
file_to_execute = os.path.join(os.path.dirname(__file__),
'plotting_server.py')
file_to_execute = file_to_execute.replace(os.path.expanduser("~"), "~", 1)
plotting_server_address = get_plotting_server_address()
if plotting_server_address not in get_local_ips():
check_config_file(
plotting_server_address) # Make sure all things are set
check_ssh_connection(
plotting_server_address) # Make sure the SSH-connection works
command = ["export DISPLAY=:0.0;", "python", "-u", file_to_execute]
# TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set.
else:
command = ["python", "-u", file_to_execute]
# With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously.
cp = ChildProcess(ip_address=plotting_server_address,
command=command,
name="Plotting_Server",
take_care_of_deconstruct=True)
stdin, stdout, stderr = cp.execute_child_process()
t2 = ParamikoPrintThread(source_pipe=stderr,
target_pipe=sys.stderr,
prefix="Plotting Server: ")
t2.setDaemon(True)
t2.start()
# The remote server is implemented such that it will scan available ports and choose one. This port then needs to be communicated to this client
# Therefore we assume here, that the first line on stdout from the remote server is the port number for subsequent communication. Also, this call blocks until the parameter server
# is ready to accept communication
str_port = stdout.readline()
try:
port = int(str_port)
except ValueError:
print(
"There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}"
.format(str_port))
sys.exit(0)
# All subsequent communication forwarded asynchronously
t1 = ParamikoPrintThread(source_pipe=stdout,
target_pipe=sys.stdout,
prefix="Plotting Server: ")
t1.setDaemon(True)
t1.start()
# In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh:
# Also, we have the ssh session open already, so why not reuse it.
if plotting_server_address not in get_local_ips():
ssh_conn = cp.get_ssh_connection()
# Needs to be in a thread since the call blocks forever.
# Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example)
# will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two
# processes terminates, the ssh-tunnel is closed for the other process as well.
t3 = threading.Thread(target=forward_tunnel,
kwargs={
"local_port": port,
"remote_host": plotting_server_address,
"remote_port": port,
"ssh_conn": ssh_conn
})
t3.setDaemon(True)
t3.start()
# Now attempt to connect to the plotting server
server_address = ("localhost", port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(tuple(server_address))
except:
raise
# Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons:
# 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded.
# 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist.
# (There shouldn't be any, but, you know, principle)
global _to_subprocess_queue
global _id_queue
_to_subprocess_queue = Queue.Queue()
_id_queue = Queue.Queue()
t1 = threading.Thread(target=push_to_server,
args=(_to_subprocess_queue, sock))
t1.setDaemon(True)
t1.start()
# if blocking:
t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock))
t2.setDaemon(True)
t2.start()
