import pytest from artemis.plotting.plotting_backend import get_plotting_server_address from artemis.remote.utils import get_local_ips from artemis.remote.virtualenv import check_diff_local_remote_virtualenv, get_remote_installed_packages ip_address = get_plotting_server_address() is_local = ip_address in get_local_ips() @pytest.mark.skipif(ip_address="" or is_local, reason="No sense for local ip") def test_check_diff_local_remote_virtualenv(): # original_virtual_env_value = get_config_value(".artemisrc", ip_address, "python") # import ConfigParser # import os # # Config = ConfigParser.ConfigParser() # Config.read(os.path.expanduser("~/.artemisrc")) # Config.set(section=ip_address,option="python",value="~/virtualenvs/test_env/bin/python") # with open(os.path.expanduser("~/.artemisrc"), 'wb') as configfile: # Config.write(configfile) check_diff_local_remote_virtualenv(ip_address, auto_install=False, auto_upgrade=False, ignore_warnings=True) # Config.set(section=ip_address,option="python",value=original_virtual_env_value) # with open(os.path.expanduser("~/.artemisrc"), 'wb') as configfile: # Config.write(configfile)
import SocketServer import os import sys import time import signal from artemis.plotting.plotting_backend import get_plotting_server_address from artemis.remote.child_processes import check_ssh_connection, check_if_port_is_free, execute_command, ChildProcess, \ ParamikoPrintThread, Nanny from artemis.remote.utils import get_local_ips, get_socket, get_remote_artemis_path from pytest import raises ip_addresses = [get_plotting_server_address()] if ip_addresses[0] not in get_local_ips(): ip_addresses.append("127.0.0.1") def test_check_ssh_connections(): for ip_address in ip_addresses: if ip_address not in get_local_ips(): check_ssh_connection(ip_address) def test_check_if_port_is_free(): for ip_address in ip_addresses: if ip_address not in get_local_ips(): with raises(AssertionError): check_if_port_is_free(ip_address, 80) else: sock, port = get_socket(ip_address, 7005) # port is now not free: with raises(SocketServer.socket.error):
def set_up_plotting_server(): """ Sets up the plotting server. """ print("Setting up Plotting Server") # First we generate the system call that starts the server # TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine file_to_execute = os.path.join(os.path.dirname(__file__), 'plotting_server.py') file_to_execute = file_to_execute.replace(os.path.expanduser("~"),"~",1) plotting_server_address = get_plotting_server_address() if plotting_server_address == "": plotting_server_address = "127.0.0.1" if plotting_server_address in get_local_ips(): command = ["python", "-u", file_to_execute] else: check_config_file(plotting_server_address) # Make sure all things are set check_ssh_connection(plotting_server_address) # Make sure the SSH-connection works command =["export DISPLAY=:0.0;", "python","-u", file_to_execute] # TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set. # With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously. global _nanny _nanny = Nanny() cp = PythonChildProcess(ip_address=plotting_server_address, command=command, name="Plotting_Server",set_up_port_for_structured_back_communication=True) _nanny.register_child_process(cp,monitor_for_termination=False,monitor_if_stuck_timeout=None,) _nanny.execute_all_child_processes(blocking=False) back_comm_queue = cp.get_queue_from_cp() try: is_debug_mode = getattr(sys, 'gettrace', None) timeout = None if is_debug_mode() else 10 server_message = back_comm_queue.get(block=True,timeout=timeout) except Queue.Empty: print("The Plotting Server did not respond for 10 seconds. It probably crashed") sys.exit(1) try: port = int(server_message.dbplot_message) except ValueError: print("There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}".format(str_port)) sys.exit(0) # In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh: # Also, we have the ssh session open already, so why not reuse it. if plotting_server_address not in get_local_ips(): ssh_conn = cp.get_ssh_connection() # Needs to be in a thread since the call blocks forever. # Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example) # will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two # processes terminates, the ssh-tunnel is closed for the other process as well. t3 = threading.Thread(target = forward_tunnel, kwargs={"local_port":port, "remote_host":plotting_server_address, "remote_port":port,"ssh_conn":ssh_conn}) t3.setDaemon(True) t3.start() # Now attempt to connect to the plotting server server_address = ("localhost", port) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect(tuple(server_address)) except: raise # Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons: # 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded. # 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist. # (There shouldn't be any, but, you know, principle) global _to_subprocess_queue global _id_queue _to_subprocess_queue = Queue.Queue() _id_queue = Queue.Queue() t1 = threading.Thread(target=push_to_server, args=(_to_subprocess_queue, sock)) t1.setDaemon(True) t1.start() # if blocking: t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock)) t2.setDaemon(True) t2.start()
def set_up_plotting_server(): """ Sets up the plotting server. """ print("Setting up Plotting Server") # First we generate the system call that starts the server # TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine file_to_execute = os.path.join(os.path.dirname(__file__), 'plotting_server.py') file_to_execute = file_to_execute.replace(os.path.expanduser("~"), "~", 1) plotting_server_address = get_plotting_server_address() if plotting_server_address not in get_local_ips(): check_config_file( plotting_server_address) # Make sure all things are set check_ssh_connection( plotting_server_address) # Make sure the SSH-connection works command = ["export DISPLAY=:0.0;", "python", "-u", file_to_execute] # TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set. else: command = ["python", "-u", file_to_execute] # With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously. cp = ChildProcess(ip_address=plotting_server_address, command=command, name="Plotting_Server", take_care_of_deconstruct=True) stdin, stdout, stderr = cp.execute_child_process() t2 = ParamikoPrintThread(source_pipe=stderr, target_pipe=sys.stderr, prefix="Plotting Server: ") t2.setDaemon(True) t2.start() # The remote server is implemented such that it will scan available ports and choose one. This port then needs to be communicated to this client # Therefore we assume here, that the first line on stdout from the remote server is the port number for subsequent communication. Also, this call blocks until the parameter server # is ready to accept communication str_port = stdout.readline() try: port = int(str_port) except ValueError: print( "There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}" .format(str_port)) sys.exit(0) # All subsequent communication forwarded asynchronously t1 = ParamikoPrintThread(source_pipe=stdout, target_pipe=sys.stdout, prefix="Plotting Server: ") t1.setDaemon(True) t1.start() # In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh: # Also, we have the ssh session open already, so why not reuse it. if plotting_server_address not in get_local_ips(): ssh_conn = cp.get_ssh_connection() # Needs to be in a thread since the call blocks forever. # Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example) # will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two # processes terminates, the ssh-tunnel is closed for the other process as well. t3 = threading.Thread(target=forward_tunnel, kwargs={ "local_port": port, "remote_host": plotting_server_address, "remote_port": port, "ssh_conn": ssh_conn }) t3.setDaemon(True) t3.start() # Now attempt to connect to the plotting server server_address = ("localhost", port) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect(tuple(server_address)) except: raise # Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons: # 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded. # 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist. # (There shouldn't be any, but, you know, principle) global _to_subprocess_queue global _id_queue _to_subprocess_queue = Queue.Queue() _id_queue = Queue.Queue() t1 = threading.Thread(target=push_to_server, args=(_to_subprocess_queue, sock)) t1.setDaemon(True) t1.start() # if blocking: t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock)) t2.setDaemon(True) t2.start()