import pytest
from artemis.plotting.plotting_backend import get_plotting_server_address
from artemis.remote.utils import get_local_ips
from artemis.remote.virtualenv import check_diff_local_remote_virtualenv, get_remote_installed_packages
ip_address = get_plotting_server_address()
is_local = ip_address in get_local_ips()
@pytest.mark.skipif(ip_address="" or is_local, reason="No sense for local ip")
def test_check_diff_local_remote_virtualenv():
# original_virtual_env_value = get_config_value(".artemisrc", ip_address, "python")
# import ConfigParser
# import os
#
# Config = ConfigParser.ConfigParser()
# Config.read(os.path.expanduser("~/.artemisrc"))
# Config.set(section=ip_address,option="python",value="~/virtualenvs/test_env/bin/python")
# with open(os.path.expanduser("~/.artemisrc"), 'wb') as configfile:
# Config.write(configfile)
check_diff_local_remote_virtualenv(ip_address,
auto_install=False,
auto_upgrade=False,
ignore_warnings=True)
# Config.set(section=ip_address,option="python",value=original_virtual_env_value)
# with open(os.path.expanduser("~/.artemisrc"), 'wb') as configfile:
# Config.write(configfile)
import SocketServer
import os
import sys
import time
import signal
from artemis.plotting.plotting_backend import get_plotting_server_address
from artemis.remote.child_processes import check_ssh_connection, check_if_port_is_free, execute_command, ChildProcess, \
ParamikoPrintThread, Nanny
from artemis.remote.utils import get_local_ips, get_socket, get_remote_artemis_path
from pytest import raises
ip_addresses = [get_plotting_server_address()]
if ip_addresses[0] not in get_local_ips():
ip_addresses.append("127.0.0.1")
def test_check_ssh_connections():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
check_ssh_connection(ip_address)
def test_check_if_port_is_free():
for ip_address in ip_addresses:
if ip_address not in get_local_ips():
with raises(AssertionError):
check_if_port_is_free(ip_address, 80)
else:
sock, port = get_socket(ip_address, 7005)
# port is now not free:
with raises(SocketServer.socket.error):
def set_up_plotting_server():
"""
Sets up the plotting server.
"""
print("Setting up Plotting Server")
# First we generate the system call that starts the server
# TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine
file_to_execute = os.path.join(os.path.dirname(__file__), 'plotting_server.py')
file_to_execute = file_to_execute.replace(os.path.expanduser("~"),"~",1)
plotting_server_address = get_plotting_server_address()
if plotting_server_address == "":
plotting_server_address = "127.0.0.1"
if plotting_server_address in get_local_ips():
command = ["python", "-u", file_to_execute]
else:
check_config_file(plotting_server_address) # Make sure all things are set
check_ssh_connection(plotting_server_address) # Make sure the SSH-connection works
command =["export DISPLAY=:0.0;", "python","-u", file_to_execute]
# TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set.
# With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously.
global _nanny
_nanny = Nanny()
cp = PythonChildProcess(ip_address=plotting_server_address, command=command, name="Plotting_Server",set_up_port_for_structured_back_communication=True)
_nanny.register_child_process(cp,monitor_for_termination=False,monitor_if_stuck_timeout=None,)
_nanny.execute_all_child_processes(blocking=False)
back_comm_queue = cp.get_queue_from_cp()
try:
is_debug_mode = getattr(sys, 'gettrace', None)
timeout = None if is_debug_mode() else 10
server_message = back_comm_queue.get(block=True,timeout=timeout)
except Queue.Empty:
print("The Plotting Server did not respond for 10 seconds. It probably crashed")
sys.exit(1)
try:
port = int(server_message.dbplot_message)
except ValueError:
print("There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}".format(str_port))
sys.exit(0)
# In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh:
# Also, we have the ssh session open already, so why not reuse it.
if plotting_server_address not in get_local_ips():
ssh_conn = cp.get_ssh_connection()
# Needs to be in a thread since the call blocks forever.
# Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example)
# will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two
# processes terminates, the ssh-tunnel is closed for the other process as well.
t3 = threading.Thread(target = forward_tunnel, kwargs={"local_port":port, "remote_host":plotting_server_address, "remote_port":port,"ssh_conn":ssh_conn})
t3.setDaemon(True)
t3.start()
# Now attempt to connect to the plotting server
server_address = ("localhost", port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(tuple(server_address))
except:
raise
# Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons:
# 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded.
# 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist.
# (There shouldn't be any, but, you know, principle)
global _to_subprocess_queue
global _id_queue
_to_subprocess_queue = Queue.Queue()
_id_queue = Queue.Queue()
t1 = threading.Thread(target=push_to_server, args=(_to_subprocess_queue, sock))
t1.setDaemon(True)
t1.start()
# if blocking:
t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock))
t2.setDaemon(True)
t2.start()
def set_up_plotting_server():
"""
Sets up the plotting server.
"""
print("Setting up Plotting Server")
# First we generate the system call that starts the server
# TODO: This assumes the same installation path relative to the home-dir on the local machine as on the remote machine
file_to_execute = os.path.join(os.path.dirname(__file__),
'plotting_server.py')
file_to_execute = file_to_execute.replace(os.path.expanduser("~"), "~", 1)
plotting_server_address = get_plotting_server_address()
if plotting_server_address not in get_local_ips():
check_config_file(
plotting_server_address) # Make sure all things are set
check_ssh_connection(
plotting_server_address) # Make sure the SSH-connection works
command = ["export DISPLAY=:0.0;", "python", "-u", file_to_execute]
# TODO: Setting DISPLAY to :0.0 is a heuristic at the moment. I don't understand yet how these DISPLAY variables are set.
else:
command = ["python", "-u", file_to_execute]
# With the command set up, we can instantiate a child process and start it. Also we want to forward stdout and stderr from the remote process asynchronously.
cp = ChildProcess(ip_address=plotting_server_address,
command=command,
name="Plotting_Server",
take_care_of_deconstruct=True)
stdin, stdout, stderr = cp.execute_child_process()
t2 = ParamikoPrintThread(source_pipe=stderr,
target_pipe=sys.stderr,
prefix="Plotting Server: ")
t2.setDaemon(True)
t2.start()
# The remote server is implemented such that it will scan available ports and choose one. This port then needs to be communicated to this client
# Therefore we assume here, that the first line on stdout from the remote server is the port number for subsequent communication. Also, this call blocks until the parameter server
# is ready to accept communication
str_port = stdout.readline()
try:
port = int(str_port)
except ValueError:
print(
"There was an incorrect string on the remote server's stdout. Make sure the server first communicates a port number. Received:\n {}"
.format(str_port))
sys.exit(0)
# All subsequent communication forwarded asynchronously
t1 = ParamikoPrintThread(source_pipe=stdout,
target_pipe=sys.stdout,
prefix="Plotting Server: ")
t1.setDaemon(True)
t1.start()
# In the remote setting we don't want to rely on the user correctly specifying their firewalls. Therefore we need to set up port forwarding through ssh:
# Also, we have the ssh session open already, so why not reuse it.
if plotting_server_address not in get_local_ips():
ssh_conn = cp.get_ssh_connection()
# Needs to be in a thread since the call blocks forever.
# Todo: this ssh tunnel is opened system-wide. That means that any subsequent attempts to open the ssh-tunnel (by another dbplot-using process, for example)
# will perform wiredly. As far as I have tested, nothing happenes and the port forwarfding works just fine in the second process, However when one of the two
# processes terminates, the ssh-tunnel is closed for the other process as well.
t3 = threading.Thread(target=forward_tunnel,
kwargs={
"local_port": port,
"remote_host": plotting_server_address,
"remote_port": port,
"ssh_conn": ssh_conn
})
t3.setDaemon(True)
t3.start()
# Now attempt to connect to the plotting server
server_address = ("localhost", port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(tuple(server_address))
except:
raise
# Once connected, set up the asynchronous threads that forward every dbplot call to the server. We make this asynchronously for two reasons:
# 1.) the queues involved can be shared between different threads (not processes), therefore allowing for asynchronous dbplot calls that are both correctly forwarded.
# 2.) sending a plot away to the server now immediatly returns, independent on any socket-related communication delays that might exist.
# (There shouldn't be any, but, you know, principle)
global _to_subprocess_queue
global _id_queue
_to_subprocess_queue = Queue.Queue()
_id_queue = Queue.Queue()
t1 = threading.Thread(target=push_to_server,
args=(_to_subprocess_queue, sock))
t1.setDaemon(True)
t1.start()
# if blocking:
t2 = threading.Thread(target=collect_from_server, args=(_id_queue, sock))
t2.setDaemon(True)
t2.start()