def check_config_file(ip_address,file_path=".artemisrc"): ''' Makes sure all required fields are present in ~./artemisrc. Also performs test for the different options if applicable :param ip_address: The section to look for. Remote ip is assumed. Makes no sense for local ip. :return: ''' mandatory_options = ["username","python"] artemisrc_path = os.path.expanduser("~/%s"%file_path) for option in mandatory_options: try: get_artemis_config_value(section=ip_address,option=option) except NoSectionError: print("Section %s could not be found in %s. Please provide it." %(ip_address, artemisrc_path)) raise except NoOptionError: print("Section %s does not contain option %s. Please provide it in %s" %(ip_address, option, artemisrc_path)) raise # optional_options = ["private_key"] try: private_key_path = get_artemis_config_value(section=ip_address,option="private_key") assert os.path.isfile(private_key_path), "The path to the private_key for %s you specified in %s is not valid. You provided %s" %(ip_address, artemisrc_path, private_key_path) except NoOptionError: pass # username & private key setup tests: try: get_ssh_connection(ip_address) except paramiko.ssh_exception.AuthenticationException as e: if "Authentication failed" in e.message: print("An AuthenticationException is being raised. Make sure you have your private key set up correctly") else: print("An AuthenticationException is being raised. Did you specify the correct username for %s in %s? You provided the username %s"% (ip_address, artemisrc_path, get_artemis_config_value(section=ip_address,option="username"))) raise except paramiko.ssh_exception.SSHException: try: private_key_path = get_artemis_config_value(section=ip_address,option="private_key") print ("Something is wrong with the private_key you specified in %s for %s . You provided %s" % (artemisrc_path, ip_address, private_key_path)) raise except NoOptionError: private_key_path = os.path.join(os.path.expanduser("~"),".ssh/id_rsa") print("You did not provide a private_key path in %s. The default path %s appears to be wrongly set up. " "Please make sure you have correctly set up your private key for %s " %(artemisrc_path,private_key_path,ip_address)) #python tests: python_path = get_artemis_config_value(section=ip_address,option="python") command = "python -c 'import os; print(os.path.isfile(os.path.expanduser(\"%s\")))'"%python_path ssh_conn = get_ssh_connection(ip_address) _,stdout,stderr = ssh_conn.exec_command(command) assert stdout.read().strip()=="True", "The provided path to the remote python installation on %s does not exist. You provided %s" %(ip_address, python_path) command = "%s -c 'print(\"Success\")'" % python_path _,stdout,stderr = ssh_conn.exec_command(command) err = stderr.read().strip() assert stdout.read().strip()=="Success" and not err, "The provided python path on %s does not seem to point to a python executable. " \ "You provided %s, which resulted in the following error on the remote machine: " %(ip_address, python_path, err)
def get_configured_machines(): """ :return: A dict<machine_name: {'ip': ip_address, 'username': username}> """ config_filename = check_or_create_artemis_config() sections = get_config_sections(config_filename) ip_sections = [s for s in sections if s.startswith('remote:')] machines = {} for sec in ip_sections: machine_name = sec[len('remote:'):] ip = get_artemis_config_value(section=sec, option='ip') username = get_artemis_config_value(section=sec, option='username') machines[machine_name] = {'ip': ip, 'username': username} return machines
def get_remote_artemis_path(remote_ip): return get_artemis_config_value( section=remote_ip, option='artemis_path', default_generator=lambda: input('Specify Remote Artemis Installation Path: ').strip(), write_default=True )
def get_remote_artemis_path(remote_ip): return get_artemis_config_value( section=remote_ip, option='artemis_path', default_generator=lambda: raw_input( 'Specify Remote Artemis Installation Path: '), write_default=True)
def test_plotting_server(): if get_artemis_config_value(section='plotting', option='backend') != 'matplotlib-web': setup_web_plotting() for i in xrange(5): dbplot(np.random.randn(10, 10, 3), 'noise') dbplot(np.random.randn(20, 2), 'lines') plt.pause(0.1)
def prepare_command(self,command): ''' All the stuff that I need to prepare for the command to definitely work :param command: :return: ''' if self.set_up_port_for_structured_back_communication: self.queue_from_cp, port = listen_on_port() if self.is_local(): address = "127.0.0.1" else: address = get_local_ips()[-1] if self.is_local(): home_dir = os.path.expanduser("~") else: _,_,stdout,_ = self._run_command("echo $$; exec echo ~") home_dir = stdout.read().strip() if type(command) == list: if self.set_up_port_for_structured_back_communication: command.append("--port=%i"%port) command.append("--address=%s"%address) if not self.local_process: command = [c.replace("python", self.get_extended_command(get_artemis_config_value(section=self.get_ip(), option="python", default_generator=lambda: sys.executable)), 1) if c.startswith("python") else c for c in command] command = [s.replace("~",home_dir) for s in command] command = " ".join([c for c in command]) else: command = [c.strip("'") for c in command] command = [c.replace("python", sys.executable, 1) if c.startswith("python") else c for c in command] command = [s.replace("~",home_dir) for s in command] elif isinstance(command, string_types) and command.startswith("python"): if self.set_up_port_for_structured_back_communication: command += " --port=%i "%port command += "--address=%s"%address if not self.local_process: command = command.replace("python", self.get_extended_command(get_artemis_config_value(section=self.get_ip(), option="python",default_generator=lambda: sys.executable)), 1) else: command = command.replace("python", sys.executable) command = command.replace("~",home_dir) else: raise NotImplementedError() return command
def get_experiment_dir(): path = os.path.expanduser( get_artemis_config_value( section="experiments", option="experiment_directory", write_default=True, default_generator=lambda: get_artemis_data_path('experiments'))) if not os.path.exists(path): make_dir(path) return path
def test_rsync(): options = ["-r"] username = get_artemis_config_value(section=ip_address, option="username") from_path = get_artemis_data_path(relative_path="tmp/tests/", make_local_dir=True) with open(os.path.join(from_path, "test1"), "wb"): pass with open(os.path.join(from_path, "test2"), "wb"): pass to_path = "%s@%s:/home/%s/temp/"%(username, ip_address, username) assert rsync(options, from_path, to_path) shutil.rmtree(from_path)
def test_rsync(): options = ["-r"] username = get_artemis_config_value(section=ip_address, option="username") from_path = get_artemis_data_path(relative_path="tmp/tests/", make_local_dir=True) with open(os.path.join(from_path, "test1"), "wb"): pass with open(os.path.join(from_path, "test2"), "wb"): pass to_path = "%s@%s:/home/%s/temp/" % (username, ip_address, username) assert rsync(options, from_path, to_path) shutil.rmtree(from_path)
def get_ssh_connection(ip_address): ''' This returns a ssh_connection to the given ip_address. Make sure to close the connection afterwards. Requires a public/private key to be set up with the remote system. The location of the private key can be specified in .artemisrc or, if not specified, will be looked for in ~/.ssh/id_rsa :param ip_address: :return: ''' import paramiko path_to_private_key = os.path.join(os.path.expanduser("~"),".ssh/id_rsa") private_key = paramiko.RSAKey.from_private_key_file(os.path.expanduser(path_to_private_key)) username = get_artemis_config_value(section=ip_address, option="username", default_generator=lambda: getpass.getuser()) ssh_conn = paramiko.SSHClient() ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_conn.connect(hostname=ip_address, username=username, pkey=private_key) return ssh_conn
def get_remote_installed_packages(ip_address): ''' This method queries a remote python installation about the installed packages. All necessary information is extracted from ~/.artemisrc :param address: Ip address of remote server :return: ''' python_executable = get_artemis_config_value(section=ip_address, option="python") function = "%s -c 'import pip; import json; print json.dumps({i.key: i.version for i in pip.get_installed_distributions() })' " % python_executable ssh_conn = get_ssh_connection(ip_address) stdin, stdout, stderr = ssh_conn.exec_command(function) err = stderr.read() if err: msg="Quering %s python installation at %s sent a message on stderr. If you are confident that the error can be ignored, catch this RuntimeError" \ "accordingly. The error is: %s"%(ip_address, python_executable, err) raise RuntimeError(msg) installed_packages = json.loads(stdout.read()) ssh_conn.close() return installed_packages
def simple_rsync(local_path, remote_path, ip_address, verbose=False): ''' This method synchronizes local_path and all subfolders with remote_path at the given address. This method executes a system rsync call. This is not a general wrapper for rsync. The call is blocking. :param local_path: :param remote_path: Assumed to be relative to the home dir :param ip_address: :return: ''' options = "-ah" if verbose: options += "v" local_path = os.path.expanduser(local_path) username = get_artemis_config_value(section=ip_address, option="username") if remote_path.startswith("~"): remote_path = remote_path[1:] if remote_path.startswith(("/")): remote_path = remote_path[1:] # to_path = "%s@%s:/home/%s/%s" % (username, address, username, remote_path) to_path = "%s@%s:~/%s" % (username, ip_address, remote_path) return rsync(options, from_path=local_path, to_path=to_path)
def install_packages_on_remote_virtualenv(ip_address, packages): ''' This function installs every package in packages on the remote virtual environment specified by the ip_address in ~/.artemisrc. In case the remote pip install -U command returns anything that is not "Successfully installed" or "Requirement already up-to-date" on stdout, the user is informed and an error that is not a SNIMissingWarning or InsecurePlatformWarning is printed. :param ip_address: ip_address, whose virtualenv is being modified :param packages: a dict {key:version} of package name and associated version :return: ''' if len(packages) == 0: return print("installing/upgrading remote packages ...") python_path = get_artemis_config_value(ip_address, "python") activate_path = os.path.join( os.path.dirname(python_path), "activate") # TODO: Make this work without the user using virtualenv activate_command = "source %s" % activate_path ssh_conn = get_ssh_connection(ip_address) for key, version in packages.items(): install_command = "pip install -U %s==%s" % (key, version) function = "; ".join([activate_command, install_command]) stdin, stdout, stderr = ssh_conn.exec_command(function) out = stdout.read() if "Successfully installed" in out or "Requirement already up-to-date" in out: pass else: print(("Error in installing %s==%s:" % (key, version))) err = stderr.read() err = "\n".join([ s for s in err.strip().split("\n") if "SNIMissingWarning" not in s and "InsecurePlatformWarning" not in s ]) print(err) ssh_conn.close() print("... Done")
from artemis.config import get_artemis_config_value from matplotlib import pyplot as plt __author__ = 'peter' _plotting_mode = get_artemis_config_value(section='plotting', option='mode') if _plotting_mode == 'safe': def redraw_figure(fig=None): plt.draw() plt.pause(0.00001) elif _plotting_mode == 'fast': def redraw_figure(fig=None): if fig is None: fig = plt.gcf() fig.canvas.flush_events() plt.show(block=False) plt.show(block=False) else: raise Exception("Unknown plotting mode: {}".format(_plotting_mode))
is_1d = not is_scalar and data.size == np.max(data.shape) if is_1d: n_unique = len(np.unique(data)) if n_unique == 2: return ImagePlot(cmap=cmap) else: return LinePlot() elif data.ndim == 2 and data.shape[1] < line_to_image_threshold: return LinePlot() else: return ImagePlot(cmap=cmap) _PLOTTING_SERVER = get_artemis_config_value(section='plotting', option='plotting_server', default_generator="") _USE_SERVER = _PLOTTING_SERVER != "" def is_server_plotting_on(): return _USE_SERVER def set_server_plotting(state): global _USE_SERVER _USE_SERVER = state def get_plotting_server_address(): return _PLOTTING_SERVER
def get_experiment_dir(): path = os.path.expanduser(get_artemis_config_value(section="experiments", option="experiment_directory", write_default=True, default_generator=lambda: get_artemis_data_path('experiments'))) if not os.path.exists(path): make_dir(path) return path
plot.plot() if hang: plt.figure(_DBPLOT_FIGURES[fig].figure.number) plt.show() else: redraw_figure(_DBPLOT_FIGURES[fig].figure) return _DBPLOT_FIGURES[fig].subplots[name].axis _PlotWindow = namedtuple('PlotWindow', ['figure', 'subplots', 'axes']) _Subplot = namedtuple('Subplot', ['axis', 'plot_object']) _DBPLOT_FIGURES = {} # An dict<figure_name: _PlotWindow(figure, OrderedDict<subplot_name:_Subplot>)> _DEFAULT_SIZE = get_artemis_config_value(section='plotting', option='default_fig_size', default_generator=lambda: (10, 8), write_default=True, read_method='eval') _draw_counters = {} _hold_plots = False _hold_plot_counter = None _default_layout = 'grid' PLOT_CONSTRUCTORS = { 'line': LinePlot, 'thick-line': partial(LinePlot, plot_kwargs={'linewidth': 3}), 'pos_line': partial(LinePlot, y_bounds=(0, None), y_bound_extend=(0, 0.05)), 'bbox': partial(BoundingBoxPlot, linewidth=2, axes_update_mode='expand'),
return TextPlot() is_1d = not is_scalar and data.size == np.max(data.shape) if is_1d: n_unique = len(np.unique(data)) if n_unique == 2: return ImagePlot(cmap=cmap) else: return LinePlot() elif data.ndim == 2 and data.shape[1] < line_to_image_threshold: return LinePlot() else: return ImagePlot(cmap=cmap) _PLOTTING_SERVER = get_artemis_config_value(section='plotting', option='plotting_server', default_generator="") _USE_SERVER = _PLOTTING_SERVER != "" def is_server_plotting_on(): return _USE_SERVER def set_server_plotting(state): global _USE_SERVER _USE_SERVER = state def get_plotting_server_address(): return _PLOTTING_SERVER
""" For dealing with files in a uniform way between machines, we have a local directory for data. The idea is that be able to put things in the code like: mnist = pkl.read(open('data/mnist.pkl')) Where the path is referenced relative to the data directory on that machine. """ def get_default_local_path(): return os.path.join(os.getenv("HOME"), '.artemis') LOCAL_DIR = get_artemis_config_value(section='fileman', option='data_dir', default_generator = get_default_local_path, write_default = True) def get_artemis_data_path(relative_path ='', make_local_dir = False): """ Get the full local path of a file relative to the Data folder. If the relative path starts with a "/", we consider it to be a local path already. TODO: Make this Windows-friendly :param relative_path: A path relative to the data directory. If it starts with "/", we consider it to be already :param make_local_dir: True to create the directory that the path points to, if it does not already exist. :return: The full path to the file """ if not relative_path.startswith('/'): # Path is considered relative to data directory. file_path = os.path.join(LOCAL_DIR, format_filename(relative_path)) else:
from artemis.config import get_artemis_config_value from matplotlib import pyplot as plt __author__ = 'peter' _plotting_mode = get_artemis_config_value(section='plotting', option='mode', default_generator=lambda: 'safe') if _plotting_mode == 'safe': def redraw_figure(fig=None): plt.draw() plt.pause(0.00001) elif _plotting_mode == 'fast': def redraw_figure(fig=None): if fig is None: fig = plt.gcf() fig.canvas.flush_events() plt.show(block=False) plt.show(block=False) else: raise Exception("Unknown plotting mode: {}".format(_plotting_mode))
import signal import sys import time import os from artemis.config import get_artemis_config_value from artemis.remote.child_processes import PythonChildProcess, RemotePythonProcess from artemis.remote.remote_execution import ParamikoPrintThread, execute_command from artemis.remote.utils import get_local_ips, get_remote_artemis_path from functools import partial ip_addresses = [get_artemis_config_value(section="tests", option="remote_server",default_generator=lambda: "127.0.0.1")] # ip_addresses=["127.0.0.1"] if ip_addresses[0] not in get_local_ips(): ip_addresses.append("127.0.0.1") def get_test_functions_path(ip_address): if ip_address in get_local_ips(): return os.path.join(os.path.dirname(__file__), "remote_test_functions.py") else: remote_path = get_remote_artemis_path(ip_address) return os.path.join(remote_path, "remote/remote_test_functions.py") def test_simple_pcp(): for ip_address in ip_addresses: command = "python %s --callback=%s"%(get_test_functions_path(ip_address),"success_function") pyc = PythonChildProcess(ip_address=ip_address,command=command) (stdin, stdout, stderr) = pyc.execute_child_process() stderr_out =stderr.readlines()
import SocketServer from artemis.config import get_artemis_config_value from pytest import raises from artemis.remote.utils import get_local_ips, check_ssh_connection, check_if_port_is_free, get_socket ip_addresses = [ get_artemis_config_value(section="tests", option="remote_server", default_generator=lambda: "127.0.0.1") ] # ip_addresses=["127.0.0.1"] if ip_addresses[0] not in get_local_ips(): ip_addresses.append("127.0.0.1") def test_check_ssh_connections(): for ip_address in ip_addresses: if ip_address not in get_local_ips(): check_ssh_connection(ip_address) def test_check_if_port_is_free(): for ip_address in ip_addresses: if ip_address not in get_local_ips(): with raises(AssertionError): check_if_port_is_free(ip_address, 80) else: sock, port = get_socket(ip_address, 7005) # port is now not free:
For dealing with files in a uniform way between machines, we have a local directory for data. The idea is that be able to put things in the code like: mnist = pkl.read(open('data/mnist.pkl')) Where the path is referenced relative to the data directory on that machine. """ def get_default_local_path(): return os.path.join(expanduser("~"), '.artemis') LOCAL_DIR = get_artemis_config_value(section='fileman', option='data_dir', default_generator=get_default_local_path, write_default=True) def get_artemis_data_path(relative_path='', make_local_dir=False): """ Get the full local path of a file relative to the Data folder. If the relative path starts with a "/", we consider it to be a local path already. TODO: Make this Windows-friendly :param relative_path: A path relative to the data directory. If it starts with "/", we consider it to be already :param make_local_dir: True to create the directory that the path points to, if it does not already exist. :return: The full path to the file """ if not relative_path.startswith('/'): # Path is considered relative to data directory. file_path = os.path.join(LOCAL_DIR, format_filename(relative_path))
def check_config_file(ip_address, file_path=".artemisrc"): ''' Makes sure all required fields are present in ~./artemisrc. Also performs test for the different options if applicable :param ip_address: The section to look for. Remote ip is assumed. Makes no sense for local ip. :return: ''' mandatory_options = ["username", "python"] artemisrc_path = os.path.expanduser("~/%s" % file_path) for option in mandatory_options: try: get_artemis_config_value(section=ip_address, option=option) except NoSectionError: print("Section %s could not be found in %s. Please provide it." % (ip_address, artemisrc_path)) raise except NoOptionError: print( "Section %s does not contain option %s. Please provide it in %s" % (ip_address, option, artemisrc_path)) raise # optional_options = ["private_key"] try: private_key_path = get_artemis_config_value(section=ip_address, option="private_key") assert os.path.isfile( private_key_path ), "The path to the private_key for %s you specified in %s is not valid. You provided %s" % ( ip_address, artemisrc_path, private_key_path) except NoOptionError: pass # username & private key setup tests: try: get_ssh_connection(ip_address) except paramiko.ssh_exception.AuthenticationException as e: if "Authentication failed" in e.message: print( "An AuthenticationException is being raised. Make sure you have your private key set up correctly" ) else: print( "An AuthenticationException is being raised. Did you specify the correct username for %s in %s? You provided the username %s" % (ip_address, artemisrc_path, get_artemis_config_value(section=ip_address, option="username"))) raise except paramiko.ssh_exception.SSHException: try: private_key_path = get_artemis_config_value(section=ip_address, option="private_key") print( "Something is wrong with the private_key you specified in %s for %s . You provided %s" % (artemisrc_path, ip_address, private_key_path)) raise except NoOptionError: private_key_path = os.path.join(os.path.expanduser("~"), ".ssh/id_rsa") print( "You did not provide a private_key path in %s. The default path %s appears to be wrongly set up. " "Please make sure you have correctly set up your private key for %s " % (artemisrc_path, private_key_path, ip_address)) #python tests: python_path = get_artemis_config_value(section=ip_address, option="python") command = "python -c 'import os; print(os.path.isfile(os.path.expanduser(\"%s\")))'" % python_path ssh_conn = get_ssh_connection(ip_address) _, stdout, stderr = ssh_conn.exec_command(command) assert stdout.read().strip( ) == "True", "The provided path to the remote python installation on %s does not exist. You provided %s" % ( ip_address, python_path) command = "%s -c 'print(\"Success\")'" % python_path _, stdout, stderr = ssh_conn.exec_command(command) err = stderr.read().strip() assert stdout.read().strip()=="Success" and not err, "The provided python path on %s does not seem to point to a python executable. " \ "You provided %s, which resulted in the following error on the remote machine: " %(ip_address, python_path, err)
plt.show() else: redraw_figure(_DBPLOT_FIGURES[fig].figure) return _DBPLOT_FIGURES[fig].subplots[name].axis _PlotWindow = namedtuple('PlotWindow', ['figure', 'subplots', 'axes']) _Subplot = namedtuple('Subplot', ['axis', 'plot_object']) _DBPLOT_FIGURES = { } # An dict<figure_name: _PlotWindow(figure, OrderedDict<subplot_name:_Subplot>)> _DEFAULT_SIZE = get_artemis_config_value(section='plotting', option='default_fig_size', default_generator=lambda: (10, 8), write_default=True, read_method='eval') _draw_counters = {} _hold_plots = False _hold_plot_counter = 0 _default_layout = 'grid' PLOT_CONSTRUCTORS = { 'line': LinePlot, 'thick-line':
def prepare_command(self, command): ''' All the stuff that I need to prepare for the command to definitely work :param command: :return: ''' if self.set_up_port_for_structured_back_communication: self.queue_from_cp, port = listen_on_port() if self.is_local(): address = "127.0.0.1" else: address = get_local_ips()[-1] if self.is_local(): home_dir = os.path.expanduser("~") else: _, _, stdout, _ = self._run_command("echo $$; exec echo ~") home_dir = stdout.read().strip() if type(command) == list: if self.set_up_port_for_structured_back_communication: command.append("--port=%i" % port) command.append("--address=%s" % address) if not self.local_process: command = [ c.replace( "python", self.get_extended_command( get_artemis_config_value( section=self.get_ip(), option="python", default_generator=lambda: sys.executable)), 1) if c.startswith("python") else c for c in command ] command = [s.replace("~", home_dir) for s in command] command = " ".join([c for c in command]) else: command = [c.strip("'") for c in command] command = [ c.replace("python", sys.executable, 1) if c.startswith("python") else c for c in command ] command = [s.replace("~", home_dir) for s in command] elif isinstance(command, string_types) and command.startswith("python"): if self.set_up_port_for_structured_back_communication: command += " --port=%i " % port command += "--address=%s" % address if not self.local_process: command = command.replace( "python", self.get_extended_command( get_artemis_config_value( section=self.get_ip(), option="python", default_generator=lambda: sys.executable)), 1) else: command = command.replace("python", sys.executable) command = command.replace("~", home_dir) else: raise NotImplementedError() return command