def prepare_command(self,command):
'''
All the stuff that I need to prepare for the command to definitely work
:param command:
:return:
'''
if self.is_local():
home_dir = os.path.expanduser("~")
else:
_,_,stdout,_ = self._run_command("echo $$; exec echo ~")
home_dir = stdout.read().strip()
if type(command) == list:
if not self.local_process:
command = [c.replace("python", self.get_extended_command(get_config_value(".artemisrc", section=self.get_ip(), option="python")), 1) if c.startswith("python") else c for c in command]
command = [s.replace("~",home_dir) for s in command]
command = " ".join([c for c in command])
else:
command = [c.strip("'") for c in command]
command = [c.replace("python", sys.executable, 1) if c.startswith("python") else c for c in command]
command = [s.replace("~",home_dir) for s in command]
elif type(command) == str or type(command) == unicode and command.startswith("python"):
if not self.local_process:
command = command.replace("python", self.get_extended_command(get_config_value(".artemisrc", section=self.get_ip(), option="python")), 1)
else:
command = command.replace("python", sys.executable)
command = command.replace("~",home_dir)
else:
raise NotImplementedError()
return command
def get_artemis_config_value(section, option, default_generator = None, write_default = False, read_method=None):
"""
Get a setting from the artemis configuration.
See docstring for get_config_value
"""
config_filename = check_or_create_artemis_config()
return get_config_value(config_filename, section=section, option=option, default_generator=default_generator, write_default=write_default, read_method=read_method)
def get_remote_artemis_path(remote_ip):
return get_config_value(
config_filename='~/.artemisrc',
section=remote_ip,
option='artemis_path',
default_generator=lambda: raw_input('Specify Remote Artemis Installation Path: '),
write_default=True
)
def test_get_config_value():
config_path = get_config_path('.testconfigrc')
if os.path.exists(config_path):
os.remove(config_path)
value = get_config_value(config_filename='.testconfigrc',
section='opts',
option='setting1',
default_generator=lambda: 'somevalue',
write_default=True)
assert value == 'somevalue'
value = get_config_value(config_filename='.testconfigrc',
section='opts',
option='setting1',
default_generator=lambda: 'someothervalue',
write_default=True)
assert value == 'somevalue'
value = get_config_value(config_filename='.testconfigrc',
section='opts',
option='setting2',
default_generator=lambda: 'blah',
write_default=True)
assert value == 'blah'
value = get_config_value(config_filename='.testconfigrc',
section='opts',
option='setting1')
assert value == 'somevalue'
value = get_config_value(config_filename='.testconfigrc',
section='opts',
option='setting2')
assert value == 'blah'
with raises(NoSectionError):
_ = get_config_value(config_filename='.testconfigrc',
section='schmopts',
option='setting3')
with raises(NoOptionError):
_ = get_config_value(config_filename='.testconfigrc',
section='opts',
option='setting3')
with raises(AssertionError):
_ = get_config_value(config_filename='.testconfigXXXrc',
section='opts',
option='setting3')
os.remove(config_path)
def get_ssh_connection(ip_address):
'''
This returns a ssh_connection to the given ip_address. Make sure to close the connection afterwards.
Requires a public/private key to be set up with the remote system. The location of the private key can be
specified in .artemisrc or, if not specified, will be looked for in ~/.ssh/id_rsa
:param ip_address:
:return:
'''
try:
path_to_private_key = get_config_value(config_filename=".artemisrc", section=ip_address, option="private_key")
except NoOptionError:
path_to_private_key = os.path.join(os.path.expanduser("~"),".ssh/id_rsa")
private_key = paramiko.RSAKey.from_private_key_file(os.path.expanduser(path_to_private_key))
username = get_config_value(config_filename=".artemisrc", section=ip_address, option="username")
ssh_conn = paramiko.SSHClient()
ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_conn.connect(hostname=ip_address, username=username, pkey=private_key)
return ssh_conn
def test_rsync():
options = ["-r"]
username = get_config_value(".artemisrc", section=ip_address, option="username")
from_path = get_local_path(relative_path="tmp/tests/", make_local_dir=True)
with open(os.path.join(from_path, "test1"), "wb"):
pass
with open(os.path.join(from_path, "test2"), "wb"):
pass
to_path = "%s@%s:/home/%s/temp/"%(username, ip_address, username)
assert rsync(options, from_path, to_path)
shutil.rmtree(from_path)
def get_artemis_config_value(section,
option,
default_generator=None,
write_default=False,
read_method=None):
"""
Get a setting from the artemis configuration.
See docstring for get_config_value
"""
config_filename = check_or_create_artemis_config()
return get_config_value(config_filename,
section=section,
option=option,
default_generator=default_generator,
write_default=write_default,
read_method=read_method)
def get_remote_installed_packages(ip_address):
'''
This method queries a remote python installation about the installed packages.
All necessary information is extracted from ~/.artemisrc
:param address: Ip address of remote server
:return:
'''
python_executable = get_config_value(config_filename=".artemisrc",
section=ip_address,
option="python")
function = "%s -c 'import pip; import json; print json.dumps({i.key: i.version for i in pip.get_installed_distributions() })' " % python_executable
ssh_conn = get_ssh_connection(ip_address)
stdin, stdout, stderr = ssh_conn.exec_command(function)
err = stderr.read()
if err:
msg="Quering %s python installation at %s sent a message on stderr. If you are confident that the error can be ignored, catch this RuntimeError" \
"accordingly. The error is: %s"%(ip_address, python_executable, err)
raise RuntimeError(msg)
installed_packages = json.loads(stdout.read())
ssh_conn.close()
return installed_packages
def simple_rsync(local_path, remote_path, ip_address, verbose=False):
'''
This method synchronizes local_path and all subfolders with remote_path at the given address.
This method executes a system rsync call. This is not a general wrapper for rsync. The call is blocking.
:param local_path:
:param remote_path: Assumed to be relative to the home dir
:param ip_address:
:return:
'''
options = "-ah"
if verbose:
options += "v"
local_path = os.path.expanduser(local_path)
username = get_config_value(config_filename=".artemisrc",
section=ip_address,
option="username")
if remote_path.startswith("~"):
remote_path = remote_path[1:]
if remote_path.startswith(("/")):
remote_path = remote_path[1:]
# to_path = "%s@%s:/home/%s/%s" % (username, address, username, remote_path)
to_path = "%s@%s:~/%s" % (username, ip_address, remote_path)
return rsync(options, from_path=local_path, to_path=to_path)
def install_packages_on_remote_virtualenv(ip_address, packages):
'''
This function installs every package in packages on the remote virtual environment specified by the ip_address in ~/.artemisrc.
In case the remote pip install -U command returns anything that is not "Successfully installed" or "Requirement already up-to-date" on stdout,
the user is informed and an error that is not a SNIMissingWarning or InsecurePlatformWarning is printed.
:param ip_address: ip_address, whose virtualenv is being modified
:param packages: a dict {key:version} of package name and associated version
:return:
'''
if len(packages) == 0:
return
print("installing/upgrading remote packages ...")
python_path = get_config_value(".artemisrc", ip_address, "python")
activate_path = os.path.join(
os.path.dirname(python_path),
"activate") # TODO: Make this work without the user using virtualenv
activate_command = "source %s" % activate_path
ssh_conn = get_ssh_connection(ip_address)
for key, version in packages.iteritems():
install_command = "pip install -U %s==%s" % (key, version)
function = "; ".join([activate_command, install_command])
stdin, stdout, stderr = ssh_conn.exec_command(function)
out = stdout.read()
if "Successfully installed" in out or "Requirement already up-to-date" in out:
pass
else:
print("Error in installing %s==%s:" % (key, version))
err = stderr.read()
err = "\n".join([
s for s in err.strip().split("\n")
if "SNIMissingWarning" not in s
and "InsecurePlatformWarning" not in s
])
print err
ssh_conn.close()
print("... Done")
def check_config_file(ip_address, file_path=".artemisrc"):
'''
Makes sure all required fields are present in ~./artemisrc.
Also performs test for the different options if applicable
:param ip_address: The section to look for. Remote ip is assumed. Makes no sense for local ip.
:return:
'''
mandatory_options = ["username", "python"]
artemisrc_path = os.path.expanduser("~/%s" % file_path)
for option in mandatory_options:
try:
get_config_value(config_filename=file_path,
section=ip_address,
option=option)
except NoSectionError:
print("Section %s could not be found in %s. Please provide it." %
(ip_address, artemisrc_path))
raise
except NoOptionError:
print(
"Section %s does not contain option %s. Please provide it in %s"
% (ip_address, option, artemisrc_path))
raise
# optional_options = ["private_key"]
try:
private_key_path = get_config_value(config_filename=file_path,
section=ip_address,
option="private_key")
assert os.path.isfile(
private_key_path
), "The path to the private_key for %s you specified in %s is not valid. You provided %s" % (
ip_address, artemisrc_path, private_key_path)
except NoOptionError:
pass
# username & private key setup tests:
try:
get_ssh_connection(ip_address)
except paramiko.ssh_exception.AuthenticationException as e:
if "Authentication failed" in e.message:
print(
"An AuthenticationException is being raised. Make sure you have your private key set up correctly"
)
else:
print(
"An AuthenticationException is being raised. Did you specify the correct username for %s in %s? You provided the username %s"
% (ip_address, artemisrc_path,
get_config_value(config_filename=".artemisrc",
section=ip_address,
option="username")))
raise
except paramiko.ssh_exception.SSHException:
try:
private_key_path = get_config_value(config_filename=file_path,
section=ip_address,
option="private_key")
print(
"Something is wrong with the private_key you specified in %s for %s . You provided %s"
% (artemisrc_path, ip_address, private_key_path))
raise
except NoOptionError:
private_key_path = os.path.join(os.path.expanduser("~"),
".ssh/id_rsa")
print(
"You did not provide a private_key path in %s. The default path %s appears to be wrongly set up. "
"Please make sure you have correctly set up your private key for %s "
% (artemisrc_path, private_key_path, ip_address))
#python tests:
python_path = get_config_value(config_filename=file_path,
section=ip_address,
option="python")
command = "python -c 'import os; print(os.path.isfile(os.path.expanduser(\"%s\")))'" % python_path
ssh_conn = get_ssh_connection(ip_address)
_, stdout, stderr = ssh_conn.exec_command(command)
assert stdout.read().strip(
) == "True", "The provided path to the remote python installation on %s does not exist. You provided %s" % (
ip_address, python_path)
command = "%s -c 'print(\"Success\")'" % python_path
_, stdout, stderr = ssh_conn.exec_command(command)
err = stderr.read().strip()
assert stdout.read().strip()=="Success" and not err, "The provided python path on %s does not seem to point to a python executable. " \
"You provided %s, which resulted in the following error on the remote machine: " %(ip_address, python_path, err)
from artemis.fileman.config_files import get_config_value
from matplotlib import pyplot as plt
__author__ = 'peter'
_plotting_mode = get_config_value('.artemisrc',
section='plotting',
option='mode',
default_generator=lambda: 'safe',
write_default=True)
if _plotting_mode == 'safe':
def redraw_figure(fig=None):
plt.draw()
plt.pause(0.00001)
elif _plotting_mode == 'fast':
def redraw_figure(fig=None):
if fig is None:
fig = plt.gcf()
fig.canvas.flush_events()
plt.show(block=False)
plt.show(block=False)
else:
raise Exception("Unknown plotting mode: {}".format(_plotting_mode))
The idea is that be able to put things in the code like:
mnist = pkl.read(open('data/mnist.pkl'))
Where the path is referenced relative to the data directory on that machine.
"""
def get_default_local_path():
return os.path.join(os.getenv("HOME"), '.artemis')
LOCAL_DIR = get_config_value('.artemisrc',
'fileman',
'data_dir',
default_generator=get_default_local_path,
write_default=True)
def get_local_path(relative_path='', make_local_dir=False):
"""
Get the full local path of a file relative to the Data folder. If the relative path starts with a "/", we consider
it to be a local path already. TODO: Make this Windows-friendly
:param relative_path: A path relative to the data directory. If it starts with "/", we consider it to be already
:param make_local_dir: True to create the directory that the path points to, if it does not already exist.
:return: The full path to the file
"""
if not relative_path.startswith('/'):
# Path is considered relative to data directory.
def test_get_config_value():
config_path = get_config_path('.testconfigrc')
if os.path.exists(config_path):
os.remove(config_path)
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting1', default_generator=lambda: 'somevalue', write_default=True)
assert value == 'somevalue'
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting1', default_generator=lambda: 'someothervalue', write_default=True)
assert value == 'somevalue'
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting2', default_generator=lambda: 'blah', write_default=True)
assert value == 'blah'
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting1')
assert value == 'somevalue'
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting2')
assert value == 'blah'
with raises(NoSectionError):
_ = get_config_value(config_filename='.testconfigrc', section='schmopts', option='setting3')
with raises(NoOptionError):
_ = get_config_value(config_filename='.testconfigrc', section='opts', option='setting3')
with raises(AssertionError):
_ = get_config_value(config_filename='.testconfigXXXrc', section='opts', option='setting3')
set_non_persistent_config_value(config_filename='.testconfigrc', section='opts', option='setting2',value="bob")
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting2')
assert value == 'bob'
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting2', use_cashed_config=False)
assert value == 'blah'
value = get_config_value(config_filename='.testconfigrc', section='opts', option='setting2')
assert value == 'bob'
set_non_persistent_config_value(config_filename='.testconfigrc', section='schmapts', option='setting2', value="bob")
with raises(NoOptionError):
_ = get_config_value(config_filename='.testconfigrc', section='schmapts', option='setting3')
with raises(NoSectionError):
_ = get_config_value(config_filename='.testconfigrc', section='schmapts', option='setting2', use_cashed_config=False)
value = get_config_value(config_filename='.testconfigrc', section='schmapts', option='setting2')
assert value == 'bob'
os.remove(config_path)
