def run_job(self, file, jobid, timeout=None, env={}):
try:
jobs_dir = ".cstar/remote-jobs/" + jobid
self.run(("mkdir", "-p", jobs_dir))
self.put_command(file, "%s/job" % (jobs_dir, ))
# Manually insert environment into script, since passing env into exec_command leads to it being
# ignored on most ssh servers. :-(
for key in env:
if _alnum_re.search(key):
raise BadEnvironmentVariable(key)
env_str = " ".join(key + "=" + self.escape(value)
for key, value in env.items())
remote_script = resource_string('cstar.resources',
'scripts/remote_job.sh')
wrapper = remote_script.decode("utf-8") % (env_str, )
self.write_command(wrapper, "%s/wrapper" % (jobs_dir, ))
cmd_cd = "cd %s" % (self.escape(jobs_dir), )
cmd_wrapper = "nohup ./wrapper"
self.exec_command(cmd_cd + ";" + cmd_wrapper)
out, err_output, status = self.read_channel()
real_output = self.read_file(jobs_dir + "/stdout")
real_error = self.read_file(jobs_dir + "/stderr")
real_status = int(self.read_file(jobs_dir + "/status"))
return ExecutionResult(cmd_wrapper, real_status, real_output,
real_error)
except:
err("Command failed : ", sys.exc_info()[0])
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def _connect(self):
try:
if self.session == None:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.hostname, 22))
self.session = Session()
self.session.handshake(sock)
if self.ssh_identity_file != None:
self.session.userauth_publickey_fromfile(
self.ssh_username, self.ssh_identity_file, '', None)
elif self.ssh_password != None:
self.session.userauth_password(self.ssh_username,
self.ssh_password)
elif self.ssh_username == None:
user = os.getlogin()
self.session.agent_auth(user)
else:
self.session.agent_auth(self.ssh_username)
self.channel = self.session.open_session()
except:
self.channel = None
self.session = None
raise BadSSHHost(
"Could not establish an SSH connection to host %s" %
(self.hostname, ))
def _connect(self):
if self.client:
# Ensure underlying client is still a valid open connection
try:
stdin, stdout, stderr = self.client.exec_command(PING_COMMAND)
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
# ConnectionResetError is raised when a connection was established but then broken
# paramiko.ssh_exception.SSHException is raised if the connection was known to be broken
self.client = None
if not self.client:
try:
self.client = paramiko.client.SSHClient()
pkey = None
if self.ssh_identity_file != None:
pkey = paramiko.RSAKey.from_private_key_file(
self.ssh_identity_file, None)
debug("Username : "******"Id file: ", self.ssh_identity_file)
self.client.set_missing_host_key_policy(
paramiko.client.AutoAddPolicy())
self.client.connect(self.hostname,
compress=True,
username=self.ssh_username,
password=self.ssh_password,
pkey=pkey)
except:
self.client = None
raise BadSSHHost(
"Could not establish an SSH connection to host %s" %
(self.hostname, ))
def run_job(self, file, jobid, timeout=None, env={}):
try:
self._connect()
transport = self.client.get_transport()
session = transport.open_session()
paramiko.agent.AgentRequestHandler(session)
dir = ".cstar/remote-jobs/" + jobid
self.run(("mkdir", "-p", dir))
self.put_command(file, "%s/job" % (dir, ))
# Manually insert environment into script, since passing env into exec_command leads to it being
# ignored on most ssh servers. :-(
for key in env:
if _alnum_re.search(key):
raise BadEnvironmentVariable(key)
env_str = " ".join(key + "=" + self.escape(value)
for key, value in env.items())
wrapper = r"""#! /bin/sh
if test -f pid; then
# We can't wait for things that aren't our children. Loop and sleep. :-(
while ! test -f status; do
sleep 10s
done
exit
fi
%s ./job >stdout 2>stderr &
echo $! >pid
wait $!
echo $? >status
""" % (env_str, )
self.write_command(wrapper, "%s/wrapper" % (dir, ))
cmd = """
cd %s
nohup ./wrapper
""" % (self.escape(dir), )
stdin, stdout, stderr = self.client.exec_command(cmd,
timeout=timeout)
stdout.channel.recv_exit_status()
real_output = self.read_file(dir + "/stdout")
real_error = self.read_file(dir + "/stderr")
real_status = int(self.read_file(dir + "/status"))
return ExecutionResult(cmd, real_status, real_output, real_error)
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def run(self, argv):
try:
cmd = " ".join(self.escape(s) for s in argv)
self.exec_command(cmd)
out, error, status = self.read_channel()
if status != 0:
err("Command %s failed with status %d on host %s" %
(cmd, status, self.hostname))
else:
debug("Command %s succeeded on host %s, output was %s and %s" %
(cmd, self.hostname, out, error))
return ExecutionResult(cmd, status, out, error)
except:
self.client = None
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def run_job(self, file, jobid, timeout=None, env={}):
try:
self._connect()
transport = self.client.get_transport()
session = transport.open_session()
paramiko.agent.AgentRequestHandler(session)
dir = ".cstar/remote-jobs/" + jobid
self.run(("mkdir", "-p", dir))
self.put_command(file, "%s/job" % (dir, ))
# Manually insert environment into script, since passing env into exec_command leads to it being
# ignored on most ssh servers. :-(
for key in env:
if _alnum_re.search(key):
raise BadEnvironmentVariable(key)
# substitute host variables in the command
env_str = self._substitute_host_variables(" ".join(
key + "=" + self.escape(value) for key, value in env.items()))
remote_script = resource_string('cstar.resources',
'scripts/remote_job.sh')
wrapper = remote_script.decode("utf-8") % (env_str, )
self.write_command(wrapper, "%s/wrapper" % (dir, ))
cmd = """
cd %s
nohup ./wrapper
""" % (self.escape(dir), )
stdin, stdout, stderr = self.client.exec_command(cmd,
timeout=timeout)
_, _, _ = self._read_results(stdin, stdout, stderr, timeout)
real_output = self.read_file(dir + "/stdout")
real_error = self.read_file(dir + "/stderr")
real_status = int(self.read_file(dir + "/status"))
return ExecutionResult(cmd, real_status, real_output, real_error)
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def run(self, argv):
try:
self._connect()
cmd = " ".join(self.escape(s) for s in argv)
stdin, stdout, stderr = self.client.exec_command(cmd)
status = stdout.channel.recv_exit_status()
out = stdout.read()
error = stderr.read()
if status != 0:
err("Command %s failed with status %d on host %s" % (cmd, status, self.hostname))
else:
debug("Command %s succeeded on host %s, output was %s and %s" %
(cmd, self.hostname, str(out, 'utf-8'), str(error, 'utf-8')))
return ExecutionResult(cmd, status, str(out, 'utf-8'), str(error, 'utf-8'))
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
self.client = None
raise BadSSHHost("SSH connection to host %s was reset" % (self.hostname,))
def _connect(self):
if self.client:
# Ensure underlying client is still a valid open connection
try:
stdin, stdout, stderr = self.client.exec_command(PING_COMMAND)
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
# ConnectionResetError is raised when a connection was established but then broken
# paramiko.ssh_exception.SSHException is raised if the connection was known to be broken
self.client = None
if not self.client:
try:
self.client = paramiko.client.SSHClient()
self.client.set_missing_host_key_policy(
paramiko.client.AutoAddPolicy())
self.client.connect(self.hostname, compress=True)
except:
self.client = None
raise BadSSHHost(
"Could not establish an SSH connection to host %s" %
(self.hostname, ))