def run_job(self, file, jobid, timeout=None, env={}):
try:
jobs_dir = ".cstar/remote-jobs/" + jobid
self.run(("mkdir", "-p", jobs_dir))
self.put_command(file, "%s/job" % (jobs_dir, ))
# Manually insert environment into script, since passing env into exec_command leads to it being
# ignored on most ssh servers. :-(
for key in env:
if _alnum_re.search(key):
raise BadEnvironmentVariable(key)
env_str = " ".join(key + "=" + self.escape(value)
for key, value in env.items())
remote_script = resource_string('cstar.resources',
'scripts/remote_job.sh')
wrapper = remote_script.decode("utf-8") % (env_str, )
self.write_command(wrapper, "%s/wrapper" % (jobs_dir, ))
cmd_cd = "cd %s" % (self.escape(jobs_dir), )
cmd_wrapper = "nohup ./wrapper"
self.exec_command(cmd_cd + ";" + cmd_wrapper)
out, err_output, status = self.read_channel()
real_output = self.read_file(jobs_dir + "/stdout")
real_error = self.read_file(jobs_dir + "/stderr")
real_status = int(self.read_file(jobs_dir + "/status"))
return ExecutionResult(cmd_wrapper, real_status, real_output,
real_error)
except:
err("Command failed : ", sys.exc_info()[0])
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def run_job(self, file, jobid, timeout=None, env={}):
try:
self._connect()
transport = self.client.get_transport()
session = transport.open_session()
paramiko.agent.AgentRequestHandler(session)
dir = ".cstar/remote-jobs/" + jobid
self.run(("mkdir", "-p", dir))
self.put_command(file, "%s/job" % (dir, ))
# Manually insert environment into script, since passing env into exec_command leads to it being
# ignored on most ssh servers. :-(
for key in env:
if _alnum_re.search(key):
raise BadEnvironmentVariable(key)
env_str = " ".join(key + "=" + self.escape(value)
for key, value in env.items())
wrapper = r"""#! /bin/sh
if test -f pid; then
# We can't wait for things that aren't our children. Loop and sleep. :-(
while ! test -f status; do
sleep 10s
done
exit
fi
%s ./job >stdout 2>stderr &
echo $! >pid
wait $!
echo $? >status
""" % (env_str, )
self.write_command(wrapper, "%s/wrapper" % (dir, ))
cmd = """
cd %s
nohup ./wrapper
""" % (self.escape(dir), )
stdin, stdout, stderr = self.client.exec_command(cmd,
timeout=timeout)
stdout.channel.recv_exit_status()
real_output = self.read_file(dir + "/stdout")
real_error = self.read_file(dir + "/stderr")
real_status = int(self.read_file(dir + "/status"))
return ExecutionResult(cmd, real_status, real_output, real_error)
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def run(self, argv):
try:
cmd = " ".join(self.escape(s) for s in argv)
self.exec_command(cmd)
out, error, status = self.read_channel()
if status != 0:
err("Command %s failed with status %d on host %s" %
(cmd, status, self.hostname))
else:
debug("Command %s succeeded on host %s, output was %s and %s" %
(cmd, self.hostname, out, error))
return ExecutionResult(cmd, status, out, error)
except:
self.client = None
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def run_job(self, file, jobid, timeout=None, env={}):
try:
self._connect()
transport = self.client.get_transport()
session = transport.open_session()
paramiko.agent.AgentRequestHandler(session)
dir = ".cstar/remote-jobs/" + jobid
self.run(("mkdir", "-p", dir))
self.put_command(file, "%s/job" % (dir, ))
# Manually insert environment into script, since passing env into exec_command leads to it being
# ignored on most ssh servers. :-(
for key in env:
if _alnum_re.search(key):
raise BadEnvironmentVariable(key)
# substitute host variables in the command
env_str = self._substitute_host_variables(" ".join(
key + "=" + self.escape(value) for key, value in env.items()))
remote_script = resource_string('cstar.resources',
'scripts/remote_job.sh')
wrapper = remote_script.decode("utf-8") % (env_str, )
self.write_command(wrapper, "%s/wrapper" % (dir, ))
cmd = """
cd %s
nohup ./wrapper
""" % (self.escape(dir), )
stdin, stdout, stderr = self.client.exec_command(cmd,
timeout=timeout)
_, _, _ = self._read_results(stdin, stdout, stderr, timeout)
real_output = self.read_file(dir + "/stdout")
real_error = self.read_file(dir + "/stderr")
real_status = int(self.read_file(dir + "/status"))
return ExecutionResult(cmd, real_status, real_output, real_error)
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
raise BadSSHHost("SSH connection to host %s was reset" %
(self.hostname, ))
def run(self, argv):
try:
self._connect()
cmd = " ".join(self.escape(s) for s in argv)
stdin, stdout, stderr = self.client.exec_command(cmd)
status = stdout.channel.recv_exit_status()
out = stdout.read()
error = stderr.read()
if status != 0:
err("Command %s failed with status %d on host %s" % (cmd, status, self.hostname))
else:
debug("Command %s succeeded on host %s, output was %s and %s" %
(cmd, self.hostname, str(out, 'utf-8'), str(error, 'utf-8')))
return ExecutionResult(cmd, status, str(out, 'utf-8'), str(error, 'utf-8'))
except (ConnectionResetError, paramiko.ssh_exception.SSHException):
self.client = None
raise BadSSHHost("SSH connection to host %s was reset" % (self.hostname,))
def __call__(self):
replaced = self.job.command.replace("{}", self.host.ip)
# Use subprocess.Popen because subprocess.run is not available in Python 3.4 used by Ubuntu Trusty
proc = subprocess.Popen(shlex.split(replaced),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
outs, errs = proc.communicate()
except subprocess.TimeoutExpired:
# The child process is not killed in case of timeout
proc.kill()
outs, errs = proc.communicate()
result = ExecutionResult(replaced, proc.returncode, str(outs, 'utf-8'),
str(errs, 'utf-8'))
save_output(self.job, self.host, result)
self.job.results.put((self.host, result))
return result