def status(url, proxy):
"""Return GRAM5 job state. Possible states:
PENDING
ACTIVE
FAILED
SUSPENDED
DONE
UNSUBMITTED
STAGE_IN
STAGE_OUT
"""
px = tempfile.NamedTemporaryFile()
px.write(proxy)
px.flush()
env = os.environ.copy()
env["X509_USER_PROXY"] = px.name
cmd = ["globusrun", "-fast-batch", "-batch", "-quiet", "-status", url]
pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True)
(out, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_submit_timeout'])
if out is None:
if pid.poll() is None:
os.kill(pid.pid, signal.SIGKILL)
raise RuntimeError("globusrun timed out")
if pid.returncode != 0:
raise RuntimeError("globusrun failed: " + err)
return out.strip()
def destroy(self):
"""
Clean up the resources used for this shell.
"""
for delegation_token_file in self._delegation_token_files:
try:
os.unlink(delegation_token_file)
except:
LOG.warning("Could not remove delegation token file %s" % delegation_token_file)
try:
self._delegation_token_files = None
self._write_buffer.close()
self._read_buffer.close()
os.close(self._fd)
os.close(self._child_fd)
try:
LOG.debug("Sending SIGKILL to process with PID %d" % (self.pid,))
os.kill(self.pid, signal.SIGKILL)
_, exitcode = os.waitpid(self.pid, 0)
msg = "%s - shell_id:%s pid:%d - Exited with status %d" % (self.username, self.shell_id, self.pid, exitcode)
except OSError:
msg = "%s - shell_id:%s pid:%d - Killed successfully" % (self.username, self.shell_id, self.pid,)
# This means the subprocess was already killed, which happens if the command was "quit"
# This can also happen if the waitpid call results in an error, which we don't care about.
LOG.info(msg)
SHELL_OUTPUT_LOGGER.info(msg)
SHELL_INPUT_LOGGER.info(msg)
finally:
self.destroyed = True
def credential_refresh(epr, proxy):
epr_file = tempfile.NamedTemporaryFile()
epr_file.write(epr)
epr_file.flush()
proxy_file = tempfile.NamedTemporaryFile()
proxy_file.write(proxy)
proxy_file.flush()
env = os.environ.copy()
env["X509_USER_PROXY"] = proxy_file.name
proc = Popen([GLOBUS_CREDENTIAL_REFRESH, "-e", epr_file.name,
"-x", proxy_file.name], env=env,
stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
(out, err) = tools.communicate_with_timeout(proc, timeout=pilot.spooler.config['job_submit_timeout'])
if out is None:
if proc.poll() is None:
os.kill(proc.pid, signal.SIGKILL)
raise TimeoutError("globus-credential-refresh timed out after %s seconds" % pilot.spooler.config['job_submit_timeout'])
if proc.returncode == 2:
raise ProxyExpiredError(err)
elif proc.returncode != 0:
raise GlobusError("Exit Code %d, %s" % (proc.returncode, err))
return True
def submit(rsl, resource, proxy):
px = tempfile.NamedTemporaryFile()
px.write(proxy)
px.flush()
jobfile = tempfile.NamedTemporaryFile()
jobfile.write(rsl)
jobfile.flush()
env = os.environ.copy()
env["X509_USER_PROXY"] = px.name
cmd = ["globusrun", "-fast-batch", "-batch", "-quiet",
"-file", jobfile.name, "-resource", resource]
pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True)
(out, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_submit_timeout'])
if out is None:
if pid.poll() is None:
os.kill(pid.pid, signal.SIGKILL)
raise RuntimeError("globusrun timed out")
if pid.returncode != 0:
raise RuntimeError("globusrun failed: " + err)
return out.strip()
def job_submit(rsl, args, proxy=None):
cmd = [GLOBUSRUN_WS, ]
cmd.extend(args)
rslfile = mktemp()
fd = open(rslfile, "w")
fd.write(rsl)
fd.close()
cmd.extend(['-f', rslfile])
proxy_filename = None
env = dict(os.environ)
if proxy:
proxy_filename = prepare_proxy(proxy)
env['X509_USER_PROXY'] = proxy_filename
pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True)
(epr, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_submit_timeout'])
if proxy_filename:
os.unlink(proxy_filename)
os.unlink(rslfile)
if epr is None:
if pid.poll() is None:
os.kill(pid.pid, signal.SIGKILL)
raise TimeoutError('globusrun-ws timed out after %s seconds' % pilot.spooler.config['job_submit_timeout'])
if pid.returncode != 0:
raise GlobusError(err)
return epr
def wsrf_query_epr(epr, proxy=None):
epr_file = tempfile.NamedTemporaryFile()
epr_file.write(epr)
epr_file.flush()
env = os.environ.copy()
if proxy is not None:
proxy_file = tempfile.NamedTemporaryFile()
proxy_file.write(proxy)
proxy_file.flush()
env["X509_USER_PROXY"] = proxy_file.name
proc = Popen([WSRF_QUERY, "-e", epr_file.name], env=env,
stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
(out, err) = tools.communicate_with_timeout(proc, timeout=pilot.spooler.config['job_query_timeout'])
if out is None:
if proc.poll() is None:
os.kill(proc.pid, signal.SIGKILL)
raise TimeoutError("wsrf-query timed out after %s seconds" % pilot.spooler.config['job_query_timeout'])
if proc.returncode == 2:
raise ProxyExpiredError(err)
elif proc.returncode != 0:
raise GlobusError("Exit Code %d, %s" % (proc.returncode, err))
return etree.parse(StringIO(out))
def job_kill(epr, proxy=None):
cmd = [GLOBUSRUN_WS, '-kill', '-host-authz', '-quiet', '-job-epr-file']
eprfile = mktemp()
fd = open(eprfile, "w")
fd.write(epr)
fd.close()
cmd.append(eprfile)
proxy_filename = None
env = dict(os.environ)
if proxy:
proxy_filename = prepare_proxy(proxy)
env['X509_USER_PROXY'] = proxy_filename
pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True)
(_, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_kill_timeout'])
if proxy_filename:
os.unlink(proxy_filename)
os.unlink(eprfile)
if pid.poll() is None:
os.kill(pid.pid, signal.SIGKILL)
raise TimeoutError('globusrun-ws timed out after %s seconds' % pilot.spooler.config['job_kill_timeout'])
if pid.returncode != 0:
raise GlobusError("RC: %d, %s" % (pid.returncode, err))
def destroy(self):
"""
Clean up the resources used for this shell.
"""
for delegation_token_file in self._delegation_token_files:
try:
os.unlink(delegation_token_file)
except:
LOG.warning("Could not remove delegation token file %s" % delegation_token_file)
try:
self._delegation_token_files = None
self._write_buffer.close()
self._read_buffer.close()
os.close(self._fd)
os.close(self._child_fd)
try:
LOG.debug("Sending SIGKILL to process with PID %d" % (self.pid,))
os.kill(self.pid, signal.SIGKILL)
_, exitcode = os.waitpid(self.pid, 0)
msg = "%s - shell_id:%s pid:%d - Exited with status %d" % (self.username, self.shell_id, self.pid, exitcode)
except OSError:
msg = "%s - shell_id:%s pid:%d - Killed successfully" % (self.username, self.shell_id, self.pid,)
# This means the subprocess was already killed, which happens if the command was "quit"
# This can also happen if the waitpid call results in an error, which we don't care about.
LOG.info(msg)
SHELL_OUTPUT_LOGGER.info(msg)
SHELL_INPUT_LOGGER.info(msg)
finally:
self.destroyed = True
def http_server():
try:
sock = eventlet.listen(('127.0.0.1', pilot.spooler.matchmaker_port))
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
atexit.register(sock.close)
except Exception, exc:
log.critical("Failed to create a web server socket: %s", str(exc))
os.kill(0, signal.SIGTERM)
def speak(self, cmdname, args=[], proxy=None, stdin=None, scratch=None):
"""
Return a tuple of (exit_code, stdout, stderr) for the given command.
If exit_code == -1 then the program timed out.
"""
pid = self.popen(cmdname, args, proxy, scratch)
timeout = float(self.config["timeout_" + cmdname])
(out, err) = tools.communicate_with_timeout(pid, stdin, timeout)
if out is None:
if pid.poll() is None:
os.kill(pid.pid, signal.SIGKILL)
return -1, out, err
return pid.returncode, out, err
def destroy(self):
"""
Clean up the resources used for this shell.
"""
try:
for delegation_token_file in self._delegation_token_files:
delegation_token_file.close()
self._delegation_token_files = None
self._write_buffer.close()
self._read_buffer.close()
os.close(self._fd)
os.close(self._child_fd)
try:
LOG.debug("Sending SIGKILL to process with PID %d" % (self.subprocess.pid,))
os.kill(self.subprocess.pid, signal.SIGKILL)
# We could try figure out which exit statuses are fine and which ones are errors.
# But that would be difficult to do correctly since os.wait might block.
except OSError:
pass # This means the subprocess was already killed, which happens if the command was "quit"
finally:
self.destroyed = True
def job_status(epr, proxy=None):
cmd = [WSRF_QUERY, ]
eprfile = mktemp()
cmd.extend(['-e', eprfile])
fd = open(eprfile, "w")
fd.write(epr)
fd.close()
proxy_filename = None
env = dict(os.environ)
if proxy:
proxy_filename = prepare_proxy(proxy)
env['X509_USER_PROXY'] = proxy_filename
pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True)
(xml, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_query_timeout'])
if proxy_filename:
os.unlink(proxy_filename)
os.unlink(eprfile)
if xml is None:
if pid.poll() is None:
os.kill(pid.pid, signal.SIGKILL)
raise TimeoutError('wsrf-query timed out after %s seconds' % pilot.spooler.config['job_query_timeout'])
if pid.returncode == 2:
raise ProxyExpiredError(err)
elif pid.returncode != 0:
raise GlobusError("Exit Code %d, %s" % (pid.returncode, err))
gstate = State(etree.fromstring(xml))
if gstate.state is None:
raise GlobusError("state element not present in wsrf-query response")
return gstate