def _eval_prompt (self, data, new_prompt=None) :
"""
This method will match the given data against the current prompt regex,
and expects to find an integer as match -- which is then returned, along
with all leading data, in a tuple
"""
with self.pty_shell.rlock :
try :
prompt = self.prompt
prompt_re = self.prompt_re
if new_prompt :
prompt = new_prompt
prompt_re = re.compile ("^(.*)%s\s*$" % prompt, re.DOTALL)
result = None
if not data :
raise se.NoSuccess ("cannot not parse prompt (%s), invalid data (%s)" \
% (prompt, data))
result = prompt_re.match (data)
if not result :
self.logger.debug ("could not parse prompt (%s) (%s)" % (prompt, data))
raise se.NoSuccess ("could not parse prompt (%s) (%s)" % (prompt, data))
txt = result.group (1)
ret = 0
if len (result.groups ()) != 2 :
if new_prompt :
self.logger.warn ("prompt does not capture exit value (%s)" % prompt)
# raise se.NoSuccess ("prompt does not capture exit value (%s)" % prompt)
else :
try :
ret = int(result.group (2))
except ValueError :
# apparently, this is not an integer. Print a warning, and
# assume success -- the calling entity needs to evaluate the
# remainder...
ret = 0
self.logger.warn ("prompt not suitable for error checks (%s)" % prompt)
txt += "\n%s" % result.group (2)
# if that worked, we can permanently set new_prompt
if new_prompt :
self.set_prompt (new_prompt)
return (ret, txt)
except Exception as e :
raise ptye.translate_exception (e, "Could not eval prompt")
def translate_exception(e, msg=None):
"""
In many cases, we should be able to roughly infer the exception cause
from the error message -- this is centrally done in this method. If
possible, it will return a new exception with a more concise error
message and appropriate exception type.
"""
if not issubclass(e.__class__, se.SagaException):
# we do not touch non-saga exceptions
return e
if not issubclass(e.__class__, se.NoSuccess):
# this seems to have a specific cause already, leave it alone
return e
cmsg = e._plain_message
if msg:
cmsg = "%s (%s)" % (cmsg, msg)
lmsg = cmsg.lower()
if 'could not resolve hostname' in lmsg:
e = se.BadParameter(cmsg)
elif 'connection timed out' in lmsg:
e = se.BadParameter(cmsg)
elif 'connection refused' in lmsg:
e = se.BadParameter(cmsg)
elif 'auth' in lmsg:
e = se.AuthorizationFailed(cmsg)
elif 'pass' in lmsg:
e = se.AuthenticationFailed(cmsg)
elif 'ssh_exchange_identification' in lmsg:
e = se.AuthenticationFailed(
"too frequent login attempts, or sshd misconfiguration: %s" % cmsg)
elif 'denied' in lmsg:
e = se.PermissionDenied(cmsg)
elif 'shared connection' in lmsg:
e = se.NoSuccess("Insufficient system resources: %s" % cmsg)
elif 'pty allocation' in lmsg:
e = se.NoSuccess("Insufficient system resources: %s" % cmsg)
elif 'Connection to master closed' in lmsg:
e = se.NoSuccess(
"Connection failed (insufficient system resources?): %s" % cmsg)
return e
def _eval_prompt(self, data, new_prompt=None):
"""
This method will match the given data against the current prompt regex,
and expects to find an integer as match -- which is then returned, along
with all leading data, in a tuple
"""
with self.pty_shell.rlock:
try:
prompt = self.prompt
prompt_re = self.prompt_re
if new_prompt:
prompt = new_prompt
prompt_re = re.compile("^(.*)%s\s*$" % prompt, re.DOTALL)
result = None
if not data:
raise se.NoSuccess ("cannot not parse prompt (%s), invalid data (%s)" \
% (prompt, data))
result = prompt_re.match(data)
if not result:
self.logger.debug("could not parse prompt (%s) (%s)" %
(prompt, data))
raise se.NoSuccess("could not parse prompt (%s) (%s)" %
(prompt, data))
if len(result.groups()) != 2:
self.logger.debug(
"prompt does not capture exit value (%s)" % prompt)
raise se.NoSuccess(
"prompt does not capture exit value (%s)" % prompt)
txt = result.group(1)
ret = int(result.group(2))
# if that worked, we can permanently set new_prompt
if new_prompt:
self.set_prompt(new_prompt)
return (ret, txt)
except Exception as e:
raise ptye.translate_exception(e, "Could not eval prompt")
def initialize(self):
""" initialize the shell connection. """
with self.pty_shell.rlock:
if self.initialized:
self.logger.warn("initialization race")
return
if self.posix:
# run a POSIX compatible shell, usually /bin/sh, in interactive mode
# also, turn off tty echo
command_shell = "exec /bin/sh -i"
# use custom shell if so requested
if 'shell' in self.options and self.options['shell']:
command_shell = "exec %s" % self.options['shell']
self.logger.info("custom command shell: %s" %
command_shell)
self.logger.debug("running command shell: %s" %
command_shell)
self.pty_shell.write(" stty -echo ; unset HISTFILE ; %s\n" %
command_shell)
# make sure this worked, and that we find the prompt. We use
# a versatile prompt pattern to account for the custom shell case.
_, out = self.find([self.prompt])
# make sure this worked, and that we find the prompt. We use
# a versatile prompt pattern to account for the custom shell case.
try:
# set and register new prompt
self.run_async(" unset PROMPT_COMMAND ; " +
" unset HISTFILE ; " +
"PS1='PROMPT-$?->'; " + "PS2=''; " +
"export PS1 PS2 2>&1 >/dev/null\n")
self.set_prompt(new_prompt="PROMPT-(\d+)->$")
self.logger.debug("got new shell prompt")
except Exception as e:
raise se.NoSuccess(
"Shell startup on target host failed: %s" % e)
try:
# got a command shell, finally!
# for local shells, we now change to the current working
# directory. Remote shells will remain in the default pwd
# (usually $HOME).
if sumisc.host_is_local(surl.Url(self.url).host):
pwd = os.getcwd()
self.run_sync(' cd %s' % pwd)
except Exception as e:
# We will ignore any errors.
self.logger.warning("local cd to %s failed" % pwd)
self.pty_shell.flush()
self.initialized = True
self.finalized = False
def __init__(self, url, session=None, logger=None, opts=None, posix=True):
if logger: self.logger = logger
else: self.logger = rul.getLogger('saga', 'PTYShell')
if session: self.session = session
else: self.session = ss.Session(default=True)
if opts: self.options = opts
else: self.options = dict()
self.logger.debug("PTYShell init %s" % self)
self.url = url # describes the shell to run
self.posix = posix # /bin/sh compatible?
self.latency = 0.0 # set by factory
self.cp_slave = None # file copy channel
self.initialized = False
self.pty_id = PTYShell._pty_id
PTYShell._pty_id += 1
self.cfg = self.session.get_config('saga.utils.pty')
# get prompt pattern from options, config, or use default
if 'prompt_pattern' in self.options:
self.prompt = self.options['prompt_pattern']
elif 'prompt_pattern' in self.cfg:
self.prompt = self.cfg['prompt_pattern'].get_value()
else:
self.prompt = DEFAULT_PROMPT
self.prompt_re = re.compile("^(.*?)%s" % self.prompt, re.DOTALL)
self.logger.info("PTY prompt pattern: %s" % self.prompt)
# we need a local dir for file staging caches. At this point we use
# $HOME, but should make this configurable (FIXME)
self.base = os.environ['HOME'] + '/.saga/adaptors/shell/'
try:
os.makedirs(self.base)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(self.base):
pass
else:
raise se.NoSuccess("could not create staging dir: %s" % e)
self.factory = supsf.PTYShellFactory()
self.pty_info = self.factory.initialize(self.url,
self.session,
self.prompt,
self.logger,
posix=self.posix)
self.pty_shell = self.factory.run_shell(self.pty_info)
self._trace('init : %s' % self.pty_shell.command)
self.initialize()
def re_raise(self):
""" :todo: describe me
:note: if job failed, that will re-raise an exception describing
why, if that exists. Otherwise, the call does nothing.
"""
if self.state == FAILED:
raise se.NoSuccess("job stderr: %s" % self.get_stderr_string())
else:
return
def get_exception(self, ttype=None):
""" :todo: describe me
:note: if job failed, that will get an exception describing
why, if that exists. Otherwise, the call returns None.
"""
if self.state == FAILED:
return se.NoSuccess("job stderr: %s" % self.get_stderr_string())
else:
return None
def __init__(self, url, session=None, logger=None, init=None, opts={}):
# print 'new pty shell to %s' % url
if logger: self.logger = logger
else: self.logger = rul.getLogger('saga', 'PTYShell')
if session: self.session = session
else: self.session = ss.Session(default=True)
self.logger.debug("PTYShell init %s" % self)
self.url = url # describes the shell to run
self.init = init # call after reconnect
self.opts = opts # options...
self.latency = 0.0 # set by factory
self.cp_slave = None # file copy channel
self.initialized = False
# get prompt pattern from config
self.cfg = self.session.get_config('saga.utils.pty')
if 'prompt_pattern' in self.cfg:
self.prompt = self.cfg['prompt_pattern'].get_value()
self.prompt_re = re.compile("^(.*?)%s" % self.prompt, re.DOTALL)
else:
self.prompt = "[\$#%>\]]\s*$"
self.prompt_re = re.compile("^(.*?)%s" % self.prompt, re.DOTALL)
self.logger.info("PTY prompt pattern: %s" % self.prompt)
# we need a local dir for file staging caches. At this point we use
# $HOME, but should make this configurable (FIXME)
self.base = os.environ['HOME'] + '/.saga/adaptors/shell/'
try:
os.makedirs(self.base)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(self.base):
pass
else:
raise se.NoSuccess("could not create staging dir: %s" % e)
self.factory = supsf.PTYShellFactory()
self.pty_info = self.factory.initialize(self.url, self.session,
self.prompt, self.logger)
self.pty_shell = self.factory.run_shell(self.pty_info)
self.initialize()
def initialize (self) :
""" initialize the shell connection. """
with self.pty_shell.rlock :
if self.initialized :
self.logger.warn ("initialization race")
return
if self.posix :
# run a POSIX compatible shell, usually /bin/sh, in interactive mode
# also, turn off tty echo
command_shell = "exec /bin/sh -i"
# use custom shell if so requested
if 'shell' in self.options and self.options['shell'] :
command_shell = "exec %s" % self.options['shell']
self.logger.info ("custom command shell: %s" % command_shell)
self.logger.debug ("running command shell: %s" % command_shell)
self.pty_shell.write (" stty -echo ; %s\n" % command_shell)
# make sure this worked, and that we find the prompt. We use
# a versatile prompt pattern to account for the custom shell case.
_, out = self.find ([self.prompt])
# make sure this worked, and that we find the prompt. We use
# a versatile prompt pattern to account for the custom shell case.
try :
# set and register new prompt
self.run_async ( " set HISTFILE=$HOME/.saga_history;"
+ " PS1='PROMPT-$?->';"
+ " PS2='';"
+ " PROMPT_COMMAND='';"
+ " export PS1 PS2 2>&1 >/dev/null;"
+ " cd $HOME 2>&1 >/dev/null\n")
self.set_prompt (new_prompt="PROMPT-(\d+)->$")
self.logger.debug ("got new shell prompt")
except Exception as e :
raise se.NoSuccess ("Shell startup on target host failed: %s" % e)
# got a command shell, finally!
self.pty_shell.flush ()
self.initialized = True
self.finalized = False
def get_adaptor(self, adaptor_name):
''' Return the adaptor module's ``Adaptor`` class for the given adaptor
name.
This method is used if adaptor or API object implementation need to
interact with other adaptors.
'''
for ctype in self._adaptor_registry.keys():
for schema in self._adaptor_registry[ctype].keys():
for info in self._adaptor_registry[ctype][schema]:
if (info['adaptor_name'] == adaptor_name):
return info['adaptor_instance']
error_msg = "No adaptor named '%s' found" % adaptor_name
self._logger.error(error_msg)
raise se.NoSuccess(error_msg)
def write(self, data, nolog=False):
"""
This method will repeatedly attempt to push the given data into the
child's stdin pipe, until it succeeds to write all data.
"""
with self.rlock:
if not self.alive(recover=False):
raise ptye.translate_exception (se.NoSuccess ("cannot write to dead process (%s)" \
% self.cache[-256:]))
try:
log = self._hide_data(data, nolog)
log = log.replace('\n', '\\n')
log = log.replace('\r', '')
if len(log) > _DEBUG_MAX:
self.logger.debug ("write: [%5d] [%5d] (%s ... %s)" \
% (self.parent_in, len(data), log[:30], log[-30:]))
else:
self.logger.debug ("write: [%5d] [%5d] (%s)" \
% (self.parent_in, len(data), log))
# attempt to write forever -- until we succeeed
while data:
# check if the pty pipe is ready for data
_, wlist, _ = select.select([], [self.parent_in], [],
_POLLDELAY)
for f in wlist:
# write will report the number of written bytes
size = os.write(f, data)
# otherwise, truncate by written data, and try again
data = data[size:]
if data:
self.logger.info("write: [%5d] [%5d]" % (f, size))
except Exception as e:
raise ptye.translate_exception(
e, "write to process failed (%s)" % e)
def initialize(self):
with self.rlock:
# already initialized?
if self.child:
self.logger.warn("initialization race: %s" %
' '.join(self.command))
return
self.logger.info("running: %s" % ' '.join(self.command))
# create the child
try:
self.child, self.child_fd = pty.fork()
except Exception as e:
raise se.NoSuccess ("Could not run (%s): %s" \
% (' '.join (self.command), e))
if not self.child:
# this is the child
try:
# all I/O set up, have a pty (*fingers crossed*), lift-off!
os.execvpe(self.command[0], self.command, os.environ)
except OSError as e:
self.logger.error ("Could not execute (%s): %s" \
% (' '.join (self.command), e))
sys.exit(-1)
else:
# this is the parent
new = termios.tcgetattr(self.child_fd)
new[3] = new[3] & ~termios.ECHO
termios.tcsetattr(self.child_fd, termios.TCSANOW, new)
self.parent_in = self.child_fd
self.parent_out = self.child_fd
def raise_return_exception(method, spectype, result):
if no_return_check:
# disable this!
return
stack = extract_stack()
for f in stack:
if 'saga/utils/signatures.py' in f[0]:
break
frame = f
msg = "\nSignature Mismatch\n"
msg += " in function : %s\n" % (frame[2])
msg += " in file : %s +%s\n" % (frame[0], frame[1])
msg += " on line : %s\n" % (frame[3])
msg += " method : %s\n" % (method.__name__)
msg += " returned type : %s\n" % (type_name(result))
msg += " instead of : %s\n" % (type_name(spectype))
msg += " This is an internal SAGA-Python error!"
raise se.NoSuccess(msg)
def read(self, size=0, timeout=0, _force=False):
"""
read some data from the child. By default, the method reads whatever is
available on the next read, up to _CHUNKSIZE, but other read sizes can
be specified.
The method will return whatever data it has at timeout::
timeout == 0 : return the content of the first successful read, with
whatever data up to 'size' have been found.
timeout < 0 : return after first read attempt, even if no data have
been available.
If no data are found, the method returns an empty string (not None).
This method will not fill the cache, but will just read whatever data it
needs (FIXME).
Note: the returned lines do *not* get '\\\\r' stripped.
"""
with self.rlock:
found_eof = False
try:
# start the timeout timer right now. Note that even if timeout is
# short, and child.poll is slow, we will nevertheless attempt at least
# one read...
start = time.time()
ret = ""
# read until we have enough data, or hit timeout ceiling...
while True:
# first, lets see if we still have data in the cache we can return
if len(self.cache):
if not size:
ret = self.cache
self.cache = ""
self.tail += ret
self.tail = self.tail[-256:]
return ret
# we don't even need all of the cache
elif size <= len(self.cache):
ret = self.cache[:size]
self.cache = self.cache[size:]
self.tail += ret
self.tail = self.tail[-256:]
return ret
# otherwise we need to read some more data, right?
# idle wait 'til the next data chunk arrives, or 'til _POLLDELAY
rlist, _, _ = select.select([self.parent_out], [], [],
_POLLDELAY)
# got some data?
for f in rlist:
# read whatever we still need
readsize = _CHUNKSIZE
if size:
readsize = size - len(ret)
buf = os.read(f, _CHUNKSIZE)
if len(buf) == 0 and sys.platform == 'darwin':
self.logger.debug("read : MacOS EOF")
self.finalize()
found_eof = True
raise se.NoSuccess("unexpected EOF (%s)" %
self.tail)
self.cache += buf.replace('\r', '')
log = buf.replace('\r', '')
log = log.replace('\n', '\\n')
# print "buf: --%s--" % buf
# print "log: --%s--" % log
if len(log) > _DEBUG_MAX:
self.logger.debug ("read : [%5d] [%5d] (%s ... %s)" \
% (f, len(log), log[:30], log[-30:]))
else:
self.logger.debug ("read : [%5d] [%5d] (%s)" \
% (f, len(log), log))
# for c in log :
# print '%s' % c
# lets see if we still got any data in the cache we can return
if len(self.cache):
if not size:
ret = self.cache
self.cache = ""
self.tail += ret
self.tail = self.tail[-256:]
return ret
# we don't even need all of the cache
elif size <= len(self.cache):
ret = self.cache[:size]
self.cache = self.cache[size:]
self.tail += ret
self.tail = self.tail[-256:]
return ret
# at this point, we do not have sufficient data -- only
# return on timeout
if timeout == 0:
# only return if we have data
if len(self.cache):
ret = self.cache
self.cache = ""
self.tail += ret
self.tail = self.tail[-256:]
return ret
elif timeout < 0:
# return of we have data or not
ret = self.cache
self.cache = ""
self.tail += ret
self.tail = self.tail[-256:]
return ret
else: # timeout > 0
# return if timeout is reached
now = time.time()
if (now - start) > timeout:
ret = self.cache
self.cache = ""
self.tail += ret
self.tail = self.tail[-256:]
return ret
except Exception as e:
if found_eof:
raise e
raise se.NoSuccess ("read from process failed '%s' : (%s)" \
% (e, self.tail))
def wait(self):
"""
blocks forever until the child finishes on its own, or is getting
killed.
Actully, we might just as well try to figure out what is going on on the
remote end of things -- so we read the pipe until the child dies...
"""
output = ""
# yes, for ever and ever...
while True:
try:
output += self.read()
except:
break
# yes, for ever and ever...
while True:
if not self.child:
# this was quick ;-)
return output
# we need to lock, as the SIGCHLD will only arrive once
with self.rlock:
# hey, kiddo, whats up?
try:
wpid, wstat = os.waitpid(self.child, 0)
except OSError as e:
if e.errno == errno.ECHILD:
# child disappeared
self.exit_code = None
self.exit_signal = None
self.finalize()
return output
# no idea what happened -- it is likely bad
raise se.NoSuccess("waitpid failed on wait")
# did we get a note about child termination?
if 0 == wpid:
# nope, all is well - carry on
continue
# Yes, we got a note.
# Well, maybe the child fooled us and is just playing dead?
if os.WIFSTOPPED (wstat) or \
os.WIFCONTINUED (wstat) :
# we don't care if someone stopped/resumed the child -- that is up
# to higher powers. For our purposes, the child is alive. Ha!
continue
# not stopped, poor thing... - soooo, what happened?? But hey,
# either way, its dead -- make sure it stays dead, to avoid
# zombie apocalypse...
self.child = None
self.finalize(wstat=wstat)
return output
def _initialize_pty(self, pty_shell, info, posix=None):
# posix: only for posix shells we use prompt triggers. sftp for example
# does not deal well with triggers (no printf).
with self.rlock:
# import pprint
# pprint.pprint (info)
shell_pass = info['pass']
key_pass = info['key_pass']
prompt = info['prompt']
logger = info['logger']
latency = info['latency']
timeout = info['ssh_timeout']
pty_shell.latency = latency
if posix == None:
posix = info['posix']
# if we did not see a decent prompt within 'delay' time, something
# went wrong. Try to prompt a prompt (duh!) Delay should be
# minimum 0.1 second (to avoid flooding of local shells), and at
# maximum 1 second (to keep startup time reasonable)
# most one second. We try to get within that range with 10*latency.
delay = min(1.0, max(0.1, 10 * latency))
try:
prompt_patterns = [
"[Pp]assword:\s*$", # password prompt
"Enter passphrase for .*:\s*$", # passphrase prompt
"Token_Response.*:\s*$", # passtoken prompt
"Enter PASSCODE:$", # RSA SecureID
"want to continue connecting", # hostkey confirmation
".*HELLO_\\d+_SAGA$", # prompt detection helper
prompt
] # greedy native shell prompt
# use a very aggressive, but portable prompt setting scheme.
# Error messages may appear for tcsh and others. Excuse
# non-posix shells
if posix:
pty_shell.write(
" export PROMPT_COMMAND='' PS1='$' ; set prompt='$'\n")
# find a prompt
n, match = pty_shell.find(prompt_patterns, delay)
# this loop will run until we finally find the shell prompt, or
# if we think we have tried enough and give up. On success
# we'll try to set a different prompt, and when we found that,
# too, we exit the loop and are be ready to running shell
# commands.
retries = 0
retry_trigger = True
used_trigger = False
found_trigger = ""
time_start = time.time()
while True:
# --------------------------------------------------------------
if n == None:
# we found none of the prompts, yet, and need to try
# again. But to avoid hanging on invalid prompts, we
# print 'HELLO_x_SAGA', and search for that one, too.
# We actually do 'printf HELLO_%d_SAGA x' so that the
# pattern only appears in the result, not in the
# command...
if time.time() - time_start > timeout:
raise se.NoSuccess(
"Could not detect shell prompt (timeout)")
# make sure we retry a finite time...
retries += 1
if not retry_trigger:
# just waiting for the *right* trigger or prompt,
# don't need new ones...
continue
if posix:
# use a very aggressive, but portable prompt setting scheme
pty_shell.write(
" export PROMPT_COMMAND='' PS1='$' > /dev/null 2>&1 || set prompt='$'\n"
)
pty_shell.write(
" printf 'HELLO_%%d_SAGA\\n' %d\n" % retries)
used_trigger = True
# FIXME: consider better timeout
n, match = pty_shell.find(prompt_patterns, delay)
# --------------------------------------------------------------
elif n == 0:
logger.info("got password prompt")
if not shell_pass:
raise se.AuthenticationFailed ("prompted for unknown password (%s)" \
% match)
pty_shell.write("%s\n" % shell_pass, nolog=True)
n, match = pty_shell.find(prompt_patterns, delay)
# --------------------------------------------------------------
elif n == 1:
logger.info("got passphrase prompt : %s" % match)
start = string.find(match, "'", 0)
end = string.find(match, "'", start + 1)
if start == -1 or end == -1:
raise se.AuthenticationFailed(
"could not extract key name (%s)" % match)
key = match[start + 1:end]
if not key in key_pass:
raise se.AuthenticationFailed ("prompted for unknown key password (%s)" \
% key)
pty_shell.write("%s\n" % key_pass[key], nolog=True)
n, match = pty_shell.find(prompt_patterns, delay)
# --------------------------------------------------------------
elif n == 2 or n == 3:
logger.info("got token prompt")
import getpass
token = getpass.getpass("enter token: ")
pty_shell.write("%s\n" % token.strip(), nolog=True)
n, match = pty_shell.find(prompt_patterns, delay)
# --------------------------------------------------------------
elif n == 4:
logger.info("got hostkey prompt")
pty_shell.write("yes\n")
n, match = pty_shell.find(prompt_patterns, delay)
# --------------------------------------------------------------
elif n == 5:
# one of the trigger commands got through -- we can now
# hope to find the prompt (or the next trigger...)
logger.debug("got shell prompt trigger (%s) (%s)" %
(n, match))
found_trigger = match
retry_trigger = False
n, match = pty_shell.find(prompt_patterns, delay)
continue
# --------------------------------------------------------------
elif n == 6:
logger.debug("got initial shell prompt (%s) (%s)" %
(n, match))
if retries:
if used_trigger:
# we already sent triggers -- so this match is only
# useful if saw the *correct* shell prompt trigger
# first
trigger = "HELLO_%d_SAGA" % retries
if not trigger in found_trigger:
logger.debug ("waiting for prompt trigger %s: (%s) (%s)" \
% (trigger, n, match))
# but more retries won't help...
retry_trigger = False
attempts = 0
n = None
while not n:
attempts += 1
n, match = pty_shell.find(
prompt_patterns, delay)
if not n:
if attempts == 1:
if posix:
pty_shell.write(
" printf 'HELLO_%%d_SAGA\\n' %d\n"
% retries)
if attempts > 100:
raise se.NoSuccess(
"Could not detect shell prompt (timeout)"
)
continue
logger.debug("Got initial shell prompt (%s) (%s)" %
(n, match))
# we are done waiting for a prompt
break
except Exception as e:
logger.exception(e)
raise ptye.translate_exception(e)
def run_copy_to (self, src, tgt, cp_flags="") :
"""
This initiates a slave copy connection. Src is interpreted as local
path, tgt as path on the remote host.
Now, this is ugly when over sftp: sftp supports recursive copy, and
wildcards, all right -- but for recursive copies, it wants the target
dir to exist -- so, we have to check if the local src is a dir, and if
so, we first create the target before the copy. Worse, for wildcards we
have to do a local expansion, and the to do the same for each entry...
"""
with self.pty_shell.rlock :
self._trace ("copy to : %s -> %s" % (src, tgt))
self.pty_shell.flush ()
info = self.pty_info
repl = dict ({'src' : src,
'tgt' : tgt,
'cp_flags' : '' # cp_flags # TODO: needs to be "translated" for specific backend
}.items () + info.items ())
# at this point, we do have a valid, living master
s_cmd = info['scripts'][info['copy_mode']]['copy_to'] % repl
s_in = info['scripts'][info['copy_mode']]['copy_to_in'] % repl
posix = info['scripts'][info['copy_mode']]['copy_is_posix']
if not s_in :
# this code path does not use an interactive shell for copy --
# so the above s_cmd is all we want to run, really. We get
# do not use the chached cp_slave in this case, but just run the
# command. We do not have a list of transferred files though,
# yet -- that should be parsed from the proc output.
cp_proc = supp.PTYProcess (s_cmd)
out = cp_proc.wait ()
if cp_proc.exit_code :
raise ptye.translate_exception (se.NoSuccess ("file copy failed: %s" % out))
return list()
# this code path uses an interactive shell to transfer files, of
# some form, such as sftp. Get the shell cp_slave from cache, and
# run the actual copy command.
if not self.cp_slave :
self._trace ("get cp slave")
self.cp_slave = self.factory.get_cp_slave (s_cmd, info, posix)
self.cp_slave.flush ()
if 'sftp' in s_cmd :
# prepare target dirs for recursive copy, if needed
import glob
src_list = glob.glob (src)
for s in src_list :
if os.path.isdir (s) :
prep = "mkdir %s/%s\n" % (tgt, os.path.basename (s))
# TODO: this doesn't deal with multiple levels of creation
self.cp_slave.flush()
self.cp_slave.write("%s\n" % prep)
self.cp_slave.find(['[\$\>\]]\s*$'], -1)
# TODO: check return values
if cp_flags == sfs.CREATE_PARENTS and os.path.split(tgt)[0]:
# TODO: this needs to be numeric and checking the flag
prep = "mkdir %s\n" % os.path.dirname(tgt)
# TODO: this doesn't deal with multiple levels of creation
self.cp_slave.flush()
self.cp_slave.write("%s\n" % prep)
self.cp_slave.find(['[\$\>\]]\s*$'], -1)
# TODO: check return values
self.cp_slave.flush()
_ = self.cp_slave.write("%s\n" % s_in)
_, out = self.cp_slave.find(['[\$\>\]]\s*$'], -1)
# FIXME: we don't really get exit codes from copy
# if self.cp_slave.exit_code != 0 :
# raise se.NoSuccess._log (info['logger'], "file copy failed: %s" % str(out))
if 'Invalid flag' in out :
raise se.NoSuccess._log (info['logger'], "sftp version not supported (%s)" % str(out))
if 'No such file or directory' in out :
raise se.DoesNotExist._log (info['logger'], "file copy failed: %s" % str(out))
if 'is not a directory' in out :
raise se.BadParameter._log (info['logger'], "File copy failed: %s" % str(out))
if 'sftp' in s_cmd :
if 'not found' in out :
raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out)
# we interpret the first word on the line as name of src file -- we
# will return a list of those
lines = out.split ('\n')
files = []
for line in lines :
elems = line.split (' ', 2)
if elems :
f = elems[0]
# remove quotes
if f :
if f[ 0] in ["'", '"', '`'] : f = f[1: ]
if f[-1] in ["'", '"', '`'] : f = f[ :-1]
# ignore empty lines
if f :
files.append (f)
info['logger'].debug ("copy done: %s" % files)
return files
def set_prompt (self, new_prompt) :
"""
:type new_prompt: string
:param new_prompt: a regular expression matching the shell prompt
The new_prompt regex is expected to be a regular expression with one set
of catching brackets, which MUST return the previous command's exit
status. This method will send a newline to the client, and expects to
find the prompt with the exit value '0'.
As a side effect, this method will discard all previous data on the pty,
thus effectively flushing the pty output.
By encoding the exit value in the command prompt, we safe one roundtrip.
The prompt on Posix compliant shells can be set, for example, via::
PS1='PROMPT-$?->'; export PS1
The newline in the example above allows to nicely anchor the regular
expression, which would look like::
PROMPT-(\d+)->$
The regex is compiled with 're.DOTALL', so the dot character matches
all characters, including line breaks. Be careful not to match more
than the exact prompt -- otherwise, a prompt search will swallow stdout
data. For example, the following regex::
PROMPT-(.+)->$
would capture arbitrary strings, and would thus match *all* of::
PROMPT-0->ls
data/ info
PROMPT-0->
and thus swallow the ls output...
Note that the string match *before* the prompt regex is non-gready -- if
the output contains multiple occurrences of the prompt, only the match
up to the first occurence is returned.
"""
def escape (txt) :
pat = re.compile(r'\x1b[^m]*m')
return pat.sub ('', txt)
with self.pty_shell.rlock :
old_prompt = self.prompt
self.prompt = new_prompt
self.prompt_re = re.compile ("^(.*?)%s\s*$" % self.prompt, re.DOTALL)
retries = 0
triggers = 0
while True :
try :
# make sure we have a non-zero waiting delay (default to
# 1 second)
delay = 10 * self.latency
if not delay :
delay = 1.0
# FIXME: how do we know that _PTY_TIMOUT suffices? In particular if
# we actually need to flush...
fret, match = self.pty_shell.find ([self.prompt], delay)
if fret == None :
retries += 1
if retries > 10 :
self.prompt = old_prompt
raise se.BadParameter ("Cannot use new prompt, parsing failed (10 retries)")
self.pty_shell.write ("\n")
self.logger.debug ("sent prompt trigger again (%d)" % retries)
triggers += 1
continue
# found a match -- lets see if this is working now...
ret, _ = self._eval_prompt (match)
if ret != 0 :
self.prompt = old_prompt
raise se.BadParameter ("could not parse exit value (%s)" \
% match)
# prompt looks valid...
break
except Exception as e :
self.prompt = old_prompt
raise ptye.translate_exception (e, "Could not set shell prompt")
# got a valid prompt -- but we have to sync the output again in
# those cases where we had to use triggers to actually get the
# prompt
if triggers > 0 :
self.run_async (' printf "SYNCHRONIZE_PROMPT\n"')
# FIXME: better timout value?
fret, match = self.pty_shell.find (["SYNCHRONIZE_PROMPT"], timeout=10.0)
if fret == None :
# not find prompt after blocking? BAD! Restart the shell
self.finalize (kill_pty=True)
raise se.NoSuccess ("Could not synchronize prompt detection")
self.find_prompt ()
def run_copy_from (self, src, tgt, cp_flags="") :
"""
This initiates a slave copy connection. Src is interpreted as path on
the remote host, tgt as local path.
We have to do the same mkdir trick as for the run_copy_to, but here we
need to expand wildcards on the *remote* side :/
"""
with self.pty_shell.rlock :
self._trace ("copy from: %s -> %s" % (src, tgt))
self.pty_shell.flush ()
info = self.pty_info
repl = dict ({'src' : src,
'tgt' : tgt,
'cp_flags' : cp_flags}.items() + info.items ())
# at this point, we do have a valid, living master
s_cmd = info['scripts'][info['copy_mode']]['copy_from'] % repl
s_in = info['scripts'][info['copy_mode']]['copy_from_in'] % repl
posix = info['scripts'][info['copy_mode']]['copy_is_posix']
if not s_in :
# this code path does not use an interactive shell for copy --
# so the above s_cmd is all we want to run, really. We get
# do not use the chached cp_slave in this case, but just run the
# command. We do not have a list of transferred files though,
# yet -- that should be parsed from the proc output.
cp_proc = supp.PTYProcess (s_cmd)
cp_proc.wait ()
if cp_proc.exit_code :
raise ptye.translate_exception (se.NoSuccess ("file copy failed: exit code %s" % cp_proc.exit_code))
return list()
if not self.cp_slave :
self._trace ("get cp slave")
self.cp_slave = self.factory.get_cp_slave (s_cmd, info, posix)
self.cp_slave.flush ()
prep = ""
if 'sftp' in s_cmd :
# prepare target dirs for recursive copy, if needed
self.cp_slave.write (" ls %s\n" % src)
_, out = self.cp_slave.find (["^sftp> "], -1)
src_list = out[1].split('\n')
for s in src_list :
if os.path.isdir (s) :
prep += "lmkdir %s/%s\n" % (tgt, os.path.basename (s))
self.cp_slave.flush ()
_ = self.cp_slave.write ("%s%s\n" % (prep, s_in))
_, out = self.cp_slave.find (['[\$\>\]] *$'], -1)
# FIXME: we don't really get exit codes from copy
# if self.cp_slave.exit_code != 0 :
# raise se.NoSuccess._log (info['logger'], "file copy failed: %s" % out)
if 'Invalid flag' in out :
raise se.NoSuccess._log (info['logger'], "sftp version not supported (%s)" % out)
if 'No such file or directory' in out :
raise se.DoesNotExist._log (info['logger'], "file copy failed: %s" % out)
if 'is not a directory' in out :
raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out)
if 'sftp' in s_cmd :
if 'not found' in out :
raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out)
# we run copy with -v, so get a list of files which have been copied
# -- we parse that list and return it. we interpret the *second*
# word on the line as name of src file.
lines = out.split ('\n')
files = []
for line in lines :
elems = line.split (' ', 3)
if elems and len(elems) > 1 and elems[0] == 'Fetching' :
f = elems[1]
# remove quotes
if f :
if f[ 0] in ["'", '"', '`'] : f = f[1: ]
if f[-1] in ["'", '"', '`'] : f = f[ :-1]
# ignore empty lines
if f :
files.append (f)
info['logger'].debug ("copy done: %s" % files)
return files
