def read_from_remote(self, src): """ :type src: string :param src: path to source file to staged from The src path is not an URL, but expected to be a path relative to the shell's URL. """ try: # FIXME: make this relative to the shell's pwd? Needs pwd in # prompt, and updating pwd state on every find_prompt. # first, write data into a tmp file fname = self.base + "/staging.%s" % id(self) _ = self.stage_from_remote(src, fname) fhandle = open(fname, 'r') out = fhandle.read() fhandle.close() os.remove(fname) return out except Exception as e: raise ptye.translate_exception(e)
def read_from_remote (self, src) : """ :type src: string :param src: path to source file to staged from The src path is not an URL, but expected to be a path relative to the shell's URL. """ try : # self._trace ("read : %s" % src) # FIXME: make this relative to the shell's pwd? Needs pwd in # prompt, and updating pwd state on every find_prompt. # first, write data into a tmp file fhandle, fname = tempfile.mkstemp(suffix='.tmp', prefix='rs_pty_staging_') _ = self.stage_from_remote (src, fname) os.close(fhandle) os.system('sync') # WTF? Why do I need this? fhandle2 = open(fname, 'r') out = fhandle2.read() fhandle2.close() os.remove(fname) return out except Exception as e : raise ptye.translate_exception (e)
def read_from_remote (self, src) : """ :type src: string :param src: path to source file to staged from The src path is not an URL, but expected to be a path relative to the shell's URL. """ try : # FIXME: make this relative to the shell's pwd? Needs pwd in # prompt, and updating pwd state on every find_prompt. # first, write data into a tmp file fname = self.base + "/staging.%s" % id(self) _ = self.factory.run_copy_from (self.pty_info, src, fname) fhandle = open (fname, 'r') out = fhandle.read () fhandle.close () os.remove (fname) return out except Exception as e : raise ptye.translate_exception (e)
def run_async (self, command) : """ Run a shell command, but don't wait for prompt -- just return. It is up to caller to eventually search for the prompt again (see :func:`find_prompt`. Meanwhile, the caller can interact with the called command, via the I/O channels. :type command: string :param command: shell command to run. For async execution, we don't care if the command is doing i/o redirection or not. """ with self.pty_shell.rlock : # we expect the shell to be in 'ground state' when running an asyncronous # command -- thus we can check if the shell is alive before doing so, # and restart if needed if not self.pty_shell.alive (recover=True) : raise se.IncorrectState ("Cannot run command:\n%s" \ % self.pty_shell.autopsy ()) try : command = command.strip () self.send ("%s\n" % command) except Exception as e : raise ptye.translate_exception (e)
def find_prompt (self, timeout=_PTY_TIMEOUT) : """ If run_async was called, a command is running on the shell. find_prompt can be used to collect its output up to the point where the shell prompt re-appears (i.e. when the command finishes). Note that this method blocks until the command finishes. Future versions of this call may add a timeout parameter. """ with self.pty_shell.rlock : try : match = None fret = None while fret == None : fret, match = self.pty_shell.find ([self.prompt], timeout) # self.logger.debug ("find prompt '%s' in '%s'" % (self.prompt, match)) ret, txt = self._eval_prompt (match) return (ret, txt) except Exception as e : raise ptye.translate_exception (e)
def find_prompt (self) : """ If run_async was called, a command is running on the shell. find_prompt can be used to collect its output up to the point where the shell prompt re-appears (i.e. when the command finishes). Note that this method blocks until the command finishes. Future versions of this call may add a timeout parameter. """ with self.pty_shell.rlock : try : match = None fret = None while fret == None : fret, match = self.pty_shell.find ([self.prompt], _PTY_TIMEOUT) # self.logger.debug ("find prompt '%s' in '%s'" % (self.prompt, match)) ret, txt = self._eval_prompt (match) return (ret, txt) except Exception as e : raise ptye.translate_exception (e)
def run_async (self, command) : """ Run a shell command, but don't wait for prompt -- just return. It is up to caller to eventually search for the prompt again (see :func:`find_prompt`. Meanwhile, the caller can interact with the called command, via the I/O channels. :type command: string :param command: shell command to run. For async execution, we don't care if the command is doing i/o redirection or not. """ with self.pty_shell.rlock : self._trace ("run async : %s" % command) self.pty_shell.flush () # we expect the shell to be in 'ground state' when running an asyncronous # command -- thus we can check if the shell is alive before doing so, # and restart if needed if not self.pty_shell.alive (recover=True) : raise se.IncorrectState ("Cannot run command:\n%s" \ % self.pty_shell.autopsy ()) try : command = command.strip () self.send ("%s\n" % command) except Exception as e : raise ptye.translate_exception (e)
def _eval_prompt (self, data, new_prompt=None) : """ This method will match the given data against the current prompt regex, and expects to find an integer as match -- which is then returned, along with all leading data, in a tuple """ with self.pty_shell.rlock : try : prompt = self.prompt prompt_re = self.prompt_re if new_prompt : prompt = new_prompt prompt_re = re.compile ("^(.*)%s\s*$" % prompt, re.DOTALL) result = None if not data : raise se.NoSuccess ("cannot not parse prompt (%s), invalid data (%s)" \ % (prompt, data)) result = prompt_re.match (data) if not result : self.logger.debug ("could not parse prompt (%s) (%s)" % (prompt, data)) raise se.NoSuccess ("could not parse prompt (%s) (%s)" % (prompt, data)) txt = result.group (1) ret = 0 if len (result.groups ()) != 2 : if new_prompt : self.logger.warn ("prompt does not capture exit value (%s)" % prompt) # raise se.NoSuccess ("prompt does not capture exit value (%s)" % prompt) else : try : ret = int(result.group (2)) except ValueError : # apparently, this is not an integer. Print a warning, and # assume success -- the calling entity needs to evaluate the # remainder... ret = 0 self.logger.warn ("prompt not suitable for error checks (%s)" % prompt) txt += "\n%s" % result.group (2) # if that worked, we can permanently set new_prompt if new_prompt : self.set_prompt (new_prompt) return (ret, txt) except Exception as e : raise ptye.translate_exception (e, "Could not eval prompt")
def write(self, data, nolog=False): """ This method will repeatedly attempt to push the given data into the child's stdin pipe, until it succeeds to write all data. """ with self.rlock: if not self.alive(recover=False): raise ptye.translate_exception (se.NoSuccess ("cannot write to dead process (%s)" \ % self.cache[-256:])) try: log = self._hide_data(data, nolog) log = log.replace('\n', '\\n') log = log.replace('\r', '') if len(log) > _DEBUG_MAX: self.logger.debug ("write: [%5d] [%5d] (%s ... %s)" \ % (self.parent_in, len(data), log[:30], log[-30:])) else: self.logger.debug ("write: [%5d] [%5d] (%s)" \ % (self.parent_in, len(data), log)) # attempt to write forever -- until we succeeed while data: # check if the pty pipe is ready for data _, wlist, _ = select.select([], [self.parent_in], [], _POLLDELAY) for f in wlist: # write will report the number of written bytes size = os.write(f, data) # otherwise, truncate by written data, and try again data = data[size:] if data: self.logger.info("write: [%5d] [%5d]" % (f, size)) except Exception as e: raise ptye.translate_exception( e, "write to process failed (%s)" % e)
def write (self, data, nolog=False) : """ This method will repeatedly attempt to push the given data into the child's stdin pipe, until it succeeds to write all data. """ with self.rlock : if not self.alive (recover=False) : raise ptye.translate_exception (se.NoSuccess ("cannot write to dead process (%s) [%5d]" \ % (self.cache[-256:], self.parent_in))) try : log = self._hide_data (data, nolog) log = log.replace ('\n', '\\n') log = log.replace ('\r', '') if len(log) > _DEBUG_MAX : self.logger.debug ("write: [%5d] [%5d] (%s ... %s)" \ % (self.parent_in, len(data), log[:30], log[-30:])) else : self.logger.debug ("write: [%5d] [%5d] (%s)" \ % (self.parent_in, len(data), log)) # attempt to write forever -- until we succeeed while data : # check if the pty pipe is ready for data _, wlist, _ = select.select ([], [self.parent_in], [], _POLLDELAY) for f in wlist : # write will report the number of written bytes size = os.write (f, data) # otherwise, truncate by written data, and try again data = data[size:] if data : self.logger.info ("write: [%5d] [%5d]" % (f, size)) except Exception as e : raise ptye.translate_exception (e, "write to process failed (%s)" % e)
def __init__ (self, command, logger=None) : """ The class constructor, which runs (execvpe) command in a separately forked process. The bew process will inherit the environment of the application process. :type command: string or list of strings :param command: The given command is what is run as a child, and fed/drained via pty pipes. If given as string, command is split into an array of strings, using :func:`shlex.split`. :type logger: :class:`radical.utils.logger.Logger` instance :param logger: logger stream to send status messages to. """ self._debug = False self.logger = logger if not self.logger : self.logger = rul.getLogger ('saga', 'PTYProcess') self.logger.debug ("PTYProcess init %s" % self) if isinstance (command, basestring) : command = shlex.split (command) if not isinstance (command, list) : raise se.BadParameter ("PTYProcess expects string or list command") if len(command) < 1 : raise se.BadParameter ("PTYProcess expects non-empty command") self.rlock = ru.RLock ("pty process %s" % command) self.command = command # list of strings too run() self.cache = "" # data cache self.tail = "" # tail of data data cache for error messages self.child = None # the process as created by subprocess.Popen self.ptyio = None # the process' io channel, from pty.fork() self.exit_code = None # child died with code (may be revived) self.exit_signal = None # child kill by signal (may be revived) self.recover_max = 3 # TODO: make configure option. This does not self.recover_attempts = 0 # apply for recovers triggered by gc_timeout! try : self.initialize () except Exception as e : raise ptye.translate_exception (e, "pty or process creation failed")
def find (self, patterns, timeout=-1) : """ Note that this method blocks until pattern is found in the shell I/O. """ with self.pty_shell.rlock : try : return self.pty_shell.find (patterns, timeout=timeout) except Exception as e : raise ptye.translate_exception (e)
def alive (self, recover=False) : """ The shell is assumed to be alive if the shell processes lives. Attempt to restart shell if recover==True """ with self.pty_shell.rlock : try : return self.pty_shell.alive (recover) except Exception as e : raise ptye.translate_exception (e)
def _eval_prompt(self, data, new_prompt=None): """ This method will match the given data against the current prompt regex, and expects to find an integer as match -- which is then returned, along with all leading data, in a tuple """ with self.pty_shell.rlock: try: prompt = self.prompt prompt_re = self.prompt_re if new_prompt: prompt = new_prompt prompt_re = re.compile("^(.*)%s\s*$" % prompt, re.DOTALL) result = None if not data: raise se.NoSuccess ("cannot not parse prompt (%s), invalid data (%s)" \ % (prompt, data)) result = prompt_re.match(data) if not result: self.logger.debug("could not parse prompt (%s) (%s)" % (prompt, data)) raise se.NoSuccess("could not parse prompt (%s) (%s)" % (prompt, data)) if len(result.groups()) != 2: self.logger.debug( "prompt does not capture exit value (%s)" % prompt) raise se.NoSuccess( "prompt does not capture exit value (%s)" % prompt) txt = result.group(1) ret = int(result.group(2)) # if that worked, we can permanently set new_prompt if new_prompt: self.set_prompt(new_prompt) return (ret, txt) except Exception as e: raise ptye.translate_exception(e, "Could not eval prompt")
def __init__(self, command, logger=None): """ The class constructor, which runs (execvpe) command in a separately forked process. The bew process will inherit the environment of the application process. :type command: string or list of strings :param command: The given command is what is run as a child, and fed/drained via pty pipes. If given as string, command is split into an array of strings, using :func:`shlex.split`. :type logger: :class:`radical.utils.logger.Logger` instance :param logger: logger stream to send status messages to. """ self.logger = logger if not self.logger: self.logger = rul.getLogger('saga', 'PTYProcess') self.logger.debug("PTYProcess init %s" % self) if isinstance(command, basestring): command = shlex.split(command) if not isinstance(command, list): raise se.BadParameter("PTYProcess expects string or list command") if len(command) < 1: raise se.BadParameter("PTYProcess expects non-empty command") self.rlock = ru.RLock("pty process %s" % command) self.command = command # list of strings too run() self.cache = "" # data cache self.tail = "" # tail of data data cache for error messages self.child = None # the process as created by subprocess.Popen self.ptyio = None # the process' io channel, from pty.fork() self.exit_code = None # child died with code (may be revived) self.exit_signal = None # child kill by signal (may be revived) self.recover_max = 3 # TODO: make configure option. This does not self.recover_attempts = 0 # apply for recovers triggered by gc_timeout! try: self.initialize() except Exception as e: raise ptye.translate_exception(e, "pty or process creation failed")
def send (self, data) : """ send data to the shell. No newline is appended! """ with self.pty_shell.rlock : if not self.pty_shell.alive (recover=False) : raise se.IncorrectState ("Cannot send data:\n%s" \ % self.pty_shell.autopsy ()) try : self.pty_shell.write ("%s" % data) except Exception as e : raise ptye.translate_exception (e)
def _eval_prompt (self, data, new_prompt=None) : """ This method will match the given data against the current prompt regex, and expects to find an integer as match -- which is then returned, along with all leading data, in a tuple """ with self.pty_shell.rlock : try : prompt = self.prompt prompt_re = self.prompt_re if new_prompt : prompt = new_prompt prompt_re = re.compile ("^(.*)%s\s*$" % prompt, re.DOTALL) result = None if not data : raise se.NoSuccess ("cannot not parse prompt (%s), invalid data (%s)" \ % (prompt, data)) result = prompt_re.match (data) if not result : self.logger.debug ("could not parse prompt (%s) (%s)" % (prompt, data)) raise se.NoSuccess ("could not parse prompt (%s) (%s)" % (prompt, data)) if len (result.groups ()) != 2 : self.logger.debug ("prompt does not capture exit value (%s)" % prompt) raise se.NoSuccess ("prompt does not capture exit value (%s)" % prompt) txt = result.group (1) ret = int(result.group (2)) # if that worked, we can permanently set new_prompt if new_prompt : self.set_prompt (new_prompt) return (ret, txt) except Exception as e : raise ptye.translate_exception (e, "Could not eval prompt")
def stage_from_remote (self, src, tgt, cp_flags="") : """ :type src: string :param tgt: path to source file to stage from. The tgt path is not an URL, but expected to be a path relative to the shell's URL. :type tgt: string :param src: path of local target file to stage to. The tgt path is not an URL, but expected to be a path relative to the current working directory. """ # FIXME: make this relative to the shell's pwd? Needs pwd in # prompt, and updating pwd state on every find_prompt. try : return self.factory.run_copy_from (self.pty_info, src, tgt, cp_flags) except Exception as e : raise ptye.translate_exception (e)
def stage_from_remote(self, src, tgt, cp_flags=""): """ :type src: string :param tgt: path to source file to stage from. The tgt path is not an URL, but expected to be a path relative to the shell's URL. :type tgt: string :param src: path of local target file to stage to. The tgt path is not an URL, but expected to be a path relative to the current working directory. """ # FIXME: make this relative to the shell's pwd? Needs pwd in # prompt, and updating pwd state on every find_prompt. try: return self.run_copy_from(src, tgt, cp_flags) except Exception as e: raise ptye.translate_exception(e)
def write_to_remote (self, src, tgt) : """ :type src: string :param src: data to be staged into the target file :type tgt: string :param tgt: path to target file to staged to The tgt path is not an URL, but expected to be a path relative to the shell's URL. The content of the given string is pasted into a file (specified by tgt) on the remote system. If that file exists, it is overwritten. A NoSuccess exception is raised if writing the file was not possible (missing permissions, incorrect path, etc.). """ try : # self._trace ("write : %s -> %s" % (src, tgt)) # FIXME: make this relative to the shell's pwd? Needs pwd in # prompt, and updating pwd state on every find_prompt. # first, write data into a tmp file fname = self.base + "/staging.%s" % id(self) fhandle = open (fname, 'wb') fhandle.write (src) fhandle.flush () fhandle.close () ret = self.stage_to_remote (fname, tgt) os.remove (fname) return ret except Exception as e : raise ptye.translate_exception (e)
def write_to_remote(self, src, tgt): """ :type src: string :param src: data to be staged into the target file :type tgt: string :param tgt: path to target file to staged to The tgt path is not an URL, but expected to be a path relative to the shell's URL. The content of the given string is pasted into a file (specified by tgt) on the remote system. If that file exists, it is overwritten. A NoSuccess exception is raised if writing the file was not possible (missing permissions, incorrect path, etc.). """ try: # self._trace ("write : %s -> %s" % (src, tgt)) # FIXME: make this relative to the shell's pwd? Needs pwd in # prompt, and updating pwd state on every find_prompt. # first, write data into a tmp file fname = self.base + "/staging.%s" % id(self) fhandle = open(fname, 'wb') fhandle.write(src) fhandle.flush() fhandle.close() ret = self.stage_to_remote(fname, tgt) os.remove(fname) return ret except Exception as e: raise ptye.translate_exception(e)
def run_copy_from (self, src, tgt, cp_flags="") : """ This initiates a slave copy connection. Src is interpreted as path on the remote host, tgt as local path. We have to do the same mkdir trick as for the run_copy_to, but here we need to expand wildcards on the *remote* side :/ """ self._trace ("copy from: %s -> %s" % (src, tgt)) with self.pty_shell.rlock : info = self.pty_info repl = dict ({'src' : src, 'tgt' : tgt, 'cp_flags' : cp_flags}.items ()+ info.items ()) # at this point, we do have a valid, living master s_cmd = info['scripts'][info['copy_type']]['copy_from'] % repl s_in = info['scripts'][info['copy_type']]['copy_from_in'] % repl if not s_in : # this code path does not use an interactive shell for copy -- # so the above s_cmd is all we want to run, really. We get # do not use the chached cp_slave in this case, but just run the # command. We do not have a list of transferred files though, # yet -- that should be parsed from the proc output. cp_proc = supp.PTYProcess (s_cmd) cp_proc.wait () if cp_proc.exit_code : raise ptye.translate_exception (se.NoSuccess ("file copy failed: %s" % out)) return list() if not self.cp_slave : self._trace ("get cp slave") self.cp_slave = self.factory.get_cp_slave (s_cmd, info) prep = "" if 'sftp' in s_cmd : # prepare target dirs for recursive copy, if needed self.cp_slave.write (" ls %s\n" % src) _, out = self.cp_slave.find (["^sftp> "], -1) src_list = out[1].split ('/n') for s in src_list : if os.path.isdir (s) : prep += "lmkdir %s/%s\n" % (tgt, os.path.basename (s)) _ = self.cp_slave.write ("%s%s\n" % (prep, s_in)) _, out = self.cp_slave.find (['[\$\>\]] *$'], -1) # FIXME: we don't really get exit codes from copy # if self.cp_slave.exit_code != 0 : # raise se.NoSuccess._log (info['logger'], "file copy failed: %s" % out) if 'Invalid flag' in out : raise se.NoSuccess._log (info['logger'], "sftp version not supported (%s)" % out) if 'No such file or directory' in out : raise se.DoesNotExist._log (info['logger'], "file copy failed: %s" % out) if 'is not a directory' in out : raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out) if 'sftp' in s_cmd : if 'not found' in out : raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out) # we run copy with -v, so get a list of files which have been copied # -- we parse that list and return it. we interpret the *second* # word on the line as name of src file. lines = out.split ('\n') files = [] for line in lines : elems = line.split (' ', 3) if elems and len(elems) > 1 and elems[0] == 'Fetching' : f = elems[1] # remove quotes if f : if f[ 0] in ["'", '"', '`'] : f = f[1: ] if f[-1] in ["'", '"', '`'] : f = f[ :-1] # ignore empty lines if f : files.append (f) info['logger'].debug ("copy done: %s" % files) return files
def find(self, patterns, timeout=0): """ This methods reads bytes from the child process until a string matching any of the given patterns is found. If that is found, all read data are returned as a string, up to (and including) the match. Note that pattern can match an empty string, and the call then will return just that, an empty string. If all patterns end with matching a newline, this method is effectively matching lines -- but note that '$' will also match the end of the (currently available) data stream. The call actually returns a tuple, containing the index of the matching pattern, and the string up to the match as described above. If no pattern is found before timeout, the call returns (None, None). Negative timeouts will block until a match is found Note that the pattern are interpreted with the re.M (multi-line) and re.S (dot matches all) regex flags. Performance: the call is doing repeated string regex searches over whatever data it finds. On complex regexes, and large data, and small read buffers, this method can be expensive. Note: the returned data get '\\\\r' stripped. Note: ansi-escape sequences are also stripped before matching, but are kept in the returned data. """ def escape(txt): pat = re.compile(r'\x1b[^m]*m') return pat.sub('', txt) _debug = False with self.rlock: try: start = time.time() # startup timestamp ret = [] # array of read lines patts = [] # compiled patterns data = self.cache # initial data to check self.cache = "" if not data: # empty cache? data = self.read(timeout=_POLLDELAY) # pre-compile the given pattern, to speed up matching for pattern in patterns: patts.append(re.compile(pattern, re.MULTILINE | re.DOTALL)) # we wait forever -- there are two ways out though: data matches # a pattern, or timeout passes while True: # skip non-lines if not data: data += self.read(timeout=_POLLDELAY) if _debug: print ">>%s<<" % data escaped = escape(data) if _debug: print 'data ==%s==' % data if _debug: print 'escaped ==%s==' % escaped # check current data for any matching pattern for n in range(0, len(patts)): match = patts[n].search(escaped) if _debug: print "==%s==" % patterns[n] if _debug: print match if match: # a pattern matched the current data: return a tuple of # pattern index and matching data. The remainder of the # data is cached. ret = escaped[0:match.end()] self.cache = escaped[match.end():] if _debug: print "~~match!~~ %s" % escaped[match.start( ):match.end()] if _debug: print "~~match!~~ %s" % (len(escaped)) if _debug: print "~~match!~~ %s" % (str(match.span())) if _debug: print "~~match!~~ %s" % (ret) return (n, ret.replace('\r', '')) # if a timeout is given, and actually passed, return # a non-match and a copy of the data we looked at if timeout == 0: return (None, str(escaped)) if timeout > 0: now = time.time() if (now - start) > timeout: self.cache = escaped return (None, str(escaped)) # no match yet, still time -- read more data data += self.read(timeout=_POLLDELAY) except se.NoSuccess as e: raise ptye.translate_exception(e, "(%s)" % data)
def _initialize_pty (self, pty_shell, info, is_shell=False) : # is_shell: only for shells we use prompt triggers. sftp for example # does not deal well with triggers (no printf). with self.rlock : shell_pass = info['pass'] key_pass = info['key_pass'] logger = info['logger'] latency = info['latency'] pty_shell.latency = latency # if we did not see a decent prompt within 'delay' time, something # went wrong. Try to prompt a prompt (duh!) Delay should be # minimum 0.1 second (to avoid flooding of local shells), and at # maximum 1 second (to keep startup time reasonable) # most one second. We try to get within that range with 100*latency. delay = min (1.0, max (0.1, 50 * latency)) try : prompt_patterns = ["[Pp]assword:\s*$", # password prompt "Enter passphrase for .*:\s*$", # passphrase prompt "want to continue connecting", # hostkey confirmation ".*HELLO_\\d+_SAGA$", # prompt detection helper "^(.*[\$#%>\]])\s*$"] # greedy native shell prompt # find a prompt # use a very aggressive, but portable prompt setting scheme pty_shell.write ("export PS1='>' >& /dev/null || set prompt='>'\n") n, match = pty_shell.find (prompt_patterns, delay) # this loop will run until we finally find the shell prompt, or # if we think we have tried enough and give up. On success # we'll try to set a different prompt, and when we found that, # too, we exit the loop and are be ready to running shell # commands. retries = 0 retry_trigger = True used_trigger = False found_trigger = "" while True : # -------------------------------------------------------------- if n == None : # we found none of the prompts, yet, and need to try # again. But to avoid hanging on invalid prompts, we # print 'HELLO_x_SAGA', and search for that one, too. # We actually do 'printf HELLO_%d_SAGA x' so that the # pattern only appears in the result, not in the # command... if retries > 100 : raise se.NoSuccess ("Could not detect shell prompt (timeout)") if not retry_trigger : # just waiting for the *right* trigger or prompt, # don't need new ones... continue retries += 1 if is_shell : # use a very aggressive, but portable prompt setting scheme pty_shell.write ("export PS1='>' >& /dev/null || set prompt='>'\n") pty_shell.write ("printf 'HELLO_%%d_SAGA\\n' %d\n" % retries) used_trigger = True # FIXME: consider better timeout n, match = pty_shell.find (prompt_patterns, delay) # -------------------------------------------------------------- elif n == 0 : logger.info ("got password prompt") if not shell_pass : raise se.AuthenticationFailed ("prompted for unknown password (%s)" \ % match) pty_shell.write ("%s\n" % shell_pass) n, match = pty_shell.find (prompt_patterns, delay) # -------------------------------------------------------------- elif n == 1 : logger.info ("got passphrase prompt : %s" % match) start = string.find (match, "'", 0) end = string.find (match, "'", start+1) if start == -1 or end == -1 : raise se.AuthenticationFailed ("could not extract key name (%s)" % match) key = match[start+1:end] if not key in key_pass : raise se.AuthenticationFailed ("prompted for unknown key password (%s)" \ % key) pty_shell.write ("%s\n" % key_pass[key]) n, match = pty_shell.find (prompt_patterns, delay) # -------------------------------------------------------------- elif n == 2 : logger.info ("got hostkey prompt") pty_shell.write ("yes\n") n, match = pty_shell.find (prompt_patterns, delay) # -------------------------------------------------------------- elif n == 3 : # one of the trigger commands got through -- we can now # hope to find the prompt (or the next trigger...) logger.debug ("got shell prompt trigger (%s) (%s)" % (n, match)) found_trigger = match retry_trigger = False n, match = pty_shell.find (prompt_patterns, delay) continue # -------------------------------------------------------------- elif n == 4 : logger.debug ("got initial shell prompt (%s) (%s)" % (n, match)) if retries : if used_trigger : # we already sent triggers -- so this match is only # useful if saw the *correct* shell prompt trigger # first trigger = "HELLO_%d_SAGA" % retries if not trigger in found_trigger : logger.debug ("waiting for prompt trigger %s: (%s) (%s)" \ % (trigger, n, match)) # but more retries won't help... retry_trigger = False n = None while not n : n, match = pty_shell.find (prompt_patterns, delay) continue logger.info ("Got initial shell prompt (%s) (%s)" \ % (n, match)) # we are done waiting for a prompt break except Exception as e : raise ptye.translate_exception (e)
def _initialize_pty(self, pty_shell, info, posix=None): # posix: only for posix shells we use prompt triggers. sftp for example # does not deal well with triggers (no printf). with self.rlock: # import pprint # pprint.pprint (info) shell_pass = info['pass'] key_pass = info['key_pass'] prompt = info['prompt'] logger = info['logger'] latency = info['latency'] timeout = info['ssh_timeout'] pty_shell.latency = latency if posix == None: posix = info['posix'] # if we did not see a decent prompt within 'delay' time, something # went wrong. Try to prompt a prompt (duh!) Delay should be # minimum 0.1 second (to avoid flooding of local shells), and at # maximum 1 second (to keep startup time reasonable) # most one second. We try to get within that range with 10*latency. delay = min(1.0, max(0.1, 10 * latency)) try: prompt_patterns = [ "[Pp]assword:\s*$", # password prompt "Enter passphrase for .*:\s*$", # passphrase prompt "Token_Response.*:\s*$", # passtoken prompt "Enter PASSCODE:$", # RSA SecureID "want to continue connecting", # hostkey confirmation ".*HELLO_\\d+_SAGA$", # prompt detection helper prompt ] # greedy native shell prompt # use a very aggressive, but portable prompt setting scheme. # Error messages may appear for tcsh and others. Excuse # non-posix shells if posix: pty_shell.write( " export PROMPT_COMMAND='' PS1='$' ; set prompt='$'\n") # find a prompt n, match = pty_shell.find(prompt_patterns, delay) # this loop will run until we finally find the shell prompt, or # if we think we have tried enough and give up. On success # we'll try to set a different prompt, and when we found that, # too, we exit the loop and are be ready to running shell # commands. retries = 0 retry_trigger = True used_trigger = False found_trigger = "" time_start = time.time() while True: # -------------------------------------------------------------- if n == None: # we found none of the prompts, yet, and need to try # again. But to avoid hanging on invalid prompts, we # print 'HELLO_x_SAGA', and search for that one, too. # We actually do 'printf HELLO_%d_SAGA x' so that the # pattern only appears in the result, not in the # command... if time.time() - time_start > timeout: raise se.NoSuccess( "Could not detect shell prompt (timeout)") # make sure we retry a finite time... retries += 1 if not retry_trigger: # just waiting for the *right* trigger or prompt, # don't need new ones... continue if posix: # use a very aggressive, but portable prompt setting scheme pty_shell.write( " export PROMPT_COMMAND='' PS1='$' > /dev/null 2>&1 || set prompt='$'\n" ) pty_shell.write( " printf 'HELLO_%%d_SAGA\\n' %d\n" % retries) used_trigger = True # FIXME: consider better timeout n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 0: logger.info("got password prompt") if not shell_pass: raise se.AuthenticationFailed ("prompted for unknown password (%s)" \ % match) pty_shell.write("%s\n" % shell_pass, nolog=True) n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 1: logger.info("got passphrase prompt : %s" % match) start = string.find(match, "'", 0) end = string.find(match, "'", start + 1) if start == -1 or end == -1: raise se.AuthenticationFailed( "could not extract key name (%s)" % match) key = match[start + 1:end] if not key in key_pass: raise se.AuthenticationFailed ("prompted for unknown key password (%s)" \ % key) pty_shell.write("%s\n" % key_pass[key], nolog=True) n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 2 or n == 3: logger.info("got token prompt") import getpass token = getpass.getpass("enter token: ") pty_shell.write("%s\n" % token.strip(), nolog=True) n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 4: logger.info("got hostkey prompt") pty_shell.write("yes\n") n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 5: # one of the trigger commands got through -- we can now # hope to find the prompt (or the next trigger...) logger.debug("got shell prompt trigger (%s) (%s)" % (n, match)) found_trigger = match retry_trigger = False n, match = pty_shell.find(prompt_patterns, delay) continue # -------------------------------------------------------------- elif n == 6: logger.debug("got initial shell prompt (%s) (%s)" % (n, match)) if retries: if used_trigger: # we already sent triggers -- so this match is only # useful if saw the *correct* shell prompt trigger # first trigger = "HELLO_%d_SAGA" % retries if not trigger in found_trigger: logger.debug ("waiting for prompt trigger %s: (%s) (%s)" \ % (trigger, n, match)) # but more retries won't help... retry_trigger = False attempts = 0 n = None while not n: attempts += 1 n, match = pty_shell.find( prompt_patterns, delay) if not n: if attempts == 1: if posix: pty_shell.write( " printf 'HELLO_%%d_SAGA\\n' %d\n" % retries) if attempts > 100: raise se.NoSuccess( "Could not detect shell prompt (timeout)" ) continue logger.debug("Got initial shell prompt (%s) (%s)" % (n, match)) # we are done waiting for a prompt break except Exception as e: logger.exception(e) raise ptye.translate_exception(e)
def run_copy_to (self, src, tgt, cp_flags="") : """ This initiates a slave copy connection. Src is interpreted as local path, tgt as path on the remote host. Now, this is ugly when over sftp: sftp supports recursive copy, and wildcards, all right -- but for recursive copies, it wants the target dir to exist -- so, we have to check if the local src is a dir, and if so, we first create the target before the copy. Worse, for wildcards we have to do a local expansion, and the to do the same for each entry... """ self._trace ("copy to : %s -> %s" % (src, tgt)) with self.pty_shell.rlock : info = self.pty_info repl = dict ({'src' : src, 'tgt' : tgt, 'cp_flags' : cp_flags}.items () + info.items ()) # at this point, we do have a valid, living master s_cmd = info['scripts'][info['copy_type']]['copy_to'] % repl s_in = info['scripts'][info['copy_type']]['copy_to_in'] % repl if not s_in : # this code path does not use an interactive shell for copy -- # so the above s_cmd is all we want to run, really. We get # do not use the chached cp_slave in this case, but just run the # command. We do not have a list of transferred files though, # yet -- that should be parsed from the proc output. cp_proc = supp.PTYProcess (s_cmd) out = cp_proc.wait () if cp_proc.exit_code : raise ptye.translate_exception (se.NoSuccess ("file copy failed: %s" % out)) return list() # this code path uses an interactive shell to transfer files, of # some form, such as sftp. Get the shell cp_slave from cache, and # run the actual copy command. if not self.cp_slave : self._trace ("get cp slave") self.cp_slave = self.factory.get_cp_slave (s_cmd, info) prep = "" if 'sftp' in s_cmd : # prepare target dirs for recursive copy, if needed import glob src_list = glob.glob (src) for s in src_list : if os.path.isdir (s) : prep += "mkdir %s/%s\n" % (tgt, os.path.basename (s)) _ = self.cp_slave.write ("%s%s\n" % (prep, s_in)) _, out = self.cp_slave.find (['[\$\>\]]\s*$'], -1) _, out = self.cp_slave.find (['[\$\>\]]\s*$'], 1.0) # FIXME: we don't really get exit codes from copy # if self.cp_slave.exit_code != 0 : # raise se.NoSuccess._log (info['logger'], "file copy failed: %s" % str(out)) if 'Invalid flag' in out : raise se.NoSuccess._log (info['logger'], "sftp version not supported (%s)" % str(out)) if 'No such file or directory' in out : raise se.DoesNotExist._log (info['logger'], "file copy failed: %s" % str(out)) if 'is not a directory' in out : raise se.BadParameter._log (info['logger'], "File copy failed: %s" % str(out)) if 'sftp' in s_cmd : if 'not found' in out : raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out) # we interpret the first word on the line as name of src file -- we # will return a list of those lines = out.split ('\n') files = [] for line in lines : elems = line.split (' ', 2) if elems : f = elems[0] # remove quotes if f : if f[ 0] in ["'", '"', '`'] : f = f[1: ] if f[-1] in ["'", '"', '`'] : f = f[ :-1] # ignore empty lines if f : files.append (f) info['logger'].debug ("copy done: %s" % files) return files
def find (self, patterns, timeout=0) : """ This methods reads bytes from the child process until a string matching any of the given patterns is found. If that is found, all read data are returned as a string, up to (and including) the match. Note that pattern can match an empty string, and the call then will return just that, an empty string. If all patterns end with matching a newline, this method is effectively matching lines -- but note that '$' will also match the end of the (currently available) data stream. The call actually returns a tuple, containing the index of the matching pattern, and the string up to the match as described above. If no pattern is found before timeout, the call returns (None, None). Negative timeouts will block until a match is found Note that the pattern are interpreted with the re.M (multi-line) and re.S (dot matches all) regex flags. Performance: the call is doing repeated string regex searches over whatever data it finds. On complex regexes, and large data, and small read buffers, this method can be expensive. Note: the returned data get '\\\\r' stripped. Note: ansi-escape sequences are also stripped before matching, but are kept in the returned data. """ def escape (txt) : pat = re.compile(r'\x1b[^m]*m') return pat.sub ('', txt) _debug = False with self.rlock : try : start = time.time () # startup timestamp ret = [] # array of read lines patts = [] # compiled patterns data = self.cache # initial data to check self.cache = "" if not data : # empty cache? data = self.read (timeout=_POLLDELAY) # pre-compile the given pattern, to speed up matching for pattern in patterns : patts.append (re.compile (pattern, re.MULTILINE | re.DOTALL)) # we wait forever -- there are two ways out though: data matches # a pattern, or timeout passes while True : # skip non-lines if not data : data += self.read (timeout=_POLLDELAY) if _debug : print ">>%s<<" % data escaped = escape (data) if _debug : print 'data ==%s==' % data if _debug : print 'escaped ==%s==' % escaped # check current data for any matching pattern for n in range (0, len(patts)) : escaped = data # escaped = escape (data) # print '-- 1 --%s--' % data # print '-- 2 --%s--' % escaped match = patts[n].search (escaped) if _debug : print "==%s==" % patterns[n] if _debug : print match if match : # a pattern matched the current data: return a tuple of # pattern index and matching data. The remainder of the # data is cached. ret = escaped[0:match.end()] self.cache = escaped[match.end():] if _debug : print "~~match!~~ %s" % escaped[match.start():match.end()] if _debug : print "~~match!~~ %s" % (len(escaped)) if _debug : print "~~match!~~ %s" % (str(match.span())) if _debug : print "~~match!~~ %s" % (ret) return (n, ret.replace('\r', '')) # if a timeout is given, and actually passed, return # a non-match and a copy of the data we looked at if timeout == 0 : return (None, str(escaped)) if timeout > 0 : now = time.time () if (now-start) > timeout : self.cache = escaped return (None, str(escaped)) # no match yet, still time -- read more data data += self.read (timeout=_POLLDELAY) except se.NoSuccess as e : raise ptye.translate_exception (e, "(%s)" % data)
def run_copy_to (self, src, tgt, cp_flags="") : """ This initiates a slave copy connection. Src is interpreted as local path, tgt as path on the remote host. Now, this is ugly when over sftp: sftp supports recursive copy, and wildcards, all right -- but for recursive copies, it wants the target dir to exist -- so, we have to check if the local src is a dir, and if so, we first create the target before the copy. Worse, for wildcards we have to do a local expansion, and the to do the same for each entry... """ with self.pty_shell.rlock : self._trace ("copy to : %s -> %s" % (src, tgt)) self.pty_shell.flush () info = self.pty_info repl = dict ({'src' : src, 'tgt' : tgt, 'cp_flags' : '' # cp_flags # TODO: needs to be "translated" for specific backend }.items () + info.items ()) # at this point, we do have a valid, living master s_cmd = info['scripts'][info['copy_mode']]['copy_to'] % repl s_in = info['scripts'][info['copy_mode']]['copy_to_in'] % repl posix = info['scripts'][info['copy_mode']]['copy_is_posix'] if not s_in : # this code path does not use an interactive shell for copy -- # so the above s_cmd is all we want to run, really. We get # do not use the chached cp_slave in this case, but just run the # command. We do not have a list of transferred files though, # yet -- that should be parsed from the proc output. cp_proc = supp.PTYProcess (s_cmd) out = cp_proc.wait () if cp_proc.exit_code : raise ptye.translate_exception (se.NoSuccess ("file copy failed: %s" % out)) return list() # this code path uses an interactive shell to transfer files, of # some form, such as sftp. Get the shell cp_slave from cache, and # run the actual copy command. if not self.cp_slave : self._trace ("get cp slave") self.cp_slave = self.factory.get_cp_slave (s_cmd, info, posix) self.cp_slave.flush () if 'sftp' in s_cmd : # prepare target dirs for recursive copy, if needed import glob src_list = glob.glob (src) for s in src_list : if os.path.isdir (s) : prep = "mkdir %s/%s\n" % (tgt, os.path.basename (s)) # TODO: this doesn't deal with multiple levels of creation self.cp_slave.flush() self.cp_slave.write("%s\n" % prep) self.cp_slave.find(['[\$\>\]]\s*$'], -1) # TODO: check return values if cp_flags == sfs.CREATE_PARENTS and os.path.split(tgt)[0]: # TODO: this needs to be numeric and checking the flag prep = "mkdir %s\n" % os.path.dirname(tgt) # TODO: this doesn't deal with multiple levels of creation self.cp_slave.flush() self.cp_slave.write("%s\n" % prep) self.cp_slave.find(['[\$\>\]]\s*$'], -1) # TODO: check return values self.cp_slave.flush() _ = self.cp_slave.write("%s\n" % s_in) _, out = self.cp_slave.find(['[\$\>\]]\s*$'], -1) # FIXME: we don't really get exit codes from copy # if self.cp_slave.exit_code != 0 : # raise se.NoSuccess._log (info['logger'], "file copy failed: %s" % str(out)) if 'Invalid flag' in out : raise se.NoSuccess._log (info['logger'], "sftp version not supported (%s)" % str(out)) if 'No such file or directory' in out : raise se.DoesNotExist._log (info['logger'], "file copy failed: %s" % str(out)) if 'is not a directory' in out : raise se.BadParameter._log (info['logger'], "File copy failed: %s" % str(out)) if 'sftp' in s_cmd : if 'not found' in out : raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out) # we interpret the first word on the line as name of src file -- we # will return a list of those lines = out.split ('\n') files = [] for line in lines : elems = line.split (' ', 2) if elems : f = elems[0] # remove quotes if f : if f[ 0] in ["'", '"', '`'] : f = f[1: ] if f[-1] in ["'", '"', '`'] : f = f[ :-1] # ignore empty lines if f : files.append (f) info['logger'].debug ("copy done: %s" % files) return files
def _initialize_pty(self, pty_shell, info, posix=None): # posix: only for posix shells we use prompt triggers. sftp for example # does not deal well with triggers (no printf). with self.rlock: # import pprint # pprint.pprint (info) shell_pass = info["pass"] key_pass = info["key_pass"] prompt = info["prompt"] logger = info["logger"] latency = info["latency"] timeout = info["ssh_timeout"] pty_shell.latency = latency if posix == None: posix = info["posix"] # if we did not see a decent prompt within 'delay' time, something # went wrong. Try to prompt a prompt (duh!) Delay should be # minimum 0.1 second (to avoid flooding of local shells), and at # maximum 1 second (to keep startup time reasonable) # most one second. We try to get within that range with 10*latency. delay = min(1.0, max(0.1, 10 * latency)) try: prompt_patterns = [ "[Pp]assword:\s*$", # password prompt "Enter passphrase for .*:\s*$", # passphrase prompt "Token_Response.*:\s*$", # passtoken prompt "Enter PASSCODE:$", # RSA SecureID "want to continue connecting", # hostkey confirmation ".*HELLO_\\d+_SAGA$", # prompt detection helper prompt, ] # greedy native shell prompt # use a very aggressive, but portable prompt setting scheme. # Error messages may appear for tcsh and others. Excuse # non-posix shells if posix: pty_shell.write(" export PS1='$' ; set prompt='$'\n") # find a prompt n, match = pty_shell.find(prompt_patterns, delay) # this loop will run until we finally find the shell prompt, or # if we think we have tried enough and give up. On success # we'll try to set a different prompt, and when we found that, # too, we exit the loop and are be ready to running shell # commands. retries = 0 retry_trigger = True used_trigger = False found_trigger = "" time_start = time.time() while True: # -------------------------------------------------------------- if n == None: # we found none of the prompts, yet, and need to try # again. But to avoid hanging on invalid prompts, we # print 'HELLO_x_SAGA', and search for that one, too. # We actually do 'printf HELLO_%d_SAGA x' so that the # pattern only appears in the result, not in the # command... if time.time() - time_start > timeout: raise se.NoSuccess("Could not detect shell prompt (timeout)") # make sure we retry a finite time... retries += 1 if not retry_trigger: # just waiting for the *right* trigger or prompt, # don't need new ones... continue if posix: # use a very aggressive, but portable prompt setting scheme pty_shell.write(" export PS1='$' > /dev/null 2>&1 || set prompt='$'\n") pty_shell.write(" printf 'HELLO_%%d_SAGA\\n' %d\n" % retries) used_trigger = True # FIXME: consider better timeout n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 0: logger.info("got password prompt") if not shell_pass: raise se.AuthenticationFailed("prompted for unknown password (%s)" % match) pty_shell.write("%s\n" % shell_pass, nolog=True) n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 1: logger.info("got passphrase prompt : %s" % match) start = string.find(match, "'", 0) end = string.find(match, "'", start + 1) if start == -1 or end == -1: raise se.AuthenticationFailed("could not extract key name (%s)" % match) key = match[start + 1 : end] if not key in key_pass: raise se.AuthenticationFailed("prompted for unknown key password (%s)" % key) pty_shell.write("%s\n" % key_pass[key], nolog=True) n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 2 or n == 3: logger.info("got token prompt") import getpass token = getpass.getpass("enter token: ") pty_shell.write("%s\n" % token.strip(), nolog=True) n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 4: logger.info("got hostkey prompt") pty_shell.write("yes\n") n, match = pty_shell.find(prompt_patterns, delay) # -------------------------------------------------------------- elif n == 5: # one of the trigger commands got through -- we can now # hope to find the prompt (or the next trigger...) logger.debug("got shell prompt trigger (%s) (%s)" % (n, match)) found_trigger = match retry_trigger = False n, match = pty_shell.find(prompt_patterns, delay) continue # -------------------------------------------------------------- elif n == 6: logger.debug("got initial shell prompt (%s) (%s)" % (n, match)) if retries: if used_trigger: # we already sent triggers -- so this match is only # useful if saw the *correct* shell prompt trigger # first trigger = "HELLO_%d_SAGA" % retries if not trigger in found_trigger: logger.debug("waiting for prompt trigger %s: (%s) (%s)" % (trigger, n, match)) # but more retries won't help... retry_trigger = False attempts = 0 n = None while not n: attempts += 1 n, match = pty_shell.find(prompt_patterns, delay) if not n: if attempts == 1: if posix: pty_shell.write(" printf 'HELLO_%%d_SAGA\\n' %d\n" % retries) if attempts > 100: raise se.NoSuccess("Could not detect shell prompt (timeout)") continue logger.debug("Got initial shell prompt (%s) (%s)" % (n, match)) # we are done waiting for a prompt break except Exception as e: logger.exception(e) raise ptye.translate_exception(e)
def set_prompt (self, new_prompt) : """ :type new_prompt: string :param new_prompt: a regular expression matching the shell prompt The new_prompt regex is expected to be a regular expression with one set of catching brackets, which MUST return the previous command's exit status. This method will send a newline to the client, and expects to find the prompt with the exit value '0'. As a side effect, this method will discard all previous data on the pty, thus effectively flushing the pty output. By encoding the exit value in the command prompt, we safe one roundtrip. The prompt on Posix compliant shells can be set, for example, via:: PS1='PROMPT-$?->'; export PS1 The newline in the example above allows to nicely anchor the regular expression, which would look like:: PROMPT-(\d+)->$ The regex is compiled with 're.DOTALL', so the dot character matches all characters, including line breaks. Be careful not to match more than the exact prompt -- otherwise, a prompt search will swallow stdout data. For example, the following regex:: PROMPT-(.+)->$ would capture arbitrary strings, and would thus match *all* of:: PROMPT-0->ls data/ info PROMPT-0-> and thus swallow the ls output... Note that the string match *before* the prompt regex is non-gready -- if the output contains multiple occurrences of the prompt, only the match up to the first occurence is returned. """ def escape (txt) : pat = re.compile(r'\x1b[^m]*m') return pat.sub ('', txt) with self.pty_shell.rlock : old_prompt = self.prompt self.prompt = new_prompt self.prompt_re = re.compile ("^(.*?)%s\s*$" % self.prompt, re.DOTALL) retries = 0 triggers = 0 while True : try : # make sure we have a non-zero waiting delay (default to # 1 second) delay = 10 * self.latency if not delay : delay = 1.0 # FIXME: how do we know that _PTY_TIMOUT suffices? In particular if # we actually need to flush... fret, match = self.pty_shell.find ([self.prompt], delay) if fret == None : retries += 1 if retries > 10 : self.prompt = old_prompt raise se.BadParameter ("Cannot use new prompt, parsing failed (10 retries)") self.pty_shell.write ("\n") self.logger.debug ("sent prompt trigger again (%d)" % retries) triggers += 1 continue # found a match -- lets see if this is working now... ret, _ = self._eval_prompt (match) if ret != 0 : self.prompt = old_prompt raise se.BadParameter ("could not parse exit value (%s)" \ % match) # prompt looks valid... break except Exception as e : self.prompt = old_prompt raise ptye.translate_exception (e, "Could not set shell prompt") # got a valid prompt -- but we have to sync the output again in # those cases where we had to use triggers to actually get the # prompt if triggers > 0 : self.run_async (' printf "SYNCHRONIZE_PROMPT\n"') # FIXME: better timout value? fret, match = self.pty_shell.find (["SYNCHRONIZE_PROMPT"], timeout=1.0) if fret == None : # not find prompt after blocking? BAD! Restart the shell self.finalize (kill_pty=True) raise se.NoSuccess ("Could not synchronize prompt detection") self.find_prompt ()
def run_sync (self, command, iomode=None, new_prompt=None) : """ Run a shell command, and report exit code, stdout and stderr (all three will be returned in a tuple). The call will block until the command finishes (more exactly, until we find the prompt again on the shell's I/O stream), and cannot be interrupted. :type command: string :param command: shell command to run. :type iomode: enum :param iomode: Defines how stdout and stderr are captured. :type new_prompt: string :param new_prompt: regular expression matching the prompt after command succeeded. We expect the ``command`` to not to do stdio redirection, as this is we want to capture that separately. We *do* allow pipes and stdin/stdout redirection. Note that SEPARATE mode will break if the job is run in the background The following iomode values are valid: * *IGNORE:* both stdout and stderr are discarded, `None` will be returned for each. * *MERGED:* both streams will be merged and returned as stdout; stderr will be `None`. This is the default. * *SEPARATE:* stdout and stderr will be captured separately, and returned individually. Note that this will require at least one more network hop! * *STDOUT:* only stdout is captured, stderr will be `None`. * *STDERR:* only stderr is captured, stdout will be `None`. * *None:* do not perform any redirection -- this is effectively the same as `MERGED` If any of the requested output streams does not return any data, an empty string is returned. If the command to be run changes the prompt to be expected for the shell, the ``new_prompt`` parameter MUST contain a regex to match the new prompt. The same conventions as for set_prompt() hold -- i.e. we expect the prompt regex to capture the exit status of the process. """ with self.pty_shell.rlock : # we expect the shell to be in 'ground state' when running a syncronous # command -- thus we can check if the shell is alive before doing so, # and restart if needed if not self.pty_shell.alive (recover=True) : raise se.IncorrectState ("Can't run command -- shell died:\n%s" \ % self.pty_shell.autopsy ()) try : command = command.strip () if command.endswith ('&') : raise se.BadParameter ("run_sync can only run foreground jobs ('%s')" \ % command) redir = "" _err = "/tmp/saga-python.ssh-job.stderr.$$" if iomode == IGNORE : redir = " 1>>/dev/null 2>>/dev/null" if iomode == MERGED : redir = " 2>&1" if iomode == SEPARATE : redir = " 2>%s" % _err if iomode == STDOUT : redir = " 2>/dev/null" if iomode == STDERR : redir = " 2>&1 1>/dev/null" if iomode == None : redir = "" self.logger.debug ('run_sync: %s%s' % (command, redir)) self.pty_shell.write ( "%s%s\n" % (command, redir)) # If given, switch to new prompt pattern right now... prompt = self.prompt if new_prompt : prompt = new_prompt # command has been started - now find prompt again. fret, match = self.pty_shell.find ([prompt], timeout=-1.0) # blocks if fret == None : # not find prompt after blocking? BAD! Restart the shell self.finalize (kill_pty=True) raise se.IncorrectState ("run_sync failed, no prompt (%s)" % command) ret, txt = self._eval_prompt (match, new_prompt) stdout = None stderr = None if iomode == IGNORE : pass if iomode == MERGED : stdout = txt if iomode == SEPARATE : stdout = txt self.pty_shell.write (" cat %s\n" % _err) fret, match = self.pty_shell.find ([self.prompt], timeout=-1.0) # blocks if fret == None : # not find prompt after blocking? BAD! Restart the shell self.finalize (kill_pty=True) raise se.IncorrectState ("run_sync failed, no prompt (%s)" \ % command) _ret, _stderr = self._eval_prompt (match) if _ret : raise se.IncorrectState ("run_sync failed, no stderr (%s: %s)" \ % (_ret, _stderr)) stderr = _stderr if iomode == STDOUT : stdout = txt if iomode == STDERR : stderr = txt if iomode == None : stdout = txt return (ret, stdout, stderr) except Exception as e : raise ptye.translate_exception (e)
def run_sync (self, command, iomode=None, new_prompt=None) : """ Run a shell command, and report exit code, stdout and stderr (all three will be returned in a tuple). The call will block until the command finishes (more exactly, until we find the prompt again on the shell's I/O stream), and cannot be interrupted. :type command: string :param command: shell command to run. :type iomode: enum :param iomode: Defines how stdout and stderr are captured. :type new_prompt: string :param new_prompt: regular expression matching the prompt after command succeeded. We expect the ``command`` to not to do stdio redirection, as this is we want to capture that separately. We *do* allow pipes and stdin/stdout redirection. Note that SEPARATE mode will break if the job is run in the background The following iomode values are valid: * *IGNORE:* both stdout and stderr are discarded, `None` will be returned for each. * *MERGED:* both streams will be merged and returned as stdout; stderr will be `None`. This is the default. * *SEPARATE:* stdout and stderr will be captured separately, and returned individually. Note that this will require at least one more network hop! * *STDOUT:* only stdout is captured, stderr will be `None`. * *STDERR:* only stderr is captured, stdout will be `None`. * *None:* do not perform any redirection -- this is effectively the same as `MERGED` If any of the requested output streams does not return any data, an empty string is returned. If the command to be run changes the prompt to be expected for the shell, the ``new_prompt`` parameter MUST contain a regex to match the new prompt. The same conventions as for set_prompt() hold -- i.e. we expect the prompt regex to capture the exit status of the process. """ with self.pty_shell.rlock : self._trace ("run sync : %s" % command) self.pty_shell.flush () # we expect the shell to be in 'ground state' when running a syncronous # command -- thus we can check if the shell is alive before doing so, # and restart if needed if not self.pty_shell.alive (recover=True) : raise se.IncorrectState ("Can't run command -- shell died:\n%s" \ % self.pty_shell.autopsy ()) try : command = command.strip () if command.endswith ('&') : raise se.BadParameter ("run_sync can only run foreground jobs ('%s')" \ % command) redir = "" _err = "/tmp/saga-python.ssh-job.stderr.$$" if iomode == IGNORE : redir = " 1>>/dev/null 2>>/dev/null" if iomode == MERGED : redir = " 2>&1" if iomode == SEPARATE : redir = " 2>%s" % _err if iomode == STDOUT : redir = " 2>/dev/null" if iomode == STDERR : redir = " 2>&1 1>/dev/null" if iomode == None : redir = "" self.logger.debug ('run_sync: %s%s' % (command, redir)) self.pty_shell.write ( "%s%s\n" % (command, redir)) # If given, switch to new prompt pattern right now... prompt = self.prompt if new_prompt : prompt = new_prompt # command has been started - now find prompt again. fret, match = self.pty_shell.find ([prompt], timeout=-1.0) # blocks if fret == None : # not find prompt after blocking? BAD! Restart the shell self.finalize (kill_pty=True) raise se.IncorrectState ("run_sync failed, no prompt (%s)" % command) ret, txt = self._eval_prompt (match, new_prompt) stdout = None stderr = None if iomode == None : iomode = STDOUT if iomode == IGNORE : pass if iomode == MERGED : stdout = txt if iomode == STDOUT : stdout = txt if iomode == SEPARATE or \ iomode == STDERR : stdout = txt self.pty_shell.write (" cat %s\n" % _err) fret, match = self.pty_shell.find ([self.prompt], timeout=-1.0) # blocks if fret == None : # not find prompt after blocking? BAD! Restart the shell self.finalize (kill_pty=True) raise se.IncorrectState ("run_sync failed, no prompt (%s)" \ % command) _ret, _stderr = self._eval_prompt (match) if _ret : raise se.IncorrectState ("run_sync failed, no stderr (%s: %s)" \ % (_ret, _stderr)) stderr = _stderr if iomode == STDERR : # got stderr in branch above stdout = None return (ret, stdout, stderr) except Exception as e : raise ptye.translate_exception (e)
def set_prompt (self, new_prompt) : """ :type new_prompt: string :param new_prompt: a regular expression matching the shell prompt The new_prompt regex is expected to be a regular expression with one set of catching brackets, which MUST return the previous command's exit status. This method will send a newline to the client, and expects to find the prompt with the exit value '0'. As a side effect, this method will discard all previous data on the pty, thus effectively flushing the pty output. By encoding the exit value in the command prompt, we safe one roundtrip. The prompt on Posix compliant shells can be set, for example, via:: PS1='PROMPT-$?->'; export PS1 The newline in the example above allows to nicely anchor the regular expression, which would look like:: PROMPT-(\d+)->$ The regex is compiled with 're.DOTALL', so the dot character matches all characters, including line breaks. Be careful not to match more than the exact prompt -- otherwise, a prompt search will swallow stdout data. For example, the following regex:: PROMPT-(.+)->$ would capture arbitrary strings, and would thus match *all* of:: PROMPT-0->ls data/ info PROMPT-0-> and thus swallow the ls output... Note that the string match *before* the prompt regex is non-gready -- if the output contains multiple occurrences of the prompt, only the match up to the first occurence is returned. """ def escape (txt) : pat = re.compile(r'\x1b[^m]*m') return pat.sub ('', txt) with self.pty_shell.rlock : old_prompt = self.prompt self.prompt = new_prompt self.prompt_re = re.compile ("^(.*?)%s\s*$" % self.prompt, re.DOTALL) retries = 0 triggers = 0 while True : try : # make sure we have a non-zero waiting delay (default to # 1 second) delay = 10 * self.latency if not delay : delay = 1.0 # FIXME: how do we know that _PTY_TIMOUT suffices? In particular if # we actually need to flush... fret, match = self.pty_shell.find ([self.prompt], delay) if fret == None : retries += 1 if retries > 10 : self.prompt = old_prompt raise se.BadParameter ("Cannot use new prompt, parsing failed (10 retries)") self.pty_shell.write ("\n") self.logger.debug ("sent prompt trigger again (%d)" % retries) triggers += 1 continue # found a match -- lets see if this is working now... ret, _ = self._eval_prompt (match) if ret != 0 : self.prompt = old_prompt raise se.BadParameter ("could not parse exit value (%s)" \ % match) # prompt looks valid... break except Exception as e : self.prompt = old_prompt raise ptye.translate_exception (e, "Could not set shell prompt") # got a valid prompt -- but we have to sync the output again in # those cases where we had to use triggers to actually get the # prompt if triggers > 0 : self.run_async (' printf "SYNCHRONIZE_PROMPT\n"') # FIXME: better timout value? fret, match = self.pty_shell.find (["SYNCHRONIZE_PROMPT"], timeout=10.0) if fret == None : # not find prompt after blocking? BAD! Restart the shell self.finalize (kill_pty=True) raise se.NoSuccess ("Could not synchronize prompt detection") self.find_prompt ()
def run_copy_from (self, src, tgt, cp_flags="") : """ This initiates a slave copy connection. Src is interpreted as path on the remote host, tgt as local path. We have to do the same mkdir trick as for the run_copy_to, but here we need to expand wildcards on the *remote* side :/ """ with self.pty_shell.rlock : self._trace ("copy from: %s -> %s" % (src, tgt)) self.pty_shell.flush () info = self.pty_info repl = dict ({'src' : src, 'tgt' : tgt, 'cp_flags' : cp_flags}.items() + info.items ()) # at this point, we do have a valid, living master s_cmd = info['scripts'][info['copy_mode']]['copy_from'] % repl s_in = info['scripts'][info['copy_mode']]['copy_from_in'] % repl posix = info['scripts'][info['copy_mode']]['copy_is_posix'] if not s_in : # this code path does not use an interactive shell for copy -- # so the above s_cmd is all we want to run, really. We get # do not use the chached cp_slave in this case, but just run the # command. We do not have a list of transferred files though, # yet -- that should be parsed from the proc output. cp_proc = supp.PTYProcess (s_cmd) cp_proc.wait () if cp_proc.exit_code : raise ptye.translate_exception (se.NoSuccess ("file copy failed: exit code %s" % cp_proc.exit_code)) return list() if not self.cp_slave : self._trace ("get cp slave") self.cp_slave = self.factory.get_cp_slave (s_cmd, info, posix) self.cp_slave.flush () prep = "" if 'sftp' in s_cmd : # prepare target dirs for recursive copy, if needed self.cp_slave.write (" ls %s\n" % src) _, out = self.cp_slave.find (["^sftp> "], -1) src_list = out[1].split('\n') for s in src_list : if os.path.isdir (s) : prep += "lmkdir %s/%s\n" % (tgt, os.path.basename (s)) self.cp_slave.flush () _ = self.cp_slave.write ("%s%s\n" % (prep, s_in)) _, out = self.cp_slave.find (['[\$\>\]] *$'], -1) # FIXME: we don't really get exit codes from copy # if self.cp_slave.exit_code != 0 : # raise se.NoSuccess._log (info['logger'], "file copy failed: %s" % out) if 'Invalid flag' in out : raise se.NoSuccess._log (info['logger'], "sftp version not supported (%s)" % out) if 'No such file or directory' in out : raise se.DoesNotExist._log (info['logger'], "file copy failed: %s" % out) if 'is not a directory' in out : raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out) if 'sftp' in s_cmd : if 'not found' in out : raise se.BadParameter._log (info['logger'], "file copy failed: %s" % out) # we run copy with -v, so get a list of files which have been copied # -- we parse that list and return it. we interpret the *second* # word on the line as name of src file. lines = out.split ('\n') files = [] for line in lines : elems = line.split (' ', 3) if elems and len(elems) > 1 and elems[0] == 'Fetching' : f = elems[1] # remove quotes if f : if f[ 0] in ["'", '"', '`'] : f = f[1: ] if f[-1] in ["'", '"', '`'] : f = f[ :-1] # ignore empty lines if f : files.append (f) info['logger'].debug ("copy done: %s" % files) return files