def _run(self, cmd, timeout, verbose, ignore_status, connect_timeout, env, options, args, log_file): ssh_cmd = self.ssh_command(connect_timeout, options) if not env.strip(): env = "" else: env = "export %s;" % env for arg in args: cmd += ' "%s"' % astring.shell_escape(arg) if env: full_cmd = '%s "%s %s"' % (ssh_cmd, env, astring.shell_escape(cmd)) else: full_cmd = '%s "%s"' % (ssh_cmd, astring.shell_escape(cmd)) result = ssh_run(full_cmd, verbose=verbose, ignore_status=ignore_status, timeout=timeout, extra_text=self.hostname, shell=True, log_file=log_file) # The error messages will show up in band (indistinguishable # from stuff sent through the SSH connection), so we have the # remote computer echo the message "Connected." before running # any cmd. Since the following 2 errors have to do with # connecting, it's safe to do these checks. if result.exit_status == 255: if re.search(r'^ssh: connect to host .* port .*: ' r'Connection timed out\r$', result.stderr): raise SSHTimeout("SSH timed out:\n%s" % result) if "Permission denied." in result.stderr: raise SSHPermissionDeniedError("SSH permission denied:\n%s" % result) if not ignore_status and result.exit_status > 0: raise process.CmdError(command=full_cmd, result=result) return result
def _run(self, cmd, timeout, verbose, ignore_status, connect_timeout, env, options, args, log_file, watch_stdout_pattern): ssh_cmd = self.ssh_command(connect_timeout, options) if not env.strip(): env = "" else: env = "export %s;" % env for arg in args: cmd += ' "%s"' % astring.shell_escape(arg) if env: full_cmd = '%s "%s %s"' % (ssh_cmd, env, astring.shell_escape(cmd)) else: full_cmd = '%s "%s"' % (ssh_cmd, astring.shell_escape(cmd)) result = ssh_run(full_cmd, verbose=verbose, ignore_status=ignore_status, timeout=timeout, extra_text=self.hostname, shell=True, log_file=log_file, watch_stdout_pattern=watch_stdout_pattern) # The error messages will show up in band (indistinguishable # from stuff sent through the SSH connection), so we have the # remote computer echo the message "Connected." before running # any cmd. Since the following 2 errors have to do with # connecting, it's safe to do these checks. if result.exit_status == 255: if re.search(r'^ssh: connect to host .* port .*: ' r'Connection timed out\r$', result.stderr): raise SSHTimeout("SSH timed out:\n%s" % result) if "Permission denied." in result.stderr: raise SSHPermissionDeniedError("SSH permission denied:\n%s" % result) if not ignore_status and result.exit_status > 0: raise process.CmdError(command=full_cmd, result=result) return result
def send_files(self, src, dst, delete_dst=False, preserve_symlinks=False, verbose=False, ssh_timeout=None): """ Copy files from a local path to the remote host. If both machines support rsync, that command will be used. If not, an scp command will be assembled. Directories will be copied recursively. If a src component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. :param src: Either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories :param dst: A file or a directory (if src contains a directory or more than one element, you must supply a directory dst). :param delete_dst: If this is true, the command will also clear out any old files at dest that are not in the src :param preserve_symlinks: Try to preserve symlinks instead of transforming them into files/dirs on copy. :param verbose: Log commands being used and their outputs. :param ssh_timeout: Timeout is used for self.ssh_run() :raises: process.CmdError if the remote copy command failed """ self.log.debug('Send files (src) %s -> (dst) %s', src, dst) # Start a master SSH connection if necessary. self.start_master_ssh() source_is_dir = False if isinstance(src, basestring): source_is_dir = os.path.isdir(src) src = [src] remote_dest = self._encode_remote_paths([dst]) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: local_sources = [astring.shell_escape(path) for path in src] rsync = self._make_rsync_cmd(local_sources, remote_dest, delete_dst, preserve_symlinks) self.ssh_run(rsync, shell=True, extra_text=self.hostname, verbose=verbose, timeout=ssh_timeout) try_scp = False except process.CmdError, details: self.log.warning("Trying scp, rsync failed: %s", details) # Make sure master ssh available self.start_master_ssh()
def receive_files(self, src, dst, delete_dst=False, preserve_perm=True, preserve_symlinks=False, verbose=False, ssh_timeout=300): """ Copy files from the remote host to a local path. If both machines support rsync, that command will be used. If not, an scp command will be assembled. Directories will be copied recursively. If a src component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. :param src: Either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories :param dst: A file or a directory (if src contains a directory or more than one element, you must supply a directory dst). :param delete_dst: If this is true, the command will also clear out any old files at dest that are not in the src :param preserve_perm: Tells get_file() to try to preserve the sources permissions on files and dirs. :param preserve_symlinks: Try to preserve symlinks instead of transforming them into files/dirs on copy. :param verbose: Log commands being used and their outputs. :param ssh_timeout: Timeout is used for self.ssh_run() :raises: process.CmdError if the remote copy command failed. """ self.log.debug('Receive files (src) %s -> (dst) %s', src, dst) # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(src, basestring): src = [src] dst = os.path.abspath(dst) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(src) local_dest = astring.shell_escape(dst) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dst, preserve_symlinks) self.ssh_run(rsync, shell=True, extra_text=self.hostname, verbose=verbose, timeout=ssh_timeout) try_scp = False except process.CmdError, e: self.log.warning("Trying scp, rsync failed: %s", e) # Make sure master ssh available self.start_master_ssh()
def git_cmd(self, cmd, ignore_status=False): """ Wraps git commands. :param cmd: Command to be executed. :param ignore_status: Whether we should suppress error.CmdError exceptions if the command did return exit code !=0 (True), or not suppress them (False). """ os.chdir(self.destination_dir) return process.run(r"%s %s" % (self.cmd, astring.shell_escape(cmd)), ignore_status=ignore_status)
def _make_ssh_cmd(self, cmd): """ Create a base ssh command string for the host which can be used to run commands directly on the machine """ base_cmd = _make_ssh_command(user=self.user, port=self.port, key_file=self.key_file, opts=self.master_ssh_option, hosts_file=self.known_hosts_file) return '%s %s "%s"' % (base_cmd, self.hostname, astring.shell_escape(cmd))
def receive_files(self, src, dst, delete_dst=False, preserve_perm=True, preserve_symlinks=False, verbose=False): """ Copy files from the remote host to a local path. If both machines support rsync, that command will be used. If not, an scp command will be assembled. Directories will be copied recursively. If a src component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. :param src: Either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories :param dst: A file or a directory (if src contains a directory or more than one element, you must supply a directory dst). :param delete_dst: If this is true, the command will also clear out any old files at dest that are not in the src :param preserve_perm: Tells get_file() to try to preserve the sources permissions on files and dirs. :param preserve_symlinks: Try to preserve symlinks instead of transforming them into files/dirs on copy. :param verbose: Log commands being used and their outputs. :raises: process.CmdError if the remote copy command failed. """ self.log.debug('Receive files (src) %s -> (dst) %s', src, dst) # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(src, basestring): src = [src] dst = os.path.abspath(dst) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(src) local_dest = astring.shell_escape(dst) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dst, preserve_symlinks) ssh_run(rsync, shell=True, extra_text=self.hostname, verbose=verbose) try_scp = False except process.CmdError, e: self.log.warning("Trying scp, rsync failed: %s", e)
def send_files(self, src, dst, delete_dst=False, preserve_symlinks=False, verbose=False): """ Copy files from a local path to the remote host. If both machines support rsync, that command will be used. If not, an scp command will be assembled. Directories will be copied recursively. If a src component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. :param src: Either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories :param dst: A file or a directory (if src contains a directory or more than one element, you must supply a directory dst). :param delete_dst: If this is true, the command will also clear out any old files at dest that are not in the src :param preserve_symlinks: Try to preserve symlinks instead of transforming them into files/dirs on copy. :param verbose: Log commands being used and their outputs. :raises: process.CmdError if the remote copy command failed """ self.log.debug('Send files (src) %s -> (dst) %s', src, dst) # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(src, basestring): source_is_dir = os.path.isdir(src) src = [src] remote_dest = self._encode_remote_paths([dst]) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: local_sources = [astring.shell_escape(path) for path in src] rsync = self._make_rsync_cmd(local_sources, remote_dest, delete_dst, preserve_symlinks) ssh_run(rsync, shell=True, extra_text=self.hostname, verbose=verbose) try_scp = False except process.CmdError, details: self.log.warning("Trying scp, rsync failed: %s", details)
def _make_rsync_compatible_globs(self, pth, is_local): """ Given an rsync-style path (pth), returns a list of globbed paths. Those will hopefully provide equivalent behaviour for scp. Does not support the full range of rsync pattern matching behaviour, only that exposed in the get/send_file interface (trailing slashes). :param pth: rsync-style path. :param is_local: Whether the paths should be interpreted as local or remote paths. """ # non-trailing slash paths should just work if len(pth) == 0 or pth[-1] != "/": return [pth] # make a function to test if a pattern matches any files if is_local: def glob_matches_files(path, pattern): return len(glob.glob(path + pattern)) > 0 else: def glob_matches_files(path, pattern): match_cmd = "ls \"%s\"%s" % (astring.shell_escape(path), pattern) result = self.run(match_cmd, ignore_status=True) return result.exit_status == 0 # take a set of globs that cover all files, and see which are needed patterns = ["*", ".[!.]*"] patterns = [p for p in patterns if glob_matches_files(pth, p)] # convert them into a set of paths suitable for the commandline if is_local: return [ "\"%s\"%s" % (astring.shell_escape(pth), pattern) for pattern in patterns ] else: return [_scp_remote_escape(pth) + pattern for pattern in patterns]
def _make_rsync_compatible_globs(self, pth, is_local): """ Given an rsync-style path (pth), returns a list of globbed paths. Those will hopefully provide equivalent behaviour for scp. Does not support the full range of rsync pattern matching behaviour, only that exposed in the get/send_file interface (trailing slashes). :param pth: rsync-style path. :param is_local: Whether the paths should be interpreted as local or remote paths. """ # non-trailing slash paths should just work if len(pth) == 0 or pth[-1] != "/": return [pth] # make a function to test if a pattern matches any files if is_local: def glob_matches_files(path, pattern): return len(glob.glob(path + pattern)) > 0 else: def glob_matches_files(path, pattern): match_cmd = 'ls "%s"%s' % (astring.shell_escape(path), pattern) result = self.run(match_cmd, ignore_status=True) return result.exit_status == 0 # take a set of globs that cover all files, and see which are needed patterns = ["*", ".[!.]*"] patterns = [p for p in patterns if glob_matches_files(pth, p)] # convert them into a set of paths suitable for the commandline if is_local: return ['"%s"%s' % (astring.shell_escape(pth), pattern) for pattern in patterns] else: return [_scp_remote_escape(pth) + pattern for pattern in patterns]
def _scp_remote_escape(filename): """ Escape special chars for SCP use. Bis-quoting has to be used with scp for remote files, "bis-quoting" as in quoting x 2. SCP does not support a newline in the filename. :param filename: the filename string to escape. :returns: The escaped filename string. The required englobing double quotes are NOT added and so should be added at some point by the caller. """ escape_chars = r' !"$&' "'" r'()*,:;<=>?[\]^`{|}' new_name = [] for char in filename: if char in escape_chars: new_name.append("\\%s" % (char,)) else: new_name.append(char) return astring.shell_escape("".join(new_name))
def glob_matches_files(path, pattern): match_cmd = "ls \"%s\"%s" % (astring.shell_escape(path), pattern) result = self.run(match_cmd, ignore_status=True) return result.exit_status == 0
class BaseRemote(object): def __init__(self, hostname, user="******", port=22, password="", key_file=None, wait_key_installed=0, extra_ssh_options=""): self.env = {} self.hostname = hostname self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0] self.user = user self.port = port self.password = password self.key_file = key_file self._use_rsync = None self.known_hosts_file = tempfile.mkstemp()[1] self.master_ssh_job = None self.master_ssh_tempdir = None self.master_ssh_option = '' self.extra_ssh_options = extra_ssh_options logger = logging.getLogger('avocado.test') self.log = SDCMAdapter(logger, extra={'prefix': str(self)}) time.sleep(wait_key_installed) self._check_install_key_required() def _check_install_key_required(self): def _safe_ssh_ping(): try: self._ssh_ping() return True except (SSHPermissionDeniedError, process.CmdError): return None if not self.key_file and self.password: try: self._ssh_ping() except (SSHPermissionDeniedError, process.CmdError): copy_id_cmd = ('ssh-copy-id -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s %s@%s' % (self.port, self.user, self.hostname)) while True: try: expect = aexpect.Expect(copy_id_cmd) expect.read_until_output_matches(['.*password:'******'Waiting for password-less SSH') if result is None: raise SSHPermissionDeniedError('Unable to configure ' 'password less SSH. ' 'Output of %s: %s' % (copy_id_cmd, expect.get_output())) else: self.log.info('Successfully configured SSH key auth') def ssh_debug_cmd(self): if self.key_file: return "SSH access -> 'ssh -i %s %s@%s'" % (self.key_file, self.user, self.ip) else: return "SSH access -> 'ssh %s@%s'" % (self.user, self.ip) def __str__(self): return 'Remote [%s@%s]' % (self.user, self.hostname) def use_rsync(self): if self._use_rsync is not None: return self._use_rsync # Check if rsync is available on the remote host. If it's not, # don't try to use it for any future file transfers. self._use_rsync = self._check_rsync() if not self._use_rsync: self.log.warning("Command rsync not available -- disabled") return self._use_rsync def _check_rsync(self): """ Check if rsync is available on the remote host. """ try: self.run("rsync --version", verbose=False) except process.CmdError: return False return True def _encode_remote_paths(self, paths, escape=True): """ Given a list of file paths, encodes it as a single remote path, in the style used by rsync and scp. """ if escape: paths = [_scp_remote_escape(path) for path in paths] return '%s@%s:"%s"' % (self.user, self.hostname, " ".join(paths)) def _make_rsync_cmd(self, src, dst, delete_dst, preserve_symlinks): """ Given a list of source paths and a destination path, produces the appropriate rsync command for copying them. Remote paths must be pre-encoded. """ ssh_cmd = _make_ssh_command(user=self.user, port=self.port, opts=self.master_ssh_option, hosts_file=self.known_hosts_file, key_file=self.key_file, extra_ssh_options=self.extra_ssh_options.replace('-tt', '-t')) if delete_dst: delete_flag = "--delete" else: delete_flag = "" if preserve_symlinks: symlink_flag = "" else: symlink_flag = "-L" command = "rsync %s %s --timeout=300 --rsh='%s' -az %s %s" return command % (symlink_flag, delete_flag, ssh_cmd, " ".join(src), dst) def _make_ssh_cmd(self, cmd): """ Create a base ssh command string for the host which can be used to run commands directly on the machine """ base_cmd = _make_ssh_command(user=self.user, port=self.port, key_file=self.key_file, opts=self.master_ssh_option, hosts_file=self.known_hosts_file, extra_ssh_options=self.extra_ssh_options) return '%s %s "%s"' % (base_cmd, self.hostname, astring.shell_escape(cmd)) def _make_scp_cmd(self, src, dst, connect_timeout=300, alive_interval=300): """ Given a list of source paths and a destination path, produces the appropriate scp command for encoding it. Remote paths must be pre-encoded. """ key_option = '' if self.key_file: key_option = '-i %s' % os.path.expanduser(self.key_file) command = ("scp -r %s -o StrictHostKeyChecking=no -o BatchMode=yes " "-o ConnectTimeout=%d -o ServerAliveInterval=%d " "-o UserKnownHostsFile=%s -P %d %s %s '%s'") return command % (self.master_ssh_option, connect_timeout, alive_interval, self.known_hosts_file, self.port, key_option, " ".join(src), dst) def _make_rsync_compatible_globs(self, pth, is_local): """ Given an rsync-style path (pth), returns a list of globbed paths. Those will hopefully provide equivalent behaviour for scp. Does not support the full range of rsync pattern matching behaviour, only that exposed in the get/send_file interface (trailing slashes). :param pth: rsync-style path. :param is_local: Whether the paths should be interpreted as local or remote paths. """ # non-trailing slash paths should just work if len(pth) == 0 or pth[-1] != "/": return [pth] # make a function to test if a pattern matches any files if is_local: def glob_matches_files(path, pattern): return len(glob.glob(path + pattern)) > 0 else: def glob_matches_files(path, pattern): match_cmd = "ls \"%s\"%s" % (astring.shell_escape(path), pattern) result = self.run(match_cmd, ignore_status=True) return result.exit_status == 0 # take a set of globs that cover all files, and see which are needed patterns = ["*", ".[!.]*"] patterns = [p for p in patterns if glob_matches_files(pth, p)] # convert them into a set of paths suitable for the commandline if is_local: return ["\"%s\"%s" % (astring.shell_escape(pth), pattern) for pattern in patterns] else: return [_scp_remote_escape(pth) + pattern for pattern in patterns] def _make_rsync_compatible_source(self, source, is_local): """ Make an rsync compatible source string. Applies the same logic as _make_rsync_compatible_globs, but applies it to an entire list of sources, producing a new list of sources, properly quoted. """ return sum((self._make_rsync_compatible_globs(path, is_local) for path in source), []) def _set_umask_perms(self, dest): """ Set permissions on all files and directories. Given a destination file/dir (recursively) set the permissions on all the files and directories to the max allowed by running umask. """ # now this looks strange but I haven't found a way in Python to _just_ # get the umask, apparently the only option is to try to set it umask = os.umask(0) os.umask(umask) max_privs = 0777 & ~umask def set_file_privs(filename): file_stat = os.stat(filename) file_privs = max_privs # if the original file permissions do not have at least one # executable bit then do not set it anywhere if not file_stat.st_mode & 0111: file_privs &= ~0111 os.chmod(filename, file_privs) # try a bottom-up walk so changes on directory permissions won't cut # our access to the files/directories inside it for root, dirs, files in os.walk(dest, topdown=False): # when setting the privileges we emulate the chmod "X" behaviour # that sets to execute only if it is a directory or any of the # owner/group/other already has execute right for dirname in dirs: os.chmod(os.path.join(root, dirname), max_privs) for filename in files: set_file_privs(os.path.join(root, filename)) # now set privs for the dest itself if os.path.isdir(dest): os.chmod(dest, max_privs) else: set_file_privs(dest) def ssh_command(self, connect_timeout=300, options='', alive_interval=300): options = "%s %s" % (options, self.master_ssh_option) base_cmd = _make_ssh_command(user=self.user, port=self.port, key_file=self.key_file, opts=options, hosts_file=self.known_hosts_file, connect_timeout=connect_timeout, alive_interval=alive_interval, extra_ssh_options=self.extra_ssh_options) return "%s %s" % (base_cmd, self.hostname) def run(self, command, timeout=None, ignore_status=False, connect_timeout=300, options='', verbose=True, args=None): raise NotImplementedError("Subclasses must implement " "the method 'run' ") def receive_files(self, src, dst, delete_dst=False, preserve_perm=True, preserve_symlinks=False, verbose=False, ssh_timeout=300): """ Copy files from the remote host to a local path. If both machines support rsync, that command will be used. If not, an scp command will be assembled. Directories will be copied recursively. If a src component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. :param src: Either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories :param dst: A file or a directory (if src contains a directory or more than one element, you must supply a directory dst). :param delete_dst: If this is true, the command will also clear out any old files at dest that are not in the src :param preserve_perm: Tells get_file() to try to preserve the sources permissions on files and dirs. :param preserve_symlinks: Try to preserve symlinks instead of transforming them into files/dirs on copy. :param verbose: Log commands being used and their outputs. :param ssh_timeout: Timeout is used for ssh_run() :raises: process.CmdError if the remote copy command failed. """ self.log.debug('Receive files (src) %s -> (dst) %s', src, dst) # Start a master SSH connection if necessary. self.start_master_ssh() if isinstance(src, basestring): src = [src] dst = os.path.abspath(dst) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(src) local_dest = astring.shell_escape(dst) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dst, preserve_symlinks) ssh_run(rsync, shell=True, extra_text=self.hostname, verbose=verbose, timeout=ssh_timeout) try_scp = False except process.CmdError, e: self.log.warning("Trying scp, rsync failed: %s", e) # Make sure master ssh available self.start_master_ssh() if try_scp: # scp has no equivalent to --delete, just drop the entire dest dir if delete_dst and os.path.isdir(dst): shutil.rmtree(dst) os.mkdir(dst) remote_source = self._make_rsync_compatible_source(src, False) if remote_source: # _make_rsync_compatible_source() already did the escaping remote_source = self._encode_remote_paths(remote_source, escape=False) local_dest = astring.shell_escape(dst) scp = self._make_scp_cmd([remote_source], local_dest) ssh_run(scp, shell=True, extra_text=self.hostname, verbose=verbose, timeout=ssh_timeout) if not preserve_perm: # we have no way to tell scp to not try to preserve the # permissions so set them after copy instead. # for rsync we could use "--no-p --chmod=ugo=rwX" but those # options are only in very recent rsync versions self._set_umask_perms(dst)
def send_files(self, src, dst, delete_dst=False, preserve_symlinks=False, verbose=False, ssh_timeout=None): """ Copy files from a local path to the remote host. If both machines support rsync, that command will be used. If not, an scp command will be assembled. Directories will be copied recursively. If a src component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. :param src: Either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories :param dst: A file or a directory (if src contains a directory or more than one element, you must supply a directory dst). :param delete_dst: If this is true, the command will also clear out any old files at dest that are not in the src :param preserve_symlinks: Try to preserve symlinks instead of transforming them into files/dirs on copy. :param verbose: Log commands being used and their outputs. :param ssh_timeout: Timeout is used for self.ssh_run() :raises: invoke.exceptions.UnexpectedExit, invoke.exceptions.Failure if the remote copy command failed """ self.log.debug('Send files (src) %s -> (dst) %s', src, dst) # Start a master SSH connection if necessary. source_is_dir = False if isinstance(src, basestring): source_is_dir = os.path.isdir(src) src = [src] remote_dest = self._encode_remote_paths([dst]) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: local_sources = [astring.shell_escape(path) for path in src] rsync = self._make_rsync_cmd(local_sources, remote_dest, delete_dst, preserve_symlinks) self.connection.local(rsync, encoding='utf-8') try_scp = False except (Failure, UnexpectedExit) as details: self.log.warning("Trying scp, rsync failed: %s", details) if try_scp: # scp has no equivalent to --delete, just drop the entire dest dir if delete_dst: dest_exists = False try: r = self.run("test -x %s" % dst, verbose=False) if r.ok: dest_exists = True except (Failure, UnexpectedExit): pass dest_is_dir = False if dest_exists: try: r = self.run("test -d %s" % dst, verbose=False) if r.ok: dest_is_dir = True except (Failure, UnexpectedExit): pass # If there is a list of more than one path, dst *has* # to be a dir. If there's a single path being transferred and # it is a dir, the dst also has to be a dir. Therefore # it has to be created on the remote machine in case it doesn't # exist, otherwise we will have an scp failure. if len(src) > 1 or source_is_dir: dest_is_dir = True if dest_exists and dest_is_dir: cmd = "rm -rf %s && mkdir %s" % (dst, dst) self.run(cmd, verbose=verbose) elif not dest_exists and dest_is_dir: cmd = "mkdir %s" % dst self.run(cmd, verbose=verbose) local_sources = self._make_rsync_compatible_source(src, True) if local_sources: scp = self._make_scp_cmd(local_sources, remote_dest) r = self.connection.local(scp) self.log.info('Command {} with status {}'.format( r.command, r.exited))
def receive_files(self, src, dst, delete_dst=False, preserve_perm=True, preserve_symlinks=False, verbose=False, ssh_timeout=300): """ Copy files from the remote host to a local path. If both machines support rsync, that command will be used. If not, an scp command will be assembled. Directories will be copied recursively. If a src component is a directory with a trailing slash, the content of the directory will be copied, otherwise, the directory itself and its content will be copied. This behavior is similar to that of the program 'rsync'. :param src: Either 1) a single file or directory, as a string 2) a list of one or more (possibly mixed) files or directories :param dst: A file or a directory (if src contains a directory or more than one element, you must supply a directory dst). :param delete_dst: If this is true, the command will also clear out any old files at dest that are not in the src :param preserve_perm: Tells get_file() to try to preserve the sources permissions on files and dirs. :param preserve_symlinks: Try to preserve symlinks instead of transforming them into files/dirs on copy. :param verbose: Log commands being used and their outputs. :param ssh_timeout: Timeout is used for self.ssh_run() :raises: invoke.exceptions.UnexpectedExit, invoke.exceptions.Failure if the remote copy command failed. """ self.log.debug('Receive files (src) %s -> (dst) %s', src, dst) # Start a master SSH connection if necessary. if isinstance(src, basestring): src = [src] dst = os.path.abspath(dst) # If rsync is disabled or fails, try scp. try_scp = True if self.use_rsync(): try: remote_source = self._encode_remote_paths(src) local_dest = astring.shell_escape(dst) rsync = self._make_rsync_cmd([remote_source], local_dest, delete_dst, preserve_symlinks) result = self.connection.local(rsync, encoding='utf-8') self.log.info(result.exited) try_scp = False except (Failure, UnexpectedExit) as e: self.log.warning("Trying scp, rsync failed: %s", e) # Make sure master ssh available if try_scp: # scp has no equivalent to --delete, just drop the entire dest dir if delete_dst and os.path.isdir(dst): shutil.rmtree(dst) os.mkdir(dst) remote_source = self._make_rsync_compatible_source(src, False) if remote_source: # _make_rsync_compatible_source() already did the escaping remote_source = self._encode_remote_paths(remote_source, escape=False) local_dest = astring.shell_escape(dst) scp = self._make_scp_cmd([remote_source], local_dest) r = self.connection.local(scp) self.log.info("Command {} with status {}".format( r.command, r.exited)) if not preserve_perm: # we have no way to tell scp to not try to preserve the # permissions so set them after copy instead. # for rsync we could use "--no-p --chmod=ugo=rwX" but those # options are only in very recent rsync versions self._set_umask_perms(dst)