def _run(self, cmd, timeout, verbose, ignore_status, connect_timeout,
env, options, args, log_file):
ssh_cmd = self.ssh_command(connect_timeout, options)
if not env.strip():
env = ""
else:
env = "export %s;" % env
for arg in args:
cmd += ' "%s"' % astring.shell_escape(arg)
if env:
full_cmd = '%s "%s %s"' % (ssh_cmd, env, astring.shell_escape(cmd))
else:
full_cmd = '%s "%s"' % (ssh_cmd, astring.shell_escape(cmd))
result = ssh_run(full_cmd, verbose=verbose,
ignore_status=ignore_status, timeout=timeout,
extra_text=self.hostname, shell=True,
log_file=log_file)
# The error messages will show up in band (indistinguishable
# from stuff sent through the SSH connection), so we have the
# remote computer echo the message "Connected." before running
# any cmd. Since the following 2 errors have to do with
# connecting, it's safe to do these checks.
if result.exit_status == 255:
if re.search(r'^ssh: connect to host .* port .*: '
r'Connection timed out\r$', result.stderr):
raise SSHTimeout("SSH timed out:\n%s" % result)
if "Permission denied." in result.stderr:
raise SSHPermissionDeniedError("SSH permission denied:\n%s" %
result)
if not ignore_status and result.exit_status > 0:
raise process.CmdError(command=full_cmd, result=result)
return result
def _run(self, cmd, timeout, verbose, ignore_status, connect_timeout,
env, options, args, log_file, watch_stdout_pattern):
ssh_cmd = self.ssh_command(connect_timeout, options)
if not env.strip():
env = ""
else:
env = "export %s;" % env
for arg in args:
cmd += ' "%s"' % astring.shell_escape(arg)
if env:
full_cmd = '%s "%s %s"' % (ssh_cmd, env, astring.shell_escape(cmd))
else:
full_cmd = '%s "%s"' % (ssh_cmd, astring.shell_escape(cmd))
result = ssh_run(full_cmd, verbose=verbose,
ignore_status=ignore_status, timeout=timeout,
extra_text=self.hostname, shell=True,
log_file=log_file,
watch_stdout_pattern=watch_stdout_pattern)
# The error messages will show up in band (indistinguishable
# from stuff sent through the SSH connection), so we have the
# remote computer echo the message "Connected." before running
# any cmd. Since the following 2 errors have to do with
# connecting, it's safe to do these checks.
if result.exit_status == 255:
if re.search(r'^ssh: connect to host .* port .*: '
r'Connection timed out\r$', result.stderr):
raise SSHTimeout("SSH timed out:\n%s" % result)
if "Permission denied." in result.stderr:
raise SSHPermissionDeniedError("SSH permission denied:\n%s" %
result)
if not ignore_status and result.exit_status > 0:
raise process.CmdError(command=full_cmd, result=result)
return result
def send_files(self,
src,
dst,
delete_dst=False,
preserve_symlinks=False,
verbose=False,
ssh_timeout=None):
"""
Copy files from a local path to the remote host.
If both machines support rsync, that command will be used.
If not, an scp command will be assembled. Directories will be
copied recursively. If a src component is a directory with a
trailing slash, the content of the directory will be copied,
otherwise, the directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
:param src: Either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed) files or directories
:param dst: A file or a directory (if src contains a
directory or more than one element, you must supply a
directory dst).
:param delete_dst: If this is true, the command will also clear
out any old files at dest that are not in the src
:param preserve_symlinks: Try to preserve symlinks instead of
transforming them into files/dirs on copy.
:param verbose: Log commands being used and their outputs.
:param ssh_timeout: Timeout is used for self.ssh_run()
:raises: process.CmdError if the remote copy command failed
"""
self.log.debug('Send files (src) %s -> (dst) %s', src, dst)
# Start a master SSH connection if necessary.
self.start_master_ssh()
source_is_dir = False
if isinstance(src, basestring):
source_is_dir = os.path.isdir(src)
src = [src]
remote_dest = self._encode_remote_paths([dst])
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
local_sources = [astring.shell_escape(path) for path in src]
rsync = self._make_rsync_cmd(local_sources, remote_dest,
delete_dst, preserve_symlinks)
self.ssh_run(rsync,
shell=True,
extra_text=self.hostname,
verbose=verbose,
timeout=ssh_timeout)
try_scp = False
except process.CmdError, details:
self.log.warning("Trying scp, rsync failed: %s", details)
# Make sure master ssh available
self.start_master_ssh()
def receive_files(self, src, dst, delete_dst=False,
preserve_perm=True, preserve_symlinks=False,
verbose=False, ssh_timeout=300):
"""
Copy files from the remote host to a local path.
If both machines support rsync, that command will be used.
If not, an scp command will be assembled. Directories will be
copied recursively. If a src component is a directory with a
trailing slash, the content of the directory will be copied,
otherwise, the directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
:param src: Either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed) files or directories
:param dst: A file or a directory (if src contains a
directory or more than one element, you must supply a
directory dst).
:param delete_dst: If this is true, the command will also clear
out any old files at dest that are not in the src
:param preserve_perm: Tells get_file() to try to preserve the sources
permissions on files and dirs.
:param preserve_symlinks: Try to preserve symlinks instead of
transforming them into files/dirs on copy.
:param verbose: Log commands being used and their outputs.
:param ssh_timeout: Timeout is used for self.ssh_run()
:raises: process.CmdError if the remote copy command failed.
"""
self.log.debug('Receive files (src) %s -> (dst) %s', src, dst)
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(src, basestring):
src = [src]
dst = os.path.abspath(dst)
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
remote_source = self._encode_remote_paths(src)
local_dest = astring.shell_escape(dst)
rsync = self._make_rsync_cmd([remote_source], local_dest,
delete_dst, preserve_symlinks)
self.ssh_run(rsync, shell=True, extra_text=self.hostname,
verbose=verbose, timeout=ssh_timeout)
try_scp = False
except process.CmdError, e:
self.log.warning("Trying scp, rsync failed: %s", e)
# Make sure master ssh available
self.start_master_ssh()
def git_cmd(self, cmd, ignore_status=False):
"""
Wraps git commands.
:param cmd: Command to be executed.
:param ignore_status: Whether we should suppress error.CmdError
exceptions if the command did return exit code !=0 (True), or
not suppress them (False).
"""
os.chdir(self.destination_dir)
return process.run(r"%s %s" % (self.cmd, astring.shell_escape(cmd)),
ignore_status=ignore_status)
def _make_ssh_cmd(self, cmd):
"""
Create a base ssh command string for the host which can be used
to run commands directly on the machine
"""
base_cmd = _make_ssh_command(user=self.user, port=self.port,
key_file=self.key_file,
opts=self.master_ssh_option,
hosts_file=self.known_hosts_file)
return '%s %s "%s"' % (base_cmd, self.hostname,
astring.shell_escape(cmd))
def _make_ssh_cmd(self, cmd):
"""
Create a base ssh command string for the host which can be used
to run commands directly on the machine
"""
base_cmd = _make_ssh_command(user=self.user, port=self.port,
key_file=self.key_file,
opts=self.master_ssh_option,
hosts_file=self.known_hosts_file)
return '%s %s "%s"' % (base_cmd, self.hostname,
astring.shell_escape(cmd))
def receive_files(self, src, dst, delete_dst=False,
preserve_perm=True, preserve_symlinks=False,
verbose=False):
"""
Copy files from the remote host to a local path.
If both machines support rsync, that command will be used.
If not, an scp command will be assembled. Directories will be
copied recursively. If a src component is a directory with a
trailing slash, the content of the directory will be copied,
otherwise, the directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
:param src: Either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed) files or directories
:param dst: A file or a directory (if src contains a
directory or more than one element, you must supply a
directory dst).
:param delete_dst: If this is true, the command will also clear
out any old files at dest that are not in the src
:param preserve_perm: Tells get_file() to try to preserve the sources
permissions on files and dirs.
:param preserve_symlinks: Try to preserve symlinks instead of
transforming them into files/dirs on copy.
:param verbose: Log commands being used and their outputs.
:raises: process.CmdError if the remote copy command failed.
"""
self.log.debug('Receive files (src) %s -> (dst) %s', src, dst)
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(src, basestring):
src = [src]
dst = os.path.abspath(dst)
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
remote_source = self._encode_remote_paths(src)
local_dest = astring.shell_escape(dst)
rsync = self._make_rsync_cmd([remote_source], local_dest,
delete_dst, preserve_symlinks)
ssh_run(rsync, shell=True, extra_text=self.hostname,
verbose=verbose)
try_scp = False
except process.CmdError, e:
self.log.warning("Trying scp, rsync failed: %s", e)
def send_files(self, src, dst, delete_dst=False,
preserve_symlinks=False, verbose=False):
"""
Copy files from a local path to the remote host.
If both machines support rsync, that command will be used.
If not, an scp command will be assembled. Directories will be
copied recursively. If a src component is a directory with a
trailing slash, the content of the directory will be copied,
otherwise, the directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
:param src: Either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed) files or directories
:param dst: A file or a directory (if src contains a
directory or more than one element, you must supply a
directory dst).
:param delete_dst: If this is true, the command will also clear
out any old files at dest that are not in the src
:param preserve_symlinks: Try to preserve symlinks instead of
transforming them into files/dirs on copy.
:param verbose: Log commands being used and their outputs.
:raises: process.CmdError if the remote copy command failed
"""
self.log.debug('Send files (src) %s -> (dst) %s', src, dst)
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(src, basestring):
source_is_dir = os.path.isdir(src)
src = [src]
remote_dest = self._encode_remote_paths([dst])
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
local_sources = [astring.shell_escape(path) for path in src]
rsync = self._make_rsync_cmd(local_sources, remote_dest,
delete_dst, preserve_symlinks)
ssh_run(rsync, shell=True, extra_text=self.hostname,
verbose=verbose)
try_scp = False
except process.CmdError, details:
self.log.warning("Trying scp, rsync failed: %s", details)
def _make_rsync_compatible_globs(self, pth, is_local):
"""
Given an rsync-style path (pth), returns a list of globbed paths.
Those will hopefully provide equivalent behaviour for scp. Does not
support the full range of rsync pattern matching behaviour, only that
exposed in the get/send_file interface (trailing slashes).
:param pth: rsync-style path.
:param is_local: Whether the paths should be interpreted as local or
remote paths.
"""
# non-trailing slash paths should just work
if len(pth) == 0 or pth[-1] != "/":
return [pth]
# make a function to test if a pattern matches any files
if is_local:
def glob_matches_files(path, pattern):
return len(glob.glob(path + pattern)) > 0
else:
def glob_matches_files(path, pattern):
match_cmd = "ls \"%s\"%s" % (astring.shell_escape(path),
pattern)
result = self.run(match_cmd, ignore_status=True)
return result.exit_status == 0
# take a set of globs that cover all files, and see which are needed
patterns = ["*", ".[!.]*"]
patterns = [p for p in patterns if glob_matches_files(pth, p)]
# convert them into a set of paths suitable for the commandline
if is_local:
return [
"\"%s\"%s" % (astring.shell_escape(pth), pattern)
for pattern in patterns
]
else:
return [_scp_remote_escape(pth) + pattern for pattern in patterns]
def _make_rsync_compatible_globs(self, pth, is_local):
"""
Given an rsync-style path (pth), returns a list of globbed paths.
Those will hopefully provide equivalent behaviour for scp. Does not
support the full range of rsync pattern matching behaviour, only that
exposed in the get/send_file interface (trailing slashes).
:param pth: rsync-style path.
:param is_local: Whether the paths should be interpreted as local or
remote paths.
"""
# non-trailing slash paths should just work
if len(pth) == 0 or pth[-1] != "/":
return [pth]
# make a function to test if a pattern matches any files
if is_local:
def glob_matches_files(path, pattern):
return len(glob.glob(path + pattern)) > 0
else:
def glob_matches_files(path, pattern):
match_cmd = 'ls "%s"%s' % (astring.shell_escape(path), pattern)
result = self.run(match_cmd, ignore_status=True)
return result.exit_status == 0
# take a set of globs that cover all files, and see which are needed
patterns = ["*", ".[!.]*"]
patterns = [p for p in patterns if glob_matches_files(pth, p)]
# convert them into a set of paths suitable for the commandline
if is_local:
return ['"%s"%s' % (astring.shell_escape(pth), pattern) for pattern in patterns]
else:
return [_scp_remote_escape(pth) + pattern for pattern in patterns]
def _scp_remote_escape(filename):
"""
Escape special chars for SCP use.
Bis-quoting has to be used with scp for remote files, "bis-quoting"
as in quoting x 2. SCP does not support a newline in the filename.
:param filename: the filename string to escape.
:returns: The escaped filename string. The required englobing double
quotes are NOT added and so should be added at some point by
the caller.
"""
escape_chars = r' !"$&' "'" r'()*,:;<=>?[\]^`{|}'
new_name = []
for char in filename:
if char in escape_chars:
new_name.append("\\%s" % (char,))
else:
new_name.append(char)
return astring.shell_escape("".join(new_name))
def _scp_remote_escape(filename):
"""
Escape special chars for SCP use.
Bis-quoting has to be used with scp for remote files, "bis-quoting"
as in quoting x 2. SCP does not support a newline in the filename.
:param filename: the filename string to escape.
:returns: The escaped filename string. The required englobing double
quotes are NOT added and so should be added at some point by
the caller.
"""
escape_chars = r' !"$&' "'" r'()*,:;<=>?[\]^`{|}'
new_name = []
for char in filename:
if char in escape_chars:
new_name.append("\\%s" % (char,))
else:
new_name.append(char)
return astring.shell_escape("".join(new_name))
def glob_matches_files(path, pattern):
match_cmd = "ls \"%s\"%s" % (astring.shell_escape(path), pattern)
result = self.run(match_cmd, ignore_status=True)
return result.exit_status == 0
class BaseRemote(object):
def __init__(self, hostname, user="******", port=22, password="",
key_file=None, wait_key_installed=0, extra_ssh_options=""):
self.env = {}
self.hostname = hostname
self.ip = socket.getaddrinfo(self.hostname, None)[0][4][0]
self.user = user
self.port = port
self.password = password
self.key_file = key_file
self._use_rsync = None
self.known_hosts_file = tempfile.mkstemp()[1]
self.master_ssh_job = None
self.master_ssh_tempdir = None
self.master_ssh_option = ''
self.extra_ssh_options = extra_ssh_options
logger = logging.getLogger('avocado.test')
self.log = SDCMAdapter(logger, extra={'prefix': str(self)})
time.sleep(wait_key_installed)
self._check_install_key_required()
def _check_install_key_required(self):
def _safe_ssh_ping():
try:
self._ssh_ping()
return True
except (SSHPermissionDeniedError, process.CmdError):
return None
if not self.key_file and self.password:
try:
self._ssh_ping()
except (SSHPermissionDeniedError, process.CmdError):
copy_id_cmd = ('ssh-copy-id -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s %s@%s' %
(self.port, self.user, self.hostname))
while True:
try:
expect = aexpect.Expect(copy_id_cmd)
expect.read_until_output_matches(['.*password:'******'Waiting for password-less SSH')
if result is None:
raise SSHPermissionDeniedError('Unable to configure '
'password less SSH. '
'Output of %s: %s' %
(copy_id_cmd,
expect.get_output()))
else:
self.log.info('Successfully configured SSH key auth')
def ssh_debug_cmd(self):
if self.key_file:
return "SSH access -> 'ssh -i %s %s@%s'" % (self.key_file,
self.user,
self.ip)
else:
return "SSH access -> 'ssh %s@%s'" % (self.user,
self.ip)
def __str__(self):
return 'Remote [%s@%s]' % (self.user, self.hostname)
def use_rsync(self):
if self._use_rsync is not None:
return self._use_rsync
# Check if rsync is available on the remote host. If it's not,
# don't try to use it for any future file transfers.
self._use_rsync = self._check_rsync()
if not self._use_rsync:
self.log.warning("Command rsync not available -- disabled")
return self._use_rsync
def _check_rsync(self):
"""
Check if rsync is available on the remote host.
"""
try:
self.run("rsync --version", verbose=False)
except process.CmdError:
return False
return True
def _encode_remote_paths(self, paths, escape=True):
"""
Given a list of file paths, encodes it as a single remote path, in
the style used by rsync and scp.
"""
if escape:
paths = [_scp_remote_escape(path) for path in paths]
return '%s@%s:"%s"' % (self.user, self.hostname, " ".join(paths))
def _make_rsync_cmd(self, src, dst, delete_dst, preserve_symlinks):
"""
Given a list of source paths and a destination path, produces the
appropriate rsync command for copying them. Remote paths must be
pre-encoded.
"""
ssh_cmd = _make_ssh_command(user=self.user, port=self.port,
opts=self.master_ssh_option,
hosts_file=self.known_hosts_file,
key_file=self.key_file,
extra_ssh_options=self.extra_ssh_options.replace('-tt', '-t'))
if delete_dst:
delete_flag = "--delete"
else:
delete_flag = ""
if preserve_symlinks:
symlink_flag = ""
else:
symlink_flag = "-L"
command = "rsync %s %s --timeout=300 --rsh='%s' -az %s %s"
return command % (symlink_flag, delete_flag, ssh_cmd,
" ".join(src), dst)
def _make_ssh_cmd(self, cmd):
"""
Create a base ssh command string for the host which can be used
to run commands directly on the machine
"""
base_cmd = _make_ssh_command(user=self.user, port=self.port,
key_file=self.key_file,
opts=self.master_ssh_option,
hosts_file=self.known_hosts_file,
extra_ssh_options=self.extra_ssh_options)
return '%s %s "%s"' % (base_cmd, self.hostname,
astring.shell_escape(cmd))
def _make_scp_cmd(self, src, dst, connect_timeout=300, alive_interval=300):
"""
Given a list of source paths and a destination path, produces the
appropriate scp command for encoding it. Remote paths must be
pre-encoded.
"""
key_option = ''
if self.key_file:
key_option = '-i %s' % os.path.expanduser(self.key_file)
command = ("scp -r %s -o StrictHostKeyChecking=no -o BatchMode=yes "
"-o ConnectTimeout=%d -o ServerAliveInterval=%d "
"-o UserKnownHostsFile=%s -P %d %s %s '%s'")
return command % (self.master_ssh_option, connect_timeout,
alive_interval, self.known_hosts_file,
self.port, key_option, " ".join(src), dst)
def _make_rsync_compatible_globs(self, pth, is_local):
"""
Given an rsync-style path (pth), returns a list of globbed paths.
Those will hopefully provide equivalent behaviour for scp. Does not
support the full range of rsync pattern matching behaviour, only that
exposed in the get/send_file interface (trailing slashes).
:param pth: rsync-style path.
:param is_local: Whether the paths should be interpreted as local or
remote paths.
"""
# non-trailing slash paths should just work
if len(pth) == 0 or pth[-1] != "/":
return [pth]
# make a function to test if a pattern matches any files
if is_local:
def glob_matches_files(path, pattern):
return len(glob.glob(path + pattern)) > 0
else:
def glob_matches_files(path, pattern):
match_cmd = "ls \"%s\"%s" % (astring.shell_escape(path), pattern)
result = self.run(match_cmd, ignore_status=True)
return result.exit_status == 0
# take a set of globs that cover all files, and see which are needed
patterns = ["*", ".[!.]*"]
patterns = [p for p in patterns if glob_matches_files(pth, p)]
# convert them into a set of paths suitable for the commandline
if is_local:
return ["\"%s\"%s" % (astring.shell_escape(pth), pattern)
for pattern in patterns]
else:
return [_scp_remote_escape(pth) + pattern
for pattern in patterns]
def _make_rsync_compatible_source(self, source, is_local):
"""
Make an rsync compatible source string.
Applies the same logic as _make_rsync_compatible_globs, but
applies it to an entire list of sources, producing a new list of
sources, properly quoted.
"""
return sum((self._make_rsync_compatible_globs(path, is_local)
for path in source), [])
def _set_umask_perms(self, dest):
"""
Set permissions on all files and directories.
Given a destination file/dir (recursively) set the permissions on
all the files and directories to the max allowed by running umask.
"""
# now this looks strange but I haven't found a way in Python to _just_
# get the umask, apparently the only option is to try to set it
umask = os.umask(0)
os.umask(umask)
max_privs = 0777 & ~umask
def set_file_privs(filename):
file_stat = os.stat(filename)
file_privs = max_privs
# if the original file permissions do not have at least one
# executable bit then do not set it anywhere
if not file_stat.st_mode & 0111:
file_privs &= ~0111
os.chmod(filename, file_privs)
# try a bottom-up walk so changes on directory permissions won't cut
# our access to the files/directories inside it
for root, dirs, files in os.walk(dest, topdown=False):
# when setting the privileges we emulate the chmod "X" behaviour
# that sets to execute only if it is a directory or any of the
# owner/group/other already has execute right
for dirname in dirs:
os.chmod(os.path.join(root, dirname), max_privs)
for filename in files:
set_file_privs(os.path.join(root, filename))
# now set privs for the dest itself
if os.path.isdir(dest):
os.chmod(dest, max_privs)
else:
set_file_privs(dest)
def ssh_command(self, connect_timeout=300, options='', alive_interval=300):
options = "%s %s" % (options, self.master_ssh_option)
base_cmd = _make_ssh_command(user=self.user, port=self.port,
key_file=self.key_file,
opts=options,
hosts_file=self.known_hosts_file,
connect_timeout=connect_timeout,
alive_interval=alive_interval,
extra_ssh_options=self.extra_ssh_options)
return "%s %s" % (base_cmd, self.hostname)
def run(self, command, timeout=None, ignore_status=False,
connect_timeout=300, options='', verbose=True, args=None):
raise NotImplementedError("Subclasses must implement "
"the method 'run' ")
def receive_files(self, src, dst, delete_dst=False,
preserve_perm=True, preserve_symlinks=False,
verbose=False, ssh_timeout=300):
"""
Copy files from the remote host to a local path.
If both machines support rsync, that command will be used.
If not, an scp command will be assembled. Directories will be
copied recursively. If a src component is a directory with a
trailing slash, the content of the directory will be copied,
otherwise, the directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
:param src: Either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed) files or directories
:param dst: A file or a directory (if src contains a
directory or more than one element, you must supply a
directory dst).
:param delete_dst: If this is true, the command will also clear
out any old files at dest that are not in the src
:param preserve_perm: Tells get_file() to try to preserve the sources
permissions on files and dirs.
:param preserve_symlinks: Try to preserve symlinks instead of
transforming them into files/dirs on copy.
:param verbose: Log commands being used and their outputs.
:param ssh_timeout: Timeout is used for ssh_run()
:raises: process.CmdError if the remote copy command failed.
"""
self.log.debug('Receive files (src) %s -> (dst) %s', src, dst)
# Start a master SSH connection if necessary.
self.start_master_ssh()
if isinstance(src, basestring):
src = [src]
dst = os.path.abspath(dst)
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
remote_source = self._encode_remote_paths(src)
local_dest = astring.shell_escape(dst)
rsync = self._make_rsync_cmd([remote_source], local_dest,
delete_dst, preserve_symlinks)
ssh_run(rsync, shell=True, extra_text=self.hostname,
verbose=verbose, timeout=ssh_timeout)
try_scp = False
except process.CmdError, e:
self.log.warning("Trying scp, rsync failed: %s", e)
# Make sure master ssh available
self.start_master_ssh()
if try_scp:
# scp has no equivalent to --delete, just drop the entire dest dir
if delete_dst and os.path.isdir(dst):
shutil.rmtree(dst)
os.mkdir(dst)
remote_source = self._make_rsync_compatible_source(src, False)
if remote_source:
# _make_rsync_compatible_source() already did the escaping
remote_source = self._encode_remote_paths(remote_source,
escape=False)
local_dest = astring.shell_escape(dst)
scp = self._make_scp_cmd([remote_source], local_dest)
ssh_run(scp, shell=True, extra_text=self.hostname,
verbose=verbose, timeout=ssh_timeout)
if not preserve_perm:
# we have no way to tell scp to not try to preserve the
# permissions so set them after copy instead.
# for rsync we could use "--no-p --chmod=ugo=rwX" but those
# options are only in very recent rsync versions
self._set_umask_perms(dst)
def send_files(self,
src,
dst,
delete_dst=False,
preserve_symlinks=False,
verbose=False,
ssh_timeout=None):
"""
Copy files from a local path to the remote host.
If both machines support rsync, that command will be used.
If not, an scp command will be assembled. Directories will be
copied recursively. If a src component is a directory with a
trailing slash, the content of the directory will be copied,
otherwise, the directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
:param src: Either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed) files or directories
:param dst: A file or a directory (if src contains a
directory or more than one element, you must supply a
directory dst).
:param delete_dst: If this is true, the command will also clear
out any old files at dest that are not in the src
:param preserve_symlinks: Try to preserve symlinks instead of
transforming them into files/dirs on copy.
:param verbose: Log commands being used and their outputs.
:param ssh_timeout: Timeout is used for self.ssh_run()
:raises: invoke.exceptions.UnexpectedExit, invoke.exceptions.Failure if the remote copy command failed
"""
self.log.debug('Send files (src) %s -> (dst) %s', src, dst)
# Start a master SSH connection if necessary.
source_is_dir = False
if isinstance(src, basestring):
source_is_dir = os.path.isdir(src)
src = [src]
remote_dest = self._encode_remote_paths([dst])
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
local_sources = [astring.shell_escape(path) for path in src]
rsync = self._make_rsync_cmd(local_sources, remote_dest,
delete_dst, preserve_symlinks)
self.connection.local(rsync, encoding='utf-8')
try_scp = False
except (Failure, UnexpectedExit) as details:
self.log.warning("Trying scp, rsync failed: %s", details)
if try_scp:
# scp has no equivalent to --delete, just drop the entire dest dir
if delete_dst:
dest_exists = False
try:
r = self.run("test -x %s" % dst, verbose=False)
if r.ok:
dest_exists = True
except (Failure, UnexpectedExit):
pass
dest_is_dir = False
if dest_exists:
try:
r = self.run("test -d %s" % dst, verbose=False)
if r.ok:
dest_is_dir = True
except (Failure, UnexpectedExit):
pass
# If there is a list of more than one path, dst *has*
# to be a dir. If there's a single path being transferred and
# it is a dir, the dst also has to be a dir. Therefore
# it has to be created on the remote machine in case it doesn't
# exist, otherwise we will have an scp failure.
if len(src) > 1 or source_is_dir:
dest_is_dir = True
if dest_exists and dest_is_dir:
cmd = "rm -rf %s && mkdir %s" % (dst, dst)
self.run(cmd, verbose=verbose)
elif not dest_exists and dest_is_dir:
cmd = "mkdir %s" % dst
self.run(cmd, verbose=verbose)
local_sources = self._make_rsync_compatible_source(src, True)
if local_sources:
scp = self._make_scp_cmd(local_sources, remote_dest)
r = self.connection.local(scp)
self.log.info('Command {} with status {}'.format(
r.command, r.exited))
def receive_files(self,
src,
dst,
delete_dst=False,
preserve_perm=True,
preserve_symlinks=False,
verbose=False,
ssh_timeout=300):
"""
Copy files from the remote host to a local path.
If both machines support rsync, that command will be used.
If not, an scp command will be assembled. Directories will be
copied recursively. If a src component is a directory with a
trailing slash, the content of the directory will be copied,
otherwise, the directory itself and its content will be copied. This
behavior is similar to that of the program 'rsync'.
:param src: Either
1) a single file or directory, as a string
2) a list of one or more (possibly mixed) files or directories
:param dst: A file or a directory (if src contains a
directory or more than one element, you must supply a
directory dst).
:param delete_dst: If this is true, the command will also clear
out any old files at dest that are not in the src
:param preserve_perm: Tells get_file() to try to preserve the sources
permissions on files and dirs.
:param preserve_symlinks: Try to preserve symlinks instead of
transforming them into files/dirs on copy.
:param verbose: Log commands being used and their outputs.
:param ssh_timeout: Timeout is used for self.ssh_run()
:raises: invoke.exceptions.UnexpectedExit, invoke.exceptions.Failure if the remote copy command failed.
"""
self.log.debug('Receive files (src) %s -> (dst) %s', src, dst)
# Start a master SSH connection if necessary.
if isinstance(src, basestring):
src = [src]
dst = os.path.abspath(dst)
# If rsync is disabled or fails, try scp.
try_scp = True
if self.use_rsync():
try:
remote_source = self._encode_remote_paths(src)
local_dest = astring.shell_escape(dst)
rsync = self._make_rsync_cmd([remote_source], local_dest,
delete_dst, preserve_symlinks)
result = self.connection.local(rsync, encoding='utf-8')
self.log.info(result.exited)
try_scp = False
except (Failure, UnexpectedExit) as e:
self.log.warning("Trying scp, rsync failed: %s", e)
# Make sure master ssh available
if try_scp:
# scp has no equivalent to --delete, just drop the entire dest dir
if delete_dst and os.path.isdir(dst):
shutil.rmtree(dst)
os.mkdir(dst)
remote_source = self._make_rsync_compatible_source(src, False)
if remote_source:
# _make_rsync_compatible_source() already did the escaping
remote_source = self._encode_remote_paths(remote_source,
escape=False)
local_dest = astring.shell_escape(dst)
scp = self._make_scp_cmd([remote_source], local_dest)
r = self.connection.local(scp)
self.log.info("Command {} with status {}".format(
r.command, r.exited))
if not preserve_perm:
# we have no way to tell scp to not try to preserve the
# permissions so set them after copy instead.
# for rsync we could use "--no-p --chmod=ugo=rwX" but those
# options are only in very recent rsync versions
self._set_umask_perms(dst)
def glob_matches_files(path, pattern):
match_cmd = "ls \"%s\"%s" % (astring.shell_escape(path), pattern)
result = self.run(match_cmd, ignore_status=True)
return result.exit_status == 0
