def test_empty_everything(self):
"""
No env.key_filename and no ssh_config = empty list
"""
with settings(use_ssh_config=False):
with settings(key_filename=""):
eq_(key_filenames(), [])
with settings(key_filename=[]):
eq_(key_filenames(), [])
def test_empty_everything(self):
"""
No env.key_filename and no ssh_config = empty list
"""
with settings(use_ssh_config=False):
with settings(key_filename=""):
eq_(key_filenames(), [])
with settings(key_filename=[]):
eq_(key_filenames(), [])
def test_just_env(self):
"""
Valid env.key_filename and no ssh_config = just env
"""
with settings(use_ssh_config=False):
with settings(key_filename="mykey"):
eq_(key_filenames(), ["mykey"])
with settings(key_filename=["foo", "bar"]):
eq_(key_filenames(), ["foo", "bar"])
def test_just_env(self):
"""
Valid env.key_filename and no ssh_config = just env
"""
with settings(use_ssh_config=False):
with settings(key_filename="mykey"):
eq_(key_filenames(), ["mykey"])
with settings(key_filename=["foo", "bar"]):
eq_(key_filenames(), ["foo", "bar"])
def test_both(self):
"""
Both env.key_filename + valid ssh_config = both show up w/ env var first
"""
with settings(use_ssh_config=True, ssh_config_path=support("ssh_config")):
with settings(key_filename="bizbaz.pub"):
eq_(key_filenames(), ["bizbaz.pub", "foobar.pub"])
with settings(key_filename=["bizbaz.pub", "whatever.pub"]):
expected = ["bizbaz.pub", "whatever.pub", "foobar.pub"]
eq_(key_filenames(), expected)
def test_both(self):
"""
Both env.key_filename + valid ssh_config = both show up w/ env var first
"""
with settings(use_ssh_config=True, ssh_config_path=support("ssh_config")):
with settings(key_filename="bizbaz.pub"):
eq_(key_filenames(), ["bizbaz.pub", "foobar.pub"])
with settings(key_filename=["bizbaz.pub", "whatever.pub"]):
expected = ["bizbaz.pub", "whatever.pub", "foobar.pub"]
eq_(key_filenames(), expected)
def ssh(ssh_opts=''):
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
return local(cmd)
def ssh(ssh_opts=''):
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
return local(cmd)
def rsync(local_path, remote_path, extra_rsync_options=""): # pylint: disable=too-many-locals
"""Rsync files/directories from local path to remote_path.
.. note::
Using absolute ``local_path`` supported but not recommended.
Args:
local_path: Local path on local host, copy files/directories from it. Should be relative.
remote_path: Remote path on remote host, copy files/directories to it. Must be absolute.
extra_rsync_options: Additional rsync options added after default '-aH --stats --force --timeout=600'
Returns:
True if some of remote files/directories are changed, False otherwise.
"""
files_dir = os.path.join(os.path.dirname(env.real_fabfile), 'files')
if not os.path.isdir(files_dir):
fname = str(inspect.stack()[1][1])
nline = str(inspect.stack()[1][2])
abort('rsync: files dir \'%s\' not exists in file %s line %s' % (files_dir, fname, nline))
local_abs_path = os.path.join(files_dir, local_path)
if not os.path.exists(local_abs_path):
fname = str(inspect.stack()[1][1])
nline = str(inspect.stack()[1][2])
abort('rsync: local path \'%s\' not exists in file %s line %s' % (local_abs_path, fname, nline))
if not os.path.isabs(remote_path):
fname = str(inspect.stack()[1][1])
nline = str(inspect.stack()[1][2])
abort('rsync: remote path \'%s\' must be absolute in file %s line %s' % (remote_path, fname, nline))
# ssh keys
ssh_keys = ""
keys = key_filenames()
if keys:
ssh_keys = " -i " + " -i ".join(keys)
# ssh port
user, host, port = normalize(env.host_string)
ssh_port = "-p %s" % port
# ssh options
ssh_options = "-e 'ssh %s%s'" % (ssh_port, ssh_keys)
# rsync options
rsync_options = '-aH --stats --force --timeout=600 %s %s --' % (ssh_options, extra_rsync_options)
# remote_prefix
if host.count(':') > 1:
# Square brackets are mandatory for IPv6 rsync address,
# even if port number is not specified
remote_prefix = "%s@[%s]" % (user, host)
else:
remote_prefix = "%s@%s" % (user, host)
# execute command
command = "rsync %s %s %s:%s" % (rsync_options, local_abs_path, remote_prefix, remote_path)
with settings(fabric.api.hide('everything')):
stdout = local(command, capture=True)
zero_transfer_regexp = re.compile(r'^Total transferred file size: 0 bytes$')
changed = True
for line in stdout.split('\n'):
line = line.strip()
if zero_transfer_regexp.match(line):
changed = False
break
return changed
def test_just_ssh_config(self):
"""
No env.key_filename + valid ssh_config = ssh value
"""
with settings(use_ssh_config=True, ssh_config_path=support("ssh_config")):
for val in ["", []]:
with settings(key_filename=val):
eq_(key_filenames(), ["foobar.pub"])
def test_just_ssh_config(self):
"""
No env.key_filename + valid ssh_config = ssh value
"""
with settings(use_ssh_config=True, ssh_config_path=support("ssh_config")):
for val in ["", []]:
with settings(key_filename=val):
eq_(key_filenames(), ["foobar.pub"])
def test_specific_host(self):
"""
SSH lookup aspect should correctly select per-host value
"""
with settings(use_ssh_config=True,
ssh_config_path=support("ssh_config"),
host_string="myhost"):
eq_(key_filenames(), ["neighbor.pub"])
def test_specific_host(self):
"""
SSH lookup aspect should correctly select per-host value
"""
with settings(
use_ssh_config=True,
ssh_config_path=support("ssh_config"),
host_string="myhost"
):
eq_(key_filenames(), ["neighbor.pub"])
def ssh(ssh_opts='', remote_cmd=None):
"""
This is the default ssh function for accessing various environments.
It handles connections that require a gateway like Skyscape.
You can optionally pass ssh options `ssh_opts` e.g
`ssh_opts =' -o StrictHostKeyChecking=no'`
The `remote_cmd` parameter allows you to run an interactive command against
a remote host, for example, you can use nsenter to run a shell inside
a docker container as follows:
`remote_cmd = 'nsenter --target $PID --mount --uts --ipc --net --pid'`
"""
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
elif "gateway_user" in env and env.gateway_user:
gw_user_host_string = ssh_host_string(env.gateway_user, env.gateway)
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
# Sometimes we want to run an interactive command against a remote host, for
# example nsenter to run a shell inside a running docker container
if remote_cmd is not None:
cmd = "{} '{}'".format(cmd, remote_cmd)
return local(cmd)
def ssh(ssh_opts='', remote_cmd=None):
"""
This is the default ssh function for accessing various environments.
It handles connections that require a gateway like Skyscape.
You can optionally pass ssh options `ssh_opts` e.g
`ssh_opts =' -o StrictHostKeyChecking=no'`
The `remote_cmd` parameter allows you to run an interactive command against
a remote host, for example, you can use nsenter to run a shell inside
a docker container as follows:
`remote_cmd = 'nsenter --target $PID --mount --uts --ipc --net --pid'`
"""
# Keys
key_string = ''
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Proxy
proxy_string = ''
if env.gateway:
gw_user, gw_host, gw_port = normalize(env.gateway)
gw_port_string = "-p %s" % gw_port
if '@' in env.gateway:
gw_user_host_string = env.gateway
elif "gateway_user" in env and env.gateway_user:
gw_user_host_string = ssh_host_string(env.gateway_user, env.gateway)
else:
gw_user_host_string = ssh_host_string(user, env.gateway)
proxy_string = '-o "ProxyCommand ssh {key_string} {ssh_opts} {gw_port_string} {gw_user_host_string} nc %h %p"'.format(
key_string=key_string,
ssh_opts=ssh_opts,
gw_port_string=gw_port_string,
gw_user_host_string=gw_user_host_string)
cmd = "ssh -A -o 'ServerAliveInterval 30' {key_string} {ssh_opts} {port_string} {user_host_string} {proxy_string}".format(
key_string=key_string,
ssh_opts=ssh_opts,
port_string=port_string,
user_host_string=ssh_host_string(user, host),
proxy_string=proxy_string)
# Sometimes we want to run an interactive command against a remote host, for
# example nsenter to run a shell inside a running docker container
if remote_cmd is not None:
cmd = "{} '{}'".format(cmd, remote_cmd)
return local(cmd)
def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False, extra_opts="", ssh_opts="", capture=False):
"""
Synchronize a remote directory with the current project directory via rsync.
Where ``upload_project()`` makes use of ``scp`` to copy one's entire
project every time it is invoked, ``rsync_project()`` uses the ``rsync``
command-line utility, which only transfers files newer than those on the
remote end.
``rsync_project()`` is thus a simple wrapper around ``rsync``; for
details on how ``rsync`` works, please see its manpage. ``rsync`` must be
installed on both your local and remote systems in order for this operation
to work correctly.
This function makes use of Fabric's ``local()`` operation, and returns the
output of that function call; thus it will return the stdout, if any, of
the resultant ``rsync`` call.
``rsync_project()`` takes the following parameters:
* ``remote_dir``: the only required parameter, this is the path to the
directory on the remote server. Due to how ``rsync`` is implemented, the
exact behavior depends on the value of ``local_dir``:
* If ``local_dir`` ends with a trailing slash, the files will be
dropped inside of ``remote_dir``. E.g.
``rsync_project("/home/username/project", "foldername/")`` will drop
the contents of ``foldername`` inside of ``/home/username/project``.
* If ``local_dir`` does **not** end with a trailing slash (and this
includes the default scenario, when ``local_dir`` is not specified),
``remote_dir`` is effectively the "parent" directory, and a new
directory named after ``local_dir`` will be created inside of it. So
``rsync_project("/home/username", "foldername")`` would create a new
directory ``/home/username/foldername`` (if needed) and place the
files there.
* ``local_dir``: by default, ``rsync_project`` uses your current working
directory as the source directory. This may be overridden by specifying
``local_dir``, which is a string passed verbatim to ``rsync``, and thus
may be a single directory (``"my_directory"``) or multiple directories
(``"dir1 dir2"``). See the ``rsync`` documentation for details.
* ``exclude``: optional, may be a single string, or an iterable of strings,
and is used to pass one or more ``--exclude`` options to ``rsync``.
* ``delete``: a boolean controlling whether ``rsync``'s ``--delete`` option
is used. If True, instructs ``rsync`` to remove remote files that no
longer exist locally. Defaults to False.
* ``extra_opts``: an optional, arbitrary string which you may use to pass
custom arguments or options to ``rsync``.
* ``ssh_opts``: Like ``extra_opts`` but specifically for the SSH options
string (rsync's ``--rsh`` flag.)
* ``capture``: Sent directly into an inner `~fabric.operations.local` call.
Furthermore, this function transparently honors Fabric's port and SSH key
settings. Calling this function when the current host string contains a
nonstandard port, or when ``env.key_filename`` is non-empty, will use the
specified port and/or SSH key filename(s).
For reference, the approximate ``rsync`` command-line call that is
constructed by this function is the following::
rsync [--delete] [--exclude exclude[0][, --exclude[1][, ...]]] \\
-pthrvz [extra_opts] <local_dir> <host_string>:<remote_dir>
.. versionadded:: 1.4.0
The ``ssh_opts`` keyword argument.
.. versionadded:: 1.4.1
The ``capture`` keyword argument.
"""
# Turn single-string exclude into a one-item list for consistency
if not hasattr(exclude, "__iter__"):
exclude = (exclude,)
# Create --exclude options from exclude list
exclude_opts = ' --exclude "%s"' * len(exclude)
# Double-backslash-escape
exclusions = tuple([str(s).replace('"', '\\\\"') for s in exclude])
# Honor SSH key(s)
key_string = ""
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# RSH
rsh_string = ""
rsh_parts = [key_string, port_string, ssh_opts]
if any(rsh_parts):
rsh_string = "--rsh='ssh %s'" % " ".join(rsh_parts)
# Set up options part of string
options_map = {
"delete": "--delete" if delete else "",
"exclude": exclude_opts % exclusions,
"rsh": rsh_string,
"extra": extra_opts,
}
options = "%(delete)s%(exclude)s -pthrvz %(extra)s %(rsh)s" % options_map
# Get local directory
if local_dir is None:
local_dir = "../" + getcwd().split(sep)[-1]
# Create and run final command string
cmd = "rsync %s %s %s@%s:%s" % (options, local_dir, user, host, remote_dir)
if output.running:
print("[%s] rsync_project: %s" % (env.host_string, cmd))
return local(cmd, capture=capture)
def _get_rsync(files, source='/', target='/', exclude=None, arguments=None):
with NamedTemporaryFile() as files_file:
with NamedTemporaryFile() as exclude_file:
cmd = ['rsync']
##
# Fix or set default files
if isinstance(files, str) or isinstance(files, unicode):
files = [files]
elif files is None:
files = []
##
# Fix or set default exclude
if isinstance(exclude, str) or isinstance(exclude, unicode):
exclude = [exclude]
elif exclude is None:
exclude = []
##
# Set common prefix to avoid rsync with root ('/') when possible
prefix = os.path.commonprefix(files + exclude)
if not os.path.exists(prefix):
prefix = os.path.dirname(prefix)
if os.path.isdir(prefix) and not prefix.endswith(os.path.sep):
prefix += os.path.sep
if (prefix.startswith(target.split(':')[-1])
and prefix.startswith(target.split(':')[-1])):
source = ':'.join(source.split(':')[:-1] + [prefix])
target = ':'.join(target.split(':')[:-1] + [prefix])
else:
prefix = ''
##
# Build files-from
for line in (files or []):
if line[len(prefix):]:
files_file.write(line[len(prefix):] + '\n')
files_file.flush()
files_file.seek(0)
if files and files_file.read():
cmd.append('--files-from={0:s}'.format(files_file.name))
##
# Build exclude-from
for line in (exclude or []):
if line[len(prefix):]:
exclude_file.write(line[len(prefix):] + '\n')
exclude_file.flush()
exclude_file.seek(0)
if exclude and exclude_file.read():
cmd.append('--exclude-from={0:s}'.format(exclude_file.name))
##
# -p preserve permissions
# -t preserve modification times
# -h output numbers in a human-readable format
# -l copy symlinks as symlinks
# -r recurse into directories
# -z compress file data during the transfer
cmd.append('-pthlrz')
if arguments:
cmd.append(arguments)
##
# Build rsh
if any([api.env.port, key_filenames()]):
cmd.append('--rsh="ssh {0:s}"'.format(' '.join(filter(bool, [
# api.env.port
api.env.port and '-p {0:s}'.format(api.env.port) or None,
# key_filenames
' '.join(['-i {0:s}'.format(key)
for key in key_filenames()])
]))))
cmd.append(source)
cmd.append(target)
yield ' '.join(cmd)
def rsync_project(remote_dir, local_dir=None, exclude=(), delete=False,
extra_opts='', ssh_opts='', capture=False):
"""
Synchronize a remote directory with the current project directory via rsync.
Where ``upload_project()`` makes use of ``scp`` to copy one's entire
project every time it is invoked, ``rsync_project()`` uses the ``rsync``
command-line utility, which only transfers files newer than those on the
remote end.
``rsync_project()`` is thus a simple wrapper around ``rsync``; for
details on how ``rsync`` works, please see its manpage. ``rsync`` must be
installed on both your local and remote systems in order for this operation
to work correctly.
This function makes use of Fabric's ``local()`` operation, and returns the
output of that function call; thus it will return the stdout, if any, of
the resultant ``rsync`` call.
``rsync_project()`` takes the following parameters:
* ``remote_dir``: the only required parameter, this is the path to the
directory on the remote server. Due to how ``rsync`` is implemented, the
exact behavior depends on the value of ``local_dir``:
* If ``local_dir`` ends with a trailing slash, the files will be
dropped inside of ``remote_dir``. E.g.
``rsync_project("/home/username/project", "foldername/")`` will drop
the contents of ``foldername`` inside of ``/home/username/project``.
* If ``local_dir`` does **not** end with a trailing slash (and this
includes the default scenario, when ``local_dir`` is not specified),
``remote_dir`` is effectively the "parent" directory, and a new
directory named after ``local_dir`` will be created inside of it. So
``rsync_project("/home/username", "foldername")`` would create a new
directory ``/home/username/foldername`` (if needed) and place the
files there.
* ``local_dir``: by default, ``rsync_project`` uses your current working
directory as the source directory. This may be overridden by specifying
``local_dir``, which is a string passed verbatim to ``rsync``, and thus
may be a single directory (``"my_directory"``) or multiple directories
(``"dir1 dir2"``). See the ``rsync`` documentation for details.
* ``exclude``: optional, may be a single string, or an iterable of strings,
and is used to pass one or more ``--exclude`` options to ``rsync``.
* ``delete``: a boolean controlling whether ``rsync``'s ``--delete`` option
is used. If True, instructs ``rsync`` to remove remote files that no
longer exist locally. Defaults to False.
* ``extra_opts``: an optional, arbitrary string which you may use to pass
custom arguments or options to ``rsync``.
* ``ssh_opts``: Like ``extra_opts`` but specifically for the SSH options
string (rsync's ``--rsh`` flag.)
* ``capture``: Sent directly into an inner `~fabric.operations.local` call.
Furthermore, this function transparently honors Fabric's port and SSH key
settings. Calling this function when the current host string contains a
nonstandard port, or when ``env.key_filename`` is non-empty, will use the
specified port and/or SSH key filename(s).
For reference, the approximate ``rsync`` command-line call that is
constructed by this function is the following::
rsync [--delete] [--exclude exclude[0][, --exclude[1][, ...]]] \\
-pthrvz [extra_opts] <local_dir> <host_string>:<remote_dir>
.. versionadded:: 1.4.0
The ``ssh_opts`` keyword argument.
.. versionadded:: 1.4.1
The ``capture`` keyword argument.
"""
# Turn single-string exclude into a one-item list for consistency
if not hasattr(exclude, '__iter__'):
exclude = (exclude,)
# Create --exclude options from exclude list
exclude_opts = ' --exclude "%s"' * len(exclude)
# Double-backslash-escape
exclusions = tuple([str(s).replace('"', '\\\\"') for s in exclude])
# Honor SSH key(s)
key_string = ""
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# RSH
rsh_string = ""
rsh_parts = [key_string, port_string, ssh_opts]
if any(rsh_parts):
rsh_string = "--rsh='ssh %s'" % " ".join(rsh_parts)
# Set up options part of string
options_map = {
'delete': '--delete' if delete else '',
'exclude': exclude_opts % exclusions,
'rsh': rsh_string,
'extra': extra_opts
}
options = "%(delete)s%(exclude)s -pthrvz %(extra)s %(rsh)s" % options_map
# Get local directory
if local_dir is None:
local_dir = '../' + getcwd().split(sep)[-1]
# Create and run final command string
cmd = "rsync %s %s %s@%s:%s" % (options, local_dir, user, host, remote_dir)
if output.running:
print("[%s] rsync_project: %s" % (env.host_string, cmd))
return local(cmd, capture=capture)
def rsync(source, target, exclude=(), delete=False, strict_host_keys=True,
rsync_opts='', ssh_opts=''):
"""
Convenient wrapper around your friendly local ``rsync``.
Specifically, it calls your local ``rsync`` program via a subprocess, and
fills in its arguments with Fabric's current target host/user/port. It
provides Python level keyword arguments for some common rsync options, and
allows you to specify custom options via a string if required (see below.)
For details on how ``rsync`` works, please see its manpage. ``rsync`` must
be installed on both your local and remote systems in order for this
function to work correctly.
This function makes use of Fabric's ``local()`` function and returns its
output; thus it will exhibit ``failed``/``succeeded``/``stdout``/``stderr``
attributes, behaves like a string consisting of ``stdout``, and so forth.
``rsync()`` takes the following parameters:
* ``source``: The local location to copy from. Actually a string passed
verbatim to ``rsync``, and thus may be a single directory
(``"my_directory"``) or multiple directories (``"dir1 dir2"``). See the
``rsync`` documentation for details.
* ``target``: The path to sync with on the remote server. Due to how
``rsync`` is implemented, the exact behavior depends on the value of
``source``:
* If ``source`` ends with a trailing slash, the files will be
dropped inside of ``target``. E.g.
``rsync("foldername/", "/home/username/project")`` will drop
the contents of ``foldername`` inside of ``/home/username/project``.
* If ``source`` does **not** end with a trailing slash,
``target`` is effectively the "parent" directory, and a new
directory named after ``source`` will be created inside of it. So
``rsync("foldername", "/home/username")`` would create a new
directory ``/home/username/foldername`` (if needed) and place the
files there.
* ``exclude``: optional, may be a single string, or an iterable of strings,
and is used to pass one or more ``--exclude`` options to ``rsync``.
* ``delete``: a boolean controlling whether ``rsync``'s ``--delete`` option
is used. If True, instructs ``rsync`` to remove remote files that no
longer exist locally. Defaults to False.
* ``strict_host_keys``: Boolean determining whether to enable/disable the
SSH-level option ``StrictHostKeyChecking`` (useful for
frequently-changing hosts such as virtual machines or cloud instances.)
Defaults to True.
* ``rsync_opts``: an optional, arbitrary string which you may use to pass
custom arguments or options to ``rsync``.
* ``ssh_opts``: Like ``rsync_opts`` but specifically for the SSH options
string (rsync's ``--rsh`` flag.)
This function transparently honors Fabric's port and SSH key
settings. Calling this function when the current host string contains a
nonstandard port, or when ``env.key_filename`` is non-empty, will use the
specified port and/or SSH key filename(s).
For reference, the approximate ``rsync`` command-line call that is
constructed by this function is the following::
rsync [--delete] [--exclude exclude[0][, --exclude[1][, ...]]] \\
-pthrvz [rsync_opts] <source> <host_string>:<target>
"""
# Turn single-string exclude into a one-item list for consistency
if not hasattr(exclude, '__iter__'):
exclude = (exclude,)
# Create --exclude options from exclude list
exclude_opts = ' --exclude "%s"' * len(exclude)
# Double-backslash-escape
exclusions = tuple([str(s).replace('"', '\\\\"') for s in exclude])
# Honor SSH key(s)
key_string = ""
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# Remote shell (SSH) options
rsh_string = ""
# Strict host key checking
disable_keys = '-o StrictHostKeyChecking=no'
if not strict_host_keys and disable_keys not in ssh_opts:
ssh_opts += ' %s' % disable_keys
rsh_parts = [key_string, port_string, ssh_opts]
if any(rsh_parts):
rsh_string = "--rsh='ssh %s'" % " ".join(rsh_parts)
# Set up options part of string
options_map = {
'delete': '--delete' if delete else '',
'exclude': exclude_opts % exclusions,
'rsh': rsh_string,
'extra': rsync_opts
}
options = "%(delete)s%(exclude)s -pthrvz %(extra)s %(rsh)s" % options_map
# Create and run final command string
if env.host.count(':') > 1:
# Square brackets are mandatory for IPv6 rsync address,
# even if port number is not specified
cmd = "rsync %s %s [%s@%s]:%s" % (options, source, user, host, target)
else:
cmd = "rsync %s %s %s@%s:%s" % (options, source, user, host, target)
return local(cmd)
def rsync_project(
remote_dir,
local_dir=None,
exclude=(),
delete=False,
extra_opts='',
ssh_opts='',
capture=False,
upload=True,
default_opts='-pthrvz',
sshpass=False
):
"""
Synchronize a remote directory with the current project directory via rsync.
Where ``upload_project()`` makes use of ``scp`` to copy one's entire
project every time it is invoked, ``rsync_project()`` uses the ``rsync``
command-line utility, which only transfers files newer than those on the
remote end.
``rsync_project()`` is thus a simple wrapper around ``rsync``; for
details on how ``rsync`` works, please see its manpage. ``rsync`` must be
installed on both your local and remote systems in order for this operation
to work correctly.
This function makes use of Fabric's ``local()`` operation, and returns the
output of that function call; thus it will return the stdout, if any, of
the resultant ``rsync`` call.
``rsync_project()`` takes the following parameters:
* ``remote_dir``: the only required parameter, this is the path to the
directory on the remote server. Due to how ``rsync`` is implemented, the
exact behavior depends on the value of ``local_dir``:
* If ``local_dir`` ends with a trailing slash, the files will be
dropped inside of ``remote_dir``. E.g.
``rsync_project("/home/username/project", "foldername/")`` will drop
the contents of ``foldername`` inside of ``/home/username/project``.
* If ``local_dir`` does **not** end with a trailing slash (and this
includes the default scenario, when ``local_dir`` is not specified),
``remote_dir`` is effectively the "parent" directory, and a new
directory named after ``local_dir`` will be created inside of it. So
``rsync_project("/home/username", "foldername")`` would create a new
directory ``/home/username/foldername`` (if needed) and place the
files there.
* ``local_dir``: by default, ``rsync_project`` uses your current working
directory as the source directory. This may be overridden by specifying
``local_dir``, which is a string passed verbatim to ``rsync``, and thus
may be a single directory (``"my_directory"``) or multiple directories
(``"dir1 dir2"``). See the ``rsync`` documentation for details.
* ``exclude``: optional, may be a single string, or an iterable of strings,
and is used to pass one or more ``--exclude`` options to ``rsync``.
* ``delete``: a boolean controlling whether ``rsync``'s ``--delete`` option
is used. If True, instructs ``rsync`` to remove remote files that no
longer exist locally. Defaults to False.
* ``extra_opts``: an optional, arbitrary string which you may use to pass
custom arguments or options to ``rsync``.
* ``ssh_opts``: Like ``extra_opts`` but specifically for the SSH options
string (rsync's ``--rsh`` flag.)
* ``capture``: Sent directly into an inner `~fabric.operations.local` call.
* ``upload``: a boolean controlling whether file synchronization is
performed up or downstream. Upstream by default.
* ``default_opts``: the default rsync options ``-pthrvz``, override if
desired (e.g. to remove verbosity, etc).
* ``sshpass``: a boolean controlling whether use ``sshpass``, If True, you
can support ``env.password`` to ``rsync``, no need input password. But
you must install ``sshpass``.
* If you use ubuntu, Type the following command:
``sudo apt-get install sshpass``
* Other platform, Go ``http://sourceforge.net/projects/sshpass/``
donwload the lastest verison, then type the following command:
``tar zxvf sshpass-1.05.tar.gz && cd sshpass-1.05 && ./configure &&
make && sudo make install``
Furthermore, this function transparently honors Fabric's port and SSH key
settings. Calling this function when the current host string contains a
nonstandard port, or when ``env.key_filename`` is non-empty, will use the
specified port and/or SSH key filename(s).
For reference, the approximate ``rsync`` command-line call that is
constructed by this function is the following::
rsync [--delete] [--exclude exclude[0][, --exclude[1][, ...]]] \\
[default_opts] [extra_opts] <local_dir> <host_string>:<remote_dir>
.. versionadded:: 1.4.0
The ``ssh_opts`` keyword argument.
.. versionadded:: 1.4.1
The ``capture`` keyword argument.
.. versionadded:: 1.8.0
The ``default_opts`` keyword argument.
"""
# Turn single-string exclude into a one-item list for consistency
if not hasattr(exclude, '__iter__'):
exclude = (exclude,)
# Create --exclude options from exclude list
exclude_opts = ' --exclude "%s"' * len(exclude)
# Double-backslash-escape
exclusions = tuple([str(s).replace('"', '\\\\"') for s in exclude])
# Honor SSH key(s)
key_string = ""
keys = key_filenames()
if keys:
key_string = "-i " + " -i ".join(keys)
# Port
user, host, port = normalize(env.host_string)
port_string = "-p %s" % port
# RSH
rsh_string = ""
rsh_parts = [key_string, port_string, ssh_opts]
gate_pass = None
if any(rsh_parts):
# when have gateway
if env.gateway:
guser, ghost, gport = normalize(env.gateway)
gate_pass = env.passwords[env.gateway]
rsh_parts.append(
"%s sshpass -p %s ssh -p %s" % (ghost, env.password, gport))
rsh_string = "--rsh='ssh %s'" % " ".join(rsh_parts)
# Set up options part of string
options_map = {
'delete': '--delete' if delete else '',
'exclude': exclude_opts % exclusions,
'rsh': rsh_string,
'default': default_opts,
'extra': extra_opts,
}
options = "%(delete)s%(exclude)s %(default)s %(extra)s %(rsh)s" % options_map
# Get local directory
if local_dir is None:
local_dir = '../' + getcwd().split(sep)[-1]
# Create and run final command string
if host.count(':') > 1:
# Square brackets are mandatory for IPv6 rsync address,
# even if port number is not specified
remote_prefix = "[%s@%s]" % (user, host)
else:
remote_prefix = "%s@%s" % (user, host)
if upload:
cmd = "rsync %s %s %s:%s" % (options, local_dir, remote_prefix, remote_dir)
else:
cmd = "rsync %s %s:%s %s" % (options, remote_prefix, remote_dir, local_dir)
if sshpass:
cmd = "sshpass -p %s %s" % (gate_pass if gate_pass else env.password, cmd)
if output.running:
print("[%s] rsync_project: %s" % (env.host_string, cmd))
return local(cmd, capture=capture)
