def get_src_dir_list():
with cd_project() as path:
sftp = SFTP(env.host_string)
# Honor cd()
path = os.path.join(env.cwd, path)
glob_pattern = os.path.join(path, '{}.*'.format(SRC_DIR))
return sftp.glob(glob_pattern)
def test_get_should_not_use_windows_slashes_in_remote_paths(self):
"""
sftp.glob() should always use Unix-style slashes.
"""
with hide('everything'):
path = "/tree/file1.txt"
sftp = SFTP(env.host_string)
eq_(sftp.glob(path), [path])
def test_get_should_not_use_windows_slashes_in_remote_paths(self):
"""
sftp.glob() should always use Unix-style slashes.
"""
with hide('everything'):
path = "/tree/file1.txt"
sftp = SFTP(env.host_string)
eq_(sftp.glob(path), [path])
def get(remote_path, local_path=None):
"""
Download one or more files from a remote host.
`~fabric.operations.get` returns an iterable containing the absolute paths
to all local files downloaded, which will be empty if ``local_path`` was a
StringIO object (see below for more on using StringIO). This object will
also exhibit a ``.failed`` attribute containing any remote file paths which
failed to download, and a ``.succeeded`` attribute equivalent to ``not
.failed``.
``remote_path`` is the remote file or directory path to download, which may
contain shell glob syntax, e.g. ``"/var/log/apache2/*.log"``, and will have
tildes replaced by the remote home directory. Relative paths will be
considered relative to the remote user's home directory, or the current
remote working directory as manipulated by `~fabric.context_managers.cd`.
If the remote path points to a directory, that directory will be downloaded
recursively.
``local_path`` is the local file path where the downloaded file or files
will be stored. If relative, it will honor the local current working
directory as manipulated by `~fabric.context_managers.lcd`. It may be
interpolated, using standard Python dict-based interpolation, with the
following variables:
* ``host``: The value of ``env.host_string``, eg ``myhostname`` or
``user@myhostname-222`` (the colon between hostname and port is turned
into a dash to maximize filesystem compatibility)
* ``dirname``: The directory part of the remote file path, e.g. the
``src/projectname`` in ``src/projectname/utils.py``.
* ``basename``: The filename part of the remote file path, e.g. the
``utils.py`` in ``src/projectname/utils.py``
* ``path``: The full remote path, e.g. ``src/projectname/utils.py``.
.. note::
When ``remote_path`` is an absolute directory path, only the inner
directories will be recreated locally and passed into the above
variables. So for example, ``get('/var/log', '%(path)s')`` would start
writing out files like ``apache2/access.log``,
``postgresql/8.4/postgresql.log``, etc, in the local working directory.
It would **not** write out e.g. ``var/log/apache2/access.log``.
Additionally, when downloading a single file, ``%(dirname)s`` and
``%(path)s`` do not make as much sense and will be empty and equivalent
to ``%(basename)s``, respectively. Thus a call like
``get('/var/log/apache2/access.log', '%(path)s')`` will save a local
file named ``access.log``, not ``var/log/apache2/access.log``.
This behavior is intended to be consistent with the command-line
``scp`` program.
If left blank, ``local_path`` defaults to ``"%(host)s/%(path)s"`` in order
to be safe for multi-host invocations.
.. warning::
If your ``local_path`` argument does not contain ``%(host)s`` and your
`~fabric.operations.get` call runs against multiple hosts, your local
files will be overwritten on each successive run!
If ``local_path`` does not make use of the above variables (i.e. if it is a
simple, explicit file path) it will act similar to ``scp`` or ``cp``,
overwriting pre-existing files if necessary, downloading into a directory
if given (e.g. ``get('/path/to/remote_file.txt', 'local_directory')`` will
create ``local_directory/remote_file.txt``) and so forth.
``local_path`` may alternately be a file-like object, such as the result of
``open('path', 'w')`` or a ``StringIO`` instance.
.. note::
Attempting to `get` a directory into a file-like object is not valid
and will result in an error.
.. note::
This function will use ``seek`` and ``tell`` to overwrite the entire
contents of the file-like object, in order to be consistent with the
behavior of `~fabric.operations.put` (which also considers the entire
file). However, unlike `~fabric.operations.put`, the file pointer will
not be restored to its previous location, as that doesn't make as much
sense here and/or may not even be possible.
.. note::
If a file-like object such as StringIO has a ``name`` attribute, that
will be used in Fabric's printed output instead of the default
``<file obj>``
.. versionchanged:: 1.0
Now honors the remote working directory as manipulated by
`~fabric.context_managers.cd`, and the local working directory as
manipulated by `~fabric.context_managers.lcd`.
.. versionchanged:: 1.0
Now allows file-like objects in the ``local_path`` argument.
.. versionchanged:: 1.0
``local_path`` may now contain interpolated path- and host-related
variables.
.. versionchanged:: 1.0
Directories may be specified in the ``remote_path`` argument and will
trigger recursive downloads.
.. versionchanged:: 1.0
Return value is now an iterable of downloaded local file paths, which
also exhibits the ``.failed`` and ``.succeeded`` attributes.
.. versionchanged:: 1.5
Allow a ``name`` attribute on file-like objects for log output
"""
# Handle empty local path / default kwarg value
local_path = local_path or "%(host)s/%(path)s"
# Test whether local_path is a path or a file-like object
local_is_path = not (hasattr(local_path, 'write') \
and callable(local_path.write))
# Honor lcd() where it makes sense
if local_is_path:
local_path = apply_lcwd(local_path, env)
ftp = SFTP(env.host_string)
with closing(ftp) as ftp:
home = ftp.normalize('.')
# Expand home directory markers (tildes, etc)
if remote_path.startswith('~'):
remote_path = remote_path.replace('~', home, 1)
if local_is_path:
local_path = os.path.expanduser(local_path)
# Honor cd() (assumes Unix style file paths on remote end)
if not os.path.isabs(remote_path):
# Honor cwd if it's set (usually by with cd():)
if env.get('cwd'):
remote_path = env.cwd.rstrip('/') + '/' + remote_path
# Otherwise, be relative to remote home directory (SFTP server's
# '.')
else:
remote_path = posixpath.join(home, remote_path)
# Track final local destination files so we can return a list
local_files = []
failed_remote_files = []
try:
# Glob remote path
names = ftp.glob(remote_path)
# Handle invalid local-file-object situations
if not local_is_path:
if len(names) > 1 or ftp.isdir(names[0]):
error("[%s] %s is a glob or directory, but local_path is a file object!" % (env.host_string, remote_path))
for remote_path in names:
if ftp.isdir(remote_path):
result = ftp.get_dir(remote_path, local_path)
local_files.extend(result)
else:
# Perform actual get. If getting to real local file path,
# add result (will be true final path value) to
# local_files. File-like objects are omitted.
result = ftp.get(remote_path, local_path, local_is_path,
os.path.basename(remote_path))
if local_is_path:
local_files.append(result)
except Exception, e:
failed_remote_files.append(remote_path)
msg = "get() encountered an exception while downloading '%s'"
error(message=msg % remote_path, exception=e)
ret = _AttributeList(local_files if local_is_path else [])
ret.failed = failed_remote_files
ret.succeeded = not ret.failed
return ret
def get(remote_path, local_path=None):
"""
Download one or more files from a remote host.
`~fabric.operations.get` returns an iterable containing the absolute paths
to all local files downloaded, which will be empty if ``local_path`` was a
StringIO object (see below for more on using StringIO). This object will
also exhibit a ``.failed`` attribute containing any remote file paths which
failed to download, and a ``.succeeded`` attribute equivalent to ``not
.failed``.
``remote_path`` is the remote file or directory path to download, which may
contain shell glob syntax, e.g. ``"/var/log/apache2/*.log"``, and will have
tildes replaced by the remote home directory. Relative paths will be
considered relative to the remote user's home directory, or the current
remote working directory as manipulated by `~fabric.context_managers.cd`.
If the remote path points to a directory, that directory will be downloaded
recursively.
``local_path`` is the local file path where the downloaded file or files
will be stored. If relative, it will honor the local current working
directory as manipulated by `~fabric.context_managers.lcd`. It may be
interpolated, using standard Python dict-based interpolation, with the
following variables:
* ``host``: The value of ``env.host_string``, eg ``myhostname`` or
``user@myhostname-222`` (the colon between hostname and port is turned
into a dash to maximize filesystem compatibility)
* ``dirname``: The directory part of the remote file path, e.g. the
``src/projectname`` in ``src/projectname/utils.py``.
* ``basename``: The filename part of the remote file path, e.g. the
``utils.py`` in ``src/projectname/utils.py``
* ``path``: The full remote path, e.g. ``src/projectname/utils.py``.
.. note::
When ``remote_path`` is an absolute directory path, only the inner
directories will be recreated locally and passed into the above
variables. So for example, ``get('/var/log', '%(path)s')`` would start
writing out files like ``apache2/access.log``,
``postgresql/8.4/postgresql.log``, etc, in the local working directory.
It would **not** write out e.g. ``var/log/apache2/access.log``.
Additionally, when downloading a single file, ``%(dirname)s`` and
``%(path)s`` do not make as much sense and will be empty and equivalent
to ``%(basename)s``, respectively. Thus a call like
``get('/var/log/apache2/access.log', '%(path)s')`` will save a local
file named ``access.log``, not ``var/log/apache2/access.log``.
This behavior is intended to be consistent with the command-line
``scp`` program.
If left blank, ``local_path`` defaults to ``"%(host)s/%(path)s"`` in order
to be safe for multi-host invocations.
.. warning::
If your ``local_path`` argument does not contain ``%(host)s`` and your
`~fabric.operations.get` call runs against multiple hosts, your local
files will be overwritten on each successive run!
If ``local_path`` does not make use of the above variables (i.e. if it is a
simple, explicit file path) it will act similar to ``scp`` or ``cp``,
overwriting pre-existing files if necessary, downloading into a directory
if given (e.g. ``get('/path/to/remote_file.txt', 'local_directory')`` will
create ``local_directory/remote_file.txt``) and so forth.
``local_path`` may alternately be a file-like object, such as the result of
``open('path', 'w')`` or a ``StringIO`` instance.
.. note::
Attempting to `get` a directory into a file-like object is not valid
and will result in an error.
.. note::
This function will use ``seek`` and ``tell`` to overwrite the entire
contents of the file-like object, in order to be consistent with the
behavior of `~fabric.operations.put` (which also considers the entire
file). However, unlike `~fabric.operations.put`, the file pointer will
not be restored to its previous location, as that doesn't make as much
sense here and/or may not even be possible.
.. note::
If a file-like object such as StringIO has a ``name`` attribute, that
will be used in Fabric's printed output instead of the default
``<file obj>``
.. versionchanged:: 1.0
Now honors the remote working directory as manipulated by
`~fabric.context_managers.cd`, and the local working directory as
manipulated by `~fabric.context_managers.lcd`.
.. versionchanged:: 1.0
Now allows file-like objects in the ``local_path`` argument.
.. versionchanged:: 1.0
``local_path`` may now contain interpolated path- and host-related
variables.
.. versionchanged:: 1.0
Directories may be specified in the ``remote_path`` argument and will
trigger recursive downloads.
.. versionchanged:: 1.0
Return value is now an iterable of downloaded local file paths, which
also exhibits the ``.failed`` and ``.succeeded`` attributes.
.. versionchanged:: 1.5
Allow a ``name`` attribute on file-like objects for log output
"""
# Handle empty local path / default kwarg value
local_path = local_path or "%(host)s/%(path)s"
# Test whether local_path is a path or a file-like object
local_is_path = not (hasattr(local_path, 'write') \
and callable(local_path.write))
# Honor lcd() where it makes sense
if local_is_path:
local_path = apply_lcwd(local_path, env)
ftp = SFTP(env.host_string)
with closing(ftp) as ftp:
home = ftp.normalize('.')
# Expand home directory markers (tildes, etc)
if remote_path.startswith('~'):
remote_path = remote_path.replace('~', home, 1)
if local_is_path:
local_path = os.path.expanduser(local_path)
# Honor cd() (assumes Unix style file paths on remote end)
if not os.path.isabs(remote_path):
# Honor cwd if it's set (usually by with cd():)
if env.get('cwd'):
remote_path = env.cwd.rstrip('/') + '/' + remote_path
# Otherwise, be relative to remote home directory (SFTP server's
# '.')
else:
remote_path = posixpath.join(home, remote_path)
# Track final local destination files so we can return a list
local_files = []
failed_remote_files = []
try:
# Glob remote path
names = ftp.glob(remote_path)
# Handle invalid local-file-object situations
if not local_is_path:
if len(names) > 1 or ftp.isdir(names[0]):
error("[%s] %s is a glob or directory, but local_path is a file object!" % (env.host_string, remote_path))
for remote_path in names:
if ftp.isdir(remote_path):
result = ftp.get_dir(remote_path, local_path)
local_files.extend(result)
else:
# Perform actual get. If getting to real local file path,
# add result (will be true final path value) to
# local_files. File-like objects are omitted.
result = ftp.get(remote_path, local_path, local_is_path,
os.path.basename(remote_path))
if local_is_path:
local_files.append(result)
except Exception, e:
failed_remote_files.append(remote_path)
msg = "get() encountered an exception while downloading '%s'"
error(message=msg % remote_path, exception=e)
ret = _AttributeList(local_files if local_is_path else [])
ret.failed = failed_remote_files
ret.succeeded = not ret.failed
return ret
