def render_sinterp(filename, context=None, template_dir=None):
"""Render a Python 2 string template.
Based on fabric.contrib.files.upload_template.
"""
if template_dir:
filename = os.path.join(template_dir, filename)
filename = apply_lcwd(filename, env)
with open(os.path.expanduser(filename)) as inputfile:
text = inputfile.read()
if context:
text = text % context
return text
def render_jinja2(template_filename, context, template_dir=None):
"""Render a Jinja2 template.
Based on fabric.contrib.files.upload_template.
Differences: adds back the trailing newline that Jinja2 eats for some
reason.
"""
from jinja2 import Environment, FileSystemLoader
template_dir = template_dir or os.getcwd()
template_dir = apply_lcwd(template_dir, env)
jenv = Environment(loader=FileSystemLoader(template_dir))
text = jenv.get_template(template_filename).render(**context or {})
return text.encode('UTF-8') + '\n'
def render_as_text(filename, context=None, template_dir=None):
try:
template_dir = template_dir or os.getcwd()
template_dir = apply_lcwd(template_dir, env)
from jinja2 import Environment, FileSystemLoader
jenv = Environment(loader=FileSystemLoader(template_dir))
text = jenv.get_template(filename).render(**context or {})
# Force to a byte representation of Unicode, or str()ification
# within Paramiko's SFTP machinery may cause decode issues for
# truly non-ASCII characters.
text = text.encode('utf-8')
return text
except ImportError:
import traceback
tb = traceback.format_exc()
abort(tb + "\nUnable to import Jinja2 -- see above.")
def get(remote_path, local_path=None):
"""
Download one or more files from a remote host.
`~fabric.operations.get` returns an iterable containing the absolute paths
to all local files downloaded, which will be empty if ``local_path`` was a
StringIO object (see below for more on using StringIO). This object will
also exhibit a ``.failed`` attribute containing any remote file paths which
failed to download, and a ``.succeeded`` attribute equivalent to ``not
.failed``.
``remote_path`` is the remote file or directory path to download, which may
contain shell glob syntax, e.g. ``"/var/log/apache2/*.log"``, and will have
tildes replaced by the remote home directory. Relative paths will be
considered relative to the remote user's home directory, or the current
remote working directory as manipulated by `~fabric.context_managers.cd`.
If the remote path points to a directory, that directory will be downloaded
recursively.
``local_path`` is the local file path where the downloaded file or files
will be stored. If relative, it will honor the local current working
directory as manipulated by `~fabric.context_managers.lcd`. It may be
interpolated, using standard Python dict-based interpolation, with the
following variables:
* ``host``: The value of ``env.host_string``, eg ``myhostname`` or
``user@myhostname-222`` (the colon between hostname and port is turned
into a dash to maximize filesystem compatibility)
* ``dirname``: The directory part of the remote file path, e.g. the
``src/projectname`` in ``src/projectname/utils.py``.
* ``basename``: The filename part of the remote file path, e.g. the
``utils.py`` in ``src/projectname/utils.py``
* ``path``: The full remote path, e.g. ``src/projectname/utils.py``.
.. note::
When ``remote_path`` is an absolute directory path, only the inner
directories will be recreated locally and passed into the above
variables. So for example, ``get('/var/log', '%(path)s')`` would start
writing out files like ``apache2/access.log``,
``postgresql/8.4/postgresql.log``, etc, in the local working directory.
It would **not** write out e.g. ``var/log/apache2/access.log``.
Additionally, when downloading a single file, ``%(dirname)s`` and
``%(path)s`` do not make as much sense and will be empty and equivalent
to ``%(basename)s``, respectively. Thus a call like
``get('/var/log/apache2/access.log', '%(path)s')`` will save a local
file named ``access.log``, not ``var/log/apache2/access.log``.
This behavior is intended to be consistent with the command-line
``scp`` program.
If left blank, ``local_path`` defaults to ``"%(host)s/%(path)s"`` in order
to be safe for multi-host invocations.
.. warning::
If your ``local_path`` argument does not contain ``%(host)s`` and your
`~fabric.operations.get` call runs against multiple hosts, your local
files will be overwritten on each successive run!
If ``local_path`` does not make use of the above variables (i.e. if it is a
simple, explicit file path) it will act similar to ``scp`` or ``cp``,
overwriting pre-existing files if necessary, downloading into a directory
if given (e.g. ``get('/path/to/remote_file.txt', 'local_directory')`` will
create ``local_directory/remote_file.txt``) and so forth.
``local_path`` may alternately be a file-like object, such as the result of
``open('path', 'w')`` or a ``StringIO`` instance.
.. note::
Attempting to `get` a directory into a file-like object is not valid
and will result in an error.
.. note::
This function will use ``seek`` and ``tell`` to overwrite the entire
contents of the file-like object, in order to be consistent with the
behavior of `~fabric.operations.put` (which also considers the entire
file). However, unlike `~fabric.operations.put`, the file pointer will
not be restored to its previous location, as that doesn't make as much
sense here and/or may not even be possible.
.. note::
If a file-like object such as StringIO has a ``name`` attribute, that
will be used in Fabric's printed output instead of the default
``<file obj>``
.. versionchanged:: 1.0
Now honors the remote working directory as manipulated by
`~fabric.context_managers.cd`, and the local working directory as
manipulated by `~fabric.context_managers.lcd`.
.. versionchanged:: 1.0
Now allows file-like objects in the ``local_path`` argument.
.. versionchanged:: 1.0
``local_path`` may now contain interpolated path- and host-related
variables.
.. versionchanged:: 1.0
Directories may be specified in the ``remote_path`` argument and will
trigger recursive downloads.
.. versionchanged:: 1.0
Return value is now an iterable of downloaded local file paths, which
also exhibits the ``.failed`` and ``.succeeded`` attributes.
.. versionchanged:: 1.5
Allow a ``name`` attribute on file-like objects for log output
"""
# Handle empty local path / default kwarg value
local_path = local_path or "%(host)s/%(path)s"
# Test whether local_path is a path or a file-like object
local_is_path = not (hasattr(local_path, 'write') \
and callable(local_path.write))
# Honor lcd() where it makes sense
if local_is_path:
local_path = apply_lcwd(local_path, env)
ftp = SFTP(env.host_string)
with closing(ftp) as ftp:
home = ftp.normalize('.')
# Expand home directory markers (tildes, etc)
if remote_path.startswith('~'):
remote_path = remote_path.replace('~', home, 1)
if local_is_path:
local_path = os.path.expanduser(local_path)
# Honor cd() (assumes Unix style file paths on remote end)
if not os.path.isabs(remote_path):
# Honor cwd if it's set (usually by with cd():)
if env.get('cwd'):
remote_path = env.cwd.rstrip('/') + '/' + remote_path
# Otherwise, be relative to remote home directory (SFTP server's
# '.')
else:
remote_path = posixpath.join(home, remote_path)
# Track final local destination files so we can return a list
local_files = []
failed_remote_files = []
try:
# Glob remote path
names = ftp.glob(remote_path)
# Handle invalid local-file-object situations
if not local_is_path:
if len(names) > 1 or ftp.isdir(names[0]):
error("[%s] %s is a glob or directory, but local_path is a file object!" % (env.host_string, remote_path))
for remote_path in names:
if ftp.isdir(remote_path):
result = ftp.get_dir(remote_path, local_path)
local_files.extend(result)
else:
# Perform actual get. If getting to real local file path,
# add result (will be true final path value) to
# local_files. File-like objects are omitted.
result = ftp.get(remote_path, local_path, local_is_path,
os.path.basename(remote_path))
if local_is_path:
local_files.append(result)
except Exception, e:
failed_remote_files.append(remote_path)
msg = "get() encountered an exception while downloading '%s'"
error(message=msg % remote_path, exception=e)
ret = _AttributeList(local_files if local_is_path else [])
ret.failed = failed_remote_files
ret.succeeded = not ret.failed
return ret
def put(local_path=None, remote_path=None, use_sudo=False,
mirror_local_mode=False, mode=None, use_glob=True):
"""
Upload one or more files to a remote host.
`~fabric.operations.put` returns an iterable containing the absolute file
paths of all remote files uploaded. This iterable also exhibits a
``.failed`` attribute containing any local file paths which failed to
upload (and may thus be used as a boolean test.) You may also check
``.succeeded`` which is equivalent to ``not .failed``.
``local_path`` may be a relative or absolute local file or directory path,
and may contain shell-style wildcards, as understood by the Python ``glob``
module (give ``use_glob=False`` to disable this behavior). Tilde expansion
(as implemented by ``os.path.expanduser``) is also performed.
``local_path`` may alternately be a file-like object, such as the result of
``open('path')`` or a ``StringIO`` instance.
.. note::
In this case, `~fabric.operations.put` will attempt to read the entire
contents of the file-like object by rewinding it using ``seek`` (and
will use ``tell`` afterwards to preserve the previous file position).
``remote_path`` may also be a relative or absolute location, but applied to
the remote host. Relative paths are relative to the remote user's home
directory, but tilde expansion (e.g. ``~/.ssh/``) will also be performed if
necessary.
An empty string, in either path argument, will be replaced by the
appropriate end's current working directory.
While the SFTP protocol (which `put` uses) has no direct ability to upload
files to locations not owned by the connecting user, you may specify
``use_sudo=True`` to work around this. When set, this setting causes `put`
to upload the local files to a temporary location on the remote end, and
then use `sudo` to move them to ``remote_path``.
In some use cases, it is desirable to force a newly uploaded file to match
the mode of its local counterpart (such as when uploading executable
scripts). To do this, specify ``mirror_local_mode=True``.
Alternately, you may use the ``mode`` kwarg to specify an exact mode, in
the same vein as ``os.chmod`` or the Unix ``chmod`` command.
`~fabric.operations.put` will honor `~fabric.context_managers.cd`, so
relative values in ``remote_path`` will be prepended by the current remote
working directory, if applicable. Thus, for example, the below snippet
would attempt to upload to ``/tmp/files/test.txt`` instead of
``~/files/test.txt``::
with cd('/tmp'):
put('/path/to/local/test.txt', 'files')
Use of `~fabric.context_managers.lcd` will affect ``local_path`` in the
same manner.
Examples::
put('bin/project.zip', '/tmp/project.zip')
put('*.py', 'cgi-bin/')
put('index.html', 'index.html', mode=0755)
.. note::
If a file-like object such as StringIO has a ``name`` attribute, that
will be used in Fabric's printed output instead of the default
``<file obj>``
.. versionchanged:: 1.0
Now honors the remote working directory as manipulated by
`~fabric.context_managers.cd`, and the local working directory as
manipulated by `~fabric.context_managers.lcd`.
.. versionchanged:: 1.0
Now allows file-like objects in the ``local_path`` argument.
.. versionchanged:: 1.0
Directories may be specified in the ``local_path`` argument and will
trigger recursive uploads.
.. versionchanged:: 1.0
Return value is now an iterable of uploaded remote file paths which
also exhibits the ``.failed`` and ``.succeeded`` attributes.
.. versionchanged:: 1.5
Allow a ``name`` attribute on file-like objects for log output
.. versionchanged:: 1.7
Added ``use_glob`` option to allow disabling of globbing.
"""
# Handle empty local path
local_path = local_path or os.getcwd()
# Test whether local_path is a path or a file-like object
local_is_path = not (hasattr(local_path, 'read') \
and callable(local_path.read))
ftp = SFTP(env.host_string)
with closing(ftp) as ftp:
home = ftp.normalize('.')
# Empty remote path implies cwd
remote_path = remote_path or home
# Expand tildes
if remote_path.startswith('~'):
remote_path = remote_path.replace('~', home, 1)
# Honor cd() (assumes Unix style file paths on remote end)
if not os.path.isabs(remote_path) and env.get('cwd'):
remote_path = env.cwd.rstrip('/') + '/' + remote_path
if local_is_path:
# Apply lcwd, expand tildes, etc
local_path = os.path.expanduser(local_path)
local_path = apply_lcwd(local_path, env)
if use_glob:
# Glob local path
names = glob(local_path)
else:
# Check if file exists first so ValueError gets raised
if os.path.exists(local_path):
names = [local_path]
else:
names = []
else:
names = [local_path]
# Make sure local arg exists
if local_is_path and not names:
err = "'%s' is not a valid local path or glob." % local_path
raise ValueError(err)
# Sanity check and wierd cases
if ftp.exists(remote_path):
if local_is_path and len(names) != 1 and not ftp.isdir(remote_path):
raise ValueError("'%s' is not a directory" % remote_path)
# Iterate over all given local files
remote_paths = []
failed_local_paths = []
for lpath in names:
try:
if local_is_path and os.path.isdir(lpath):
p = ftp.put_dir(lpath, remote_path, use_sudo,
mirror_local_mode, mode)
remote_paths.extend(p)
else:
p = ftp.put(lpath, remote_path, use_sudo, mirror_local_mode,
mode, local_is_path)
remote_paths.append(p)
except Exception, e:
msg = "put() encountered an exception while uploading '%s'"
failure = lpath if local_is_path else "<StringIO>"
failed_local_paths.append(failure)
error(message=msg % lpath, exception=e)
ret = _AttributeList(remote_paths)
ret.failed = failed_local_paths
ret.succeeded = not ret.failed
return ret
def upload_template(filename, destination, context=None, use_jinja=False,
template_dir=None, use_sudo=False, backup=True, mirror_local_mode=False,
mode=None, pty=None):
"""
Render and upload a template text file to a remote host.
Returns the result of the inner call to `~fabric.operations.put` -- see its
documentation for details.
``filename`` should be the path to a text file, which may contain `Python
string interpolation formatting
<http://docs.python.org/library/stdtypes.html#string-formatting>`_ and will
be rendered with the given context dictionary ``context`` (if given.)
Alternately, if ``use_jinja`` is set to True and you have the Jinja2
templating library available, Jinja will be used to render the template
instead. Templates will be loaded from the invoking user's current working
directory by default, or from ``template_dir`` if given.
The resulting rendered file will be uploaded to the remote file path
``destination``. If the destination file already exists, it will be
renamed with a ``.bak`` extension unless ``backup=False`` is specified.
By default, the file will be copied to ``destination`` as the logged-in
user; specify ``use_sudo=True`` to use `sudo` instead.
The ``mirror_local_mode`` and ``mode`` kwargs are passed directly to an
internal `~fabric.operations.put` call; please see its documentation for
details on these two options.
The ``pty`` kwarg will be passed verbatim to any internal
`~fabric.operations.run`/`~fabric.operations.sudo` calls, such as those
used for testing directory-ness, making backups, etc.
.. versionchanged:: 1.1
Added the ``backup``, ``mirror_local_mode`` and ``mode`` kwargs.
.. versionchanged:: 1.9
Added the ``pty`` kwarg.
"""
func = use_sudo and sudo or run
if pty is not None:
func = partial(func, pty=pty)
# Normalize destination to be an actual filename, due to using StringIO
with settings(hide('everything'), warn_only=True):
if func('test -d %s' % _expand_path(destination)).succeeded:
sep = "" if destination.endswith('/') else "/"
destination += sep + os.path.basename(filename)
# Use mode kwarg to implement mirror_local_mode, again due to using
# StringIO
if mirror_local_mode and mode is None:
mode = os.stat(filename).st_mode
# To prevent put() from trying to do this
# logic itself
mirror_local_mode = False
# Process template
text = None
if use_jinja:
try:
template_dir = template_dir or os.getcwd()
template_dir = apply_lcwd(template_dir, env)
from jinja2 import Environment, FileSystemLoader
jenv = Environment(loader=FileSystemLoader(template_dir))
text = jenv.get_template(filename).render(**context or {})
# Force to a byte representation of Unicode, or str()ification
# within Paramiko's SFTP machinery may cause decode issues for
# truly non-ASCII characters.
text = text.encode('utf-8')
except ImportError:
import traceback
tb = traceback.format_exc()
abort(tb + "\nUnable to import Jinja2 -- see above.")
else:
filename = apply_lcwd(filename, env)
with open(os.path.expanduser(filename)) as inputfile:
text = inputfile.read()
if context:
text = text % context
# Back up original file
if backup and exists(destination):
func("cp %s{,.bak}" % _expand_path(destination))
# Upload the file.
return put(
local_path=StringIO(text),
remote_path=destination,
use_sudo=use_sudo,
mirror_local_mode=mirror_local_mode,
mode=mode
)
def render_template(filename, context=None):
filename = apply_lcwd(filename, env)
with open(expanduser(filename)) as inputfile:
text = Template(inputfile.read())
return text.substitute(context or {})
def upload_template(filename,
destination,
context=None,
use_jinja=False,
template_dir=None,
use_sudo=False,
backup=True,
mirror_local_mode=False,
mode=None,
pty=None,
keep_trailing_newline=False,
temp_dir=''):
"""
Render and upload a template text file to a remote host.
Returns the result of the inner call to `~fabric.operations.put` -- see its
documentation for details.
``filename`` should be the path to a text file, which may contain `Python
string interpolation formatting
<http://docs.python.org/library/stdtypes.html#string-formatting>`_ and will
be rendered with the given context dictionary ``context`` (if given.)
Alternately, if ``use_jinja`` is set to True and you have the Jinja2
templating library available, Jinja will be used to render the template
instead. Templates will be loaded from the invoking user's current working
directory by default, or from ``template_dir`` if given.
The resulting rendered file will be uploaded to the remote file path
``destination``. If the destination file already exists, it will be
renamed with a ``.bak`` extension unless ``backup=False`` is specified.
By default, the file will be copied to ``destination`` as the logged-in
user; specify ``use_sudo=True`` to use `sudo` instead.
The ``mirror_local_mode``, ``mode``, and ``temp_dir`` kwargs are passed
directly to an internal `~fabric.operations.put` call; please see its
documentation for details on these two options.
The ``pty`` kwarg will be passed verbatim to any internal
`~fabric.operations.run`/`~fabric.operations.sudo` calls, such as those
used for testing directory-ness, making backups, etc.
The ``keep_trailing_newline`` kwarg will be passed when creating
Jinja2 Environment which is False by default, same as Jinja2's
behaviour.
.. versionchanged:: 1.1
Added the ``backup``, ``mirror_local_mode`` and ``mode`` kwargs.
.. versionchanged:: 1.9
Added the ``pty`` kwarg.
.. versionchanged:: 1.11
Added the ``keep_trailing_newline`` kwarg.
.. versionchanged:: 1.11
Added the ``temp_dir`` kwarg.
"""
func = use_sudo and sudo or run
if pty is not None:
func = partial(func, pty=pty)
# Normalize destination to be an actual filename, due to using StringIO
with settings(hide('everything'), warn_only=True):
if func('test -d %s' % _expand_path(destination)).succeeded:
sep = "" if destination.endswith('/') else "/"
destination += sep + os.path.basename(filename)
# Use mode kwarg to implement mirror_local_mode, again due to using
# StringIO
if mirror_local_mode and mode is None:
mode = os.stat(apply_lcwd(filename, env)).st_mode
# To prevent put() from trying to do this
# logic itself
mirror_local_mode = False
# Process template
text = None
if use_jinja:
try:
template_dir = template_dir or os.getcwd()
template_dir = apply_lcwd(template_dir, env)
from jinja2 import Environment, FileSystemLoader
jenv = Environment(loader=FileSystemLoader(template_dir),
keep_trailing_newline=keep_trailing_newline)
text = jenv.get_template(filename).render(**context or {})
# Force to a byte representation of Unicode, or str()ification
# within Paramiko's SFTP machinery may cause decode issues for
# truly non-ASCII characters.
text = text.encode('utf-8')
except ImportError:
import traceback
tb = traceback.format_exc()
abort(tb + "\nUnable to import Jinja2 -- see above.")
else:
if template_dir:
filename = os.path.join(template_dir, filename)
filename = apply_lcwd(filename, env)
with open(os.path.expanduser(filename)) as inputfile:
text = inputfile.read()
if context:
text = text % context
# Back up original file
if backup and exists(destination):
func("cp %s{,.bak}" % _expand_path(destination))
# Upload the file.
return put(local_path=StringIO(text),
remote_path=destination,
use_sudo=use_sudo,
mirror_local_mode=mirror_local_mode,
mode=mode,
temp_dir=temp_dir)
