def render_sinterp(filename, context=None, template_dir=None): """Render a Python 2 string template. Based on fabric.contrib.files.upload_template. """ if template_dir: filename = os.path.join(template_dir, filename) filename = apply_lcwd(filename, env) with open(os.path.expanduser(filename)) as inputfile: text = inputfile.read() if context: text = text % context return text
def render_jinja2(template_filename, context, template_dir=None): """Render a Jinja2 template. Based on fabric.contrib.files.upload_template. Differences: adds back the trailing newline that Jinja2 eats for some reason. """ from jinja2 import Environment, FileSystemLoader template_dir = template_dir or os.getcwd() template_dir = apply_lcwd(template_dir, env) jenv = Environment(loader=FileSystemLoader(template_dir)) text = jenv.get_template(template_filename).render(**context or {}) return text.encode('UTF-8') + '\n'
def render_as_text(filename, context=None, template_dir=None): try: template_dir = template_dir or os.getcwd() template_dir = apply_lcwd(template_dir, env) from jinja2 import Environment, FileSystemLoader jenv = Environment(loader=FileSystemLoader(template_dir)) text = jenv.get_template(filename).render(**context or {}) # Force to a byte representation of Unicode, or str()ification # within Paramiko's SFTP machinery may cause decode issues for # truly non-ASCII characters. text = text.encode('utf-8') return text except ImportError: import traceback tb = traceback.format_exc() abort(tb + "\nUnable to import Jinja2 -- see above.")
def get(remote_path, local_path=None): """ Download one or more files from a remote host. `~fabric.operations.get` returns an iterable containing the absolute paths to all local files downloaded, which will be empty if ``local_path`` was a StringIO object (see below for more on using StringIO). This object will also exhibit a ``.failed`` attribute containing any remote file paths which failed to download, and a ``.succeeded`` attribute equivalent to ``not .failed``. ``remote_path`` is the remote file or directory path to download, which may contain shell glob syntax, e.g. ``"/var/log/apache2/*.log"``, and will have tildes replaced by the remote home directory. Relative paths will be considered relative to the remote user's home directory, or the current remote working directory as manipulated by `~fabric.context_managers.cd`. If the remote path points to a directory, that directory will be downloaded recursively. ``local_path`` is the local file path where the downloaded file or files will be stored. If relative, it will honor the local current working directory as manipulated by `~fabric.context_managers.lcd`. It may be interpolated, using standard Python dict-based interpolation, with the following variables: * ``host``: The value of ``env.host_string``, eg ``myhostname`` or ``user@myhostname-222`` (the colon between hostname and port is turned into a dash to maximize filesystem compatibility) * ``dirname``: The directory part of the remote file path, e.g. the ``src/projectname`` in ``src/projectname/utils.py``. * ``basename``: The filename part of the remote file path, e.g. the ``utils.py`` in ``src/projectname/utils.py`` * ``path``: The full remote path, e.g. ``src/projectname/utils.py``. .. note:: When ``remote_path`` is an absolute directory path, only the inner directories will be recreated locally and passed into the above variables. So for example, ``get('/var/log', '%(path)s')`` would start writing out files like ``apache2/access.log``, ``postgresql/8.4/postgresql.log``, etc, in the local working directory. It would **not** write out e.g. ``var/log/apache2/access.log``. Additionally, when downloading a single file, ``%(dirname)s`` and ``%(path)s`` do not make as much sense and will be empty and equivalent to ``%(basename)s``, respectively. Thus a call like ``get('/var/log/apache2/access.log', '%(path)s')`` will save a local file named ``access.log``, not ``var/log/apache2/access.log``. This behavior is intended to be consistent with the command-line ``scp`` program. If left blank, ``local_path`` defaults to ``"%(host)s/%(path)s"`` in order to be safe for multi-host invocations. .. warning:: If your ``local_path`` argument does not contain ``%(host)s`` and your `~fabric.operations.get` call runs against multiple hosts, your local files will be overwritten on each successive run! If ``local_path`` does not make use of the above variables (i.e. if it is a simple, explicit file path) it will act similar to ``scp`` or ``cp``, overwriting pre-existing files if necessary, downloading into a directory if given (e.g. ``get('/path/to/remote_file.txt', 'local_directory')`` will create ``local_directory/remote_file.txt``) and so forth. ``local_path`` may alternately be a file-like object, such as the result of ``open('path', 'w')`` or a ``StringIO`` instance. .. note:: Attempting to `get` a directory into a file-like object is not valid and will result in an error. .. note:: This function will use ``seek`` and ``tell`` to overwrite the entire contents of the file-like object, in order to be consistent with the behavior of `~fabric.operations.put` (which also considers the entire file). However, unlike `~fabric.operations.put`, the file pointer will not be restored to its previous location, as that doesn't make as much sense here and/or may not even be possible. .. note:: If a file-like object such as StringIO has a ``name`` attribute, that will be used in Fabric's printed output instead of the default ``<file obj>`` .. versionchanged:: 1.0 Now honors the remote working directory as manipulated by `~fabric.context_managers.cd`, and the local working directory as manipulated by `~fabric.context_managers.lcd`. .. versionchanged:: 1.0 Now allows file-like objects in the ``local_path`` argument. .. versionchanged:: 1.0 ``local_path`` may now contain interpolated path- and host-related variables. .. versionchanged:: 1.0 Directories may be specified in the ``remote_path`` argument and will trigger recursive downloads. .. versionchanged:: 1.0 Return value is now an iterable of downloaded local file paths, which also exhibits the ``.failed`` and ``.succeeded`` attributes. .. versionchanged:: 1.5 Allow a ``name`` attribute on file-like objects for log output """ # Handle empty local path / default kwarg value local_path = local_path or "%(host)s/%(path)s" # Test whether local_path is a path or a file-like object local_is_path = not (hasattr(local_path, 'write') \ and callable(local_path.write)) # Honor lcd() where it makes sense if local_is_path: local_path = apply_lcwd(local_path, env) ftp = SFTP(env.host_string) with closing(ftp) as ftp: home = ftp.normalize('.') # Expand home directory markers (tildes, etc) if remote_path.startswith('~'): remote_path = remote_path.replace('~', home, 1) if local_is_path: local_path = os.path.expanduser(local_path) # Honor cd() (assumes Unix style file paths on remote end) if not os.path.isabs(remote_path): # Honor cwd if it's set (usually by with cd():) if env.get('cwd'): remote_path = env.cwd.rstrip('/') + '/' + remote_path # Otherwise, be relative to remote home directory (SFTP server's # '.') else: remote_path = posixpath.join(home, remote_path) # Track final local destination files so we can return a list local_files = [] failed_remote_files = [] try: # Glob remote path names = ftp.glob(remote_path) # Handle invalid local-file-object situations if not local_is_path: if len(names) > 1 or ftp.isdir(names[0]): error("[%s] %s is a glob or directory, but local_path is a file object!" % (env.host_string, remote_path)) for remote_path in names: if ftp.isdir(remote_path): result = ftp.get_dir(remote_path, local_path) local_files.extend(result) else: # Perform actual get. If getting to real local file path, # add result (will be true final path value) to # local_files. File-like objects are omitted. result = ftp.get(remote_path, local_path, local_is_path, os.path.basename(remote_path)) if local_is_path: local_files.append(result) except Exception, e: failed_remote_files.append(remote_path) msg = "get() encountered an exception while downloading '%s'" error(message=msg % remote_path, exception=e) ret = _AttributeList(local_files if local_is_path else []) ret.failed = failed_remote_files ret.succeeded = not ret.failed return ret
def put(local_path=None, remote_path=None, use_sudo=False, mirror_local_mode=False, mode=None, use_glob=True): """ Upload one or more files to a remote host. `~fabric.operations.put` returns an iterable containing the absolute file paths of all remote files uploaded. This iterable also exhibits a ``.failed`` attribute containing any local file paths which failed to upload (and may thus be used as a boolean test.) You may also check ``.succeeded`` which is equivalent to ``not .failed``. ``local_path`` may be a relative or absolute local file or directory path, and may contain shell-style wildcards, as understood by the Python ``glob`` module (give ``use_glob=False`` to disable this behavior). Tilde expansion (as implemented by ``os.path.expanduser``) is also performed. ``local_path`` may alternately be a file-like object, such as the result of ``open('path')`` or a ``StringIO`` instance. .. note:: In this case, `~fabric.operations.put` will attempt to read the entire contents of the file-like object by rewinding it using ``seek`` (and will use ``tell`` afterwards to preserve the previous file position). ``remote_path`` may also be a relative or absolute location, but applied to the remote host. Relative paths are relative to the remote user's home directory, but tilde expansion (e.g. ``~/.ssh/``) will also be performed if necessary. An empty string, in either path argument, will be replaced by the appropriate end's current working directory. While the SFTP protocol (which `put` uses) has no direct ability to upload files to locations not owned by the connecting user, you may specify ``use_sudo=True`` to work around this. When set, this setting causes `put` to upload the local files to a temporary location on the remote end, and then use `sudo` to move them to ``remote_path``. In some use cases, it is desirable to force a newly uploaded file to match the mode of its local counterpart (such as when uploading executable scripts). To do this, specify ``mirror_local_mode=True``. Alternately, you may use the ``mode`` kwarg to specify an exact mode, in the same vein as ``os.chmod`` or the Unix ``chmod`` command. `~fabric.operations.put` will honor `~fabric.context_managers.cd`, so relative values in ``remote_path`` will be prepended by the current remote working directory, if applicable. Thus, for example, the below snippet would attempt to upload to ``/tmp/files/test.txt`` instead of ``~/files/test.txt``:: with cd('/tmp'): put('/path/to/local/test.txt', 'files') Use of `~fabric.context_managers.lcd` will affect ``local_path`` in the same manner. Examples:: put('bin/project.zip', '/tmp/project.zip') put('*.py', 'cgi-bin/') put('index.html', 'index.html', mode=0755) .. note:: If a file-like object such as StringIO has a ``name`` attribute, that will be used in Fabric's printed output instead of the default ``<file obj>`` .. versionchanged:: 1.0 Now honors the remote working directory as manipulated by `~fabric.context_managers.cd`, and the local working directory as manipulated by `~fabric.context_managers.lcd`. .. versionchanged:: 1.0 Now allows file-like objects in the ``local_path`` argument. .. versionchanged:: 1.0 Directories may be specified in the ``local_path`` argument and will trigger recursive uploads. .. versionchanged:: 1.0 Return value is now an iterable of uploaded remote file paths which also exhibits the ``.failed`` and ``.succeeded`` attributes. .. versionchanged:: 1.5 Allow a ``name`` attribute on file-like objects for log output .. versionchanged:: 1.7 Added ``use_glob`` option to allow disabling of globbing. """ # Handle empty local path local_path = local_path or os.getcwd() # Test whether local_path is a path or a file-like object local_is_path = not (hasattr(local_path, 'read') \ and callable(local_path.read)) ftp = SFTP(env.host_string) with closing(ftp) as ftp: home = ftp.normalize('.') # Empty remote path implies cwd remote_path = remote_path or home # Expand tildes if remote_path.startswith('~'): remote_path = remote_path.replace('~', home, 1) # Honor cd() (assumes Unix style file paths on remote end) if not os.path.isabs(remote_path) and env.get('cwd'): remote_path = env.cwd.rstrip('/') + '/' + remote_path if local_is_path: # Apply lcwd, expand tildes, etc local_path = os.path.expanduser(local_path) local_path = apply_lcwd(local_path, env) if use_glob: # Glob local path names = glob(local_path) else: # Check if file exists first so ValueError gets raised if os.path.exists(local_path): names = [local_path] else: names = [] else: names = [local_path] # Make sure local arg exists if local_is_path and not names: err = "'%s' is not a valid local path or glob." % local_path raise ValueError(err) # Sanity check and wierd cases if ftp.exists(remote_path): if local_is_path and len(names) != 1 and not ftp.isdir(remote_path): raise ValueError("'%s' is not a directory" % remote_path) # Iterate over all given local files remote_paths = [] failed_local_paths = [] for lpath in names: try: if local_is_path and os.path.isdir(lpath): p = ftp.put_dir(lpath, remote_path, use_sudo, mirror_local_mode, mode) remote_paths.extend(p) else: p = ftp.put(lpath, remote_path, use_sudo, mirror_local_mode, mode, local_is_path) remote_paths.append(p) except Exception, e: msg = "put() encountered an exception while uploading '%s'" failure = lpath if local_is_path else "<StringIO>" failed_local_paths.append(failure) error(message=msg % lpath, exception=e) ret = _AttributeList(remote_paths) ret.failed = failed_local_paths ret.succeeded = not ret.failed return ret
def upload_template(filename, destination, context=None, use_jinja=False, template_dir=None, use_sudo=False, backup=True, mirror_local_mode=False, mode=None, pty=None): """ Render and upload a template text file to a remote host. Returns the result of the inner call to `~fabric.operations.put` -- see its documentation for details. ``filename`` should be the path to a text file, which may contain `Python string interpolation formatting <http://docs.python.org/library/stdtypes.html#string-formatting>`_ and will be rendered with the given context dictionary ``context`` (if given.) Alternately, if ``use_jinja`` is set to True and you have the Jinja2 templating library available, Jinja will be used to render the template instead. Templates will be loaded from the invoking user's current working directory by default, or from ``template_dir`` if given. The resulting rendered file will be uploaded to the remote file path ``destination``. If the destination file already exists, it will be renamed with a ``.bak`` extension unless ``backup=False`` is specified. By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. The ``mirror_local_mode`` and ``mode`` kwargs are passed directly to an internal `~fabric.operations.put` call; please see its documentation for details on these two options. The ``pty`` kwarg will be passed verbatim to any internal `~fabric.operations.run`/`~fabric.operations.sudo` calls, such as those used for testing directory-ness, making backups, etc. .. versionchanged:: 1.1 Added the ``backup``, ``mirror_local_mode`` and ``mode`` kwargs. .. versionchanged:: 1.9 Added the ``pty`` kwarg. """ func = use_sudo and sudo or run if pty is not None: func = partial(func, pty=pty) # Normalize destination to be an actual filename, due to using StringIO with settings(hide('everything'), warn_only=True): if func('test -d %s' % _expand_path(destination)).succeeded: sep = "" if destination.endswith('/') else "/" destination += sep + os.path.basename(filename) # Use mode kwarg to implement mirror_local_mode, again due to using # StringIO if mirror_local_mode and mode is None: mode = os.stat(filename).st_mode # To prevent put() from trying to do this # logic itself mirror_local_mode = False # Process template text = None if use_jinja: try: template_dir = template_dir or os.getcwd() template_dir = apply_lcwd(template_dir, env) from jinja2 import Environment, FileSystemLoader jenv = Environment(loader=FileSystemLoader(template_dir)) text = jenv.get_template(filename).render(**context or {}) # Force to a byte representation of Unicode, or str()ification # within Paramiko's SFTP machinery may cause decode issues for # truly non-ASCII characters. text = text.encode('utf-8') except ImportError: import traceback tb = traceback.format_exc() abort(tb + "\nUnable to import Jinja2 -- see above.") else: filename = apply_lcwd(filename, env) with open(os.path.expanduser(filename)) as inputfile: text = inputfile.read() if context: text = text % context # Back up original file if backup and exists(destination): func("cp %s{,.bak}" % _expand_path(destination)) # Upload the file. return put( local_path=StringIO(text), remote_path=destination, use_sudo=use_sudo, mirror_local_mode=mirror_local_mode, mode=mode )
def render_template(filename, context=None): filename = apply_lcwd(filename, env) with open(expanduser(filename)) as inputfile: text = Template(inputfile.read()) return text.substitute(context or {})
def upload_template(filename, destination, context=None, use_jinja=False, template_dir=None, use_sudo=False, backup=True, mirror_local_mode=False, mode=None, pty=None, keep_trailing_newline=False, temp_dir=''): """ Render and upload a template text file to a remote host. Returns the result of the inner call to `~fabric.operations.put` -- see its documentation for details. ``filename`` should be the path to a text file, which may contain `Python string interpolation formatting <http://docs.python.org/library/stdtypes.html#string-formatting>`_ and will be rendered with the given context dictionary ``context`` (if given.) Alternately, if ``use_jinja`` is set to True and you have the Jinja2 templating library available, Jinja will be used to render the template instead. Templates will be loaded from the invoking user's current working directory by default, or from ``template_dir`` if given. The resulting rendered file will be uploaded to the remote file path ``destination``. If the destination file already exists, it will be renamed with a ``.bak`` extension unless ``backup=False`` is specified. By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. The ``mirror_local_mode``, ``mode``, and ``temp_dir`` kwargs are passed directly to an internal `~fabric.operations.put` call; please see its documentation for details on these two options. The ``pty`` kwarg will be passed verbatim to any internal `~fabric.operations.run`/`~fabric.operations.sudo` calls, such as those used for testing directory-ness, making backups, etc. The ``keep_trailing_newline`` kwarg will be passed when creating Jinja2 Environment which is False by default, same as Jinja2's behaviour. .. versionchanged:: 1.1 Added the ``backup``, ``mirror_local_mode`` and ``mode`` kwargs. .. versionchanged:: 1.9 Added the ``pty`` kwarg. .. versionchanged:: 1.11 Added the ``keep_trailing_newline`` kwarg. .. versionchanged:: 1.11 Added the ``temp_dir`` kwarg. """ func = use_sudo and sudo or run if pty is not None: func = partial(func, pty=pty) # Normalize destination to be an actual filename, due to using StringIO with settings(hide('everything'), warn_only=True): if func('test -d %s' % _expand_path(destination)).succeeded: sep = "" if destination.endswith('/') else "/" destination += sep + os.path.basename(filename) # Use mode kwarg to implement mirror_local_mode, again due to using # StringIO if mirror_local_mode and mode is None: mode = os.stat(apply_lcwd(filename, env)).st_mode # To prevent put() from trying to do this # logic itself mirror_local_mode = False # Process template text = None if use_jinja: try: template_dir = template_dir or os.getcwd() template_dir = apply_lcwd(template_dir, env) from jinja2 import Environment, FileSystemLoader jenv = Environment(loader=FileSystemLoader(template_dir), keep_trailing_newline=keep_trailing_newline) text = jenv.get_template(filename).render(**context or {}) # Force to a byte representation of Unicode, or str()ification # within Paramiko's SFTP machinery may cause decode issues for # truly non-ASCII characters. text = text.encode('utf-8') except ImportError: import traceback tb = traceback.format_exc() abort(tb + "\nUnable to import Jinja2 -- see above.") else: if template_dir: filename = os.path.join(template_dir, filename) filename = apply_lcwd(filename, env) with open(os.path.expanduser(filename)) as inputfile: text = inputfile.read() if context: text = text % context # Back up original file if backup and exists(destination): func("cp %s{,.bak}" % _expand_path(destination)) # Upload the file. return put(local_path=StringIO(text), remote_path=destination, use_sudo=use_sudo, mirror_local_mode=mirror_local_mode, mode=mode, temp_dir=temp_dir)