Example #1
0
def file(path=None,
         contents=None,
         source=None,
         url=None,
         md5=None,
         use_sudo=False,
         owner=None,
         group='',
         mode=None):
    """
    Require a file

    You can provide either:
    - contents: the required contents of the file
    - source: the filename of a local file to upload
    - url: the address of a file to download (path is optional)
    """
    func = use_sudo and sudo or run

    # 1) Only a path is given
    if path and not (contents or source or url):
        assert path
        if not is_file(path):
            func('touch "%(path)s"' % locals())

    # 2) A URL is specified (path is optional)
    elif url:
        if not path:
            path = os.path.basename(urlparse(url).path)

        if not is_file(path) or md5 and md5sum(path) != md5:
            func('wget --progress=dot %(url)s' % locals())

    # 3) A local filename, or a content string, is specified
    else:
        if source:
            assert not contents
            contents = open(source).read()
        else:
            tmp_file = NamedTemporaryFile(delete=False)
            tmp_file.write(contents)
            tmp_file.close()

        if not is_file(
                path) or md5sum(path) != hashlib.md5(contents).hexdigest():
            with settings(hide('running')):
                if source:
                    put(source, path, use_sudo=use_sudo)
                else:
                    put(tmp_file.name, path, use_sudo=use_sudo)
                    os.remove(tmp_file.name)

    # Ensure correct owner
    if owner:
        func('chown %(owner)s:%(group)s "%(path)s"' % locals())

    # Ensure correct mode
    if mode:
        func('chmod %(mode)s "%(path)s"' % locals())
Example #2
0
def file(path=None, contents=None, source=None, url=None, md5=None, use_sudo=False, owner=None, group='', mode=None):
    """
    Require a file

    You can provide either:
    - contents: the required contents of the file
    - source: the filename of a local file to upload
    - url: the address of a file to download (path is optional)
    """
    func = use_sudo and sudo or run

    # 1) Only a path is given
    if path and not (contents or source or url):
        assert path
        if not is_file(path):
            func('touch "%(path)s"' % locals())

    # 2) A URL is specified (path is optional)
    elif url:
        if not path:
            path = os.path.basename(urlparse(url).path)

        if not is_file(path) or md5 and md5sum(path) != md5:
            func('wget --progress=dot %(url)s' % locals())

    # 3) A local filename, or a content string, is specified
    else:
        if source:
            assert not contents
            contents = open(source).read()
        else:
            tmp_file = NamedTemporaryFile(delete=False)
            tmp_file.write(contents)
            tmp_file.close()

        if not is_file(path) or md5sum(path) != hashlib.md5(contents).hexdigest():
            with settings(hide('running')):
                if source:
                    put(source, path, use_sudo=use_sudo)
                else:
                    put(tmp_file.name, path, use_sudo=use_sudo)
                    os.remove(tmp_file.name)

    # Ensure correct owner
    if owner:
        func('chown %(owner)s:%(group)s "%(path)s"' % locals())

    # Ensure correct mode
    if mode:
        func('chmod %(mode)s "%(path)s"' % locals())
Example #3
0
def _upload_file(local_file, drop, md5_local, tries_left, user, group, use_sudo):
    remote_files = put(local_file, drop, mode="0444", use_sudo=use_sudo)
    if len(remote_files) == 1:
        rf = remote_files[0]
        if rf in remote_files.failed:
            if tries_left == 0:
                print "Upload failed... aborting..."
                return None
            else:
                print "Upload failed... trying again.  " + str(tries_left) + " tries left."
                return _upload_file(local_file, drop, md5_local, tries_left - 1, user, group, use_sudo)
        else:
            md5_remote = md5sum(rf)
            if md5_local == md5_remote:
                print "MD5 match.  Uploaded succeeded."
                sudo("chown {u}:{g} {f}".format(u=user, g=group, f=rf))
                return rf
            else:
                print "MD5 mismatch."
                print "Local: " + md5_local
                print "Remote: " + md5_remote
                if tries_left == 0:
                    print "Upload failed... aborting..."
                    return None
                else:
                    print "Upload failed... trying again.  " + str(tries_left) + " tries left."
                    return _upload_file(local_file, drop, md5_local, tries_left - 1, user, group, use_sudo)
Example #4
0
def git_require_no_update():
    """
    Test working_copy() with update=False
    """
    from fabric.api import run

    from fabtools.files import md5sum
    from fabtools import require

    run("tar -c -f wc_old.tar --exclude .git wc")
    old_md5 = md5sum("wc_old.tar")

    require.git.working_copy(REMOTE_URL, path="wc", update=False)

    # Test that the working tree was not updated
    run("tar -c -f wc_new.tar --exclude .git wc")
    new_md5 = md5sum("wc_new.tar")
    assert old_md5 == new_md5
Example #5
0
def git_require_no_update():
    """
    Test working_copy() with update=False
    """
    from fabric.api import run

    from fabtools.files import md5sum
    from fabtools import require

    run('tar -c -f wc_old.tar --exclude .git wc')
    old_md5 = md5sum('wc_old.tar')

    require.git.working_copy(REMOTE_URL, path='wc', update=False)

    # Test that the working tree was not updated
    run('tar -c -f wc_new.tar --exclude .git wc')
    new_md5 = md5sum('wc_new.tar')
    assert old_md5 == new_md5
Example #6
0
def test_git_require_no_update():
    """
    Test working_copy() with update=False
    """

    from fabtools.require.git import working_copy

    try:
        working_copy(REMOTE_URL, path='wc')

        run('tar -c -f wc_old.tar --exclude .git wc')
        old_md5 = md5sum('wc_old.tar')

        working_copy(REMOTE_URL, path='wc', update=False)

        # Test that the working tree was not updated
        run('tar -c -f wc_new.tar --exclude .git wc')
        new_md5 = md5sum('wc_new.tar')
        assert old_md5 == new_md5

    finally:
        run('rm -rf wc')
Example #7
0
def test_git_require_no_update():
    """
    Test working_copy() with update=False
    """

    from fabtools.require.git import working_copy

    try:
        working_copy(REMOTE_URL, path='wc')

        run('tar -c -f wc_old.tar --exclude .git wc')
        old_md5 = md5sum('wc_old.tar')

        working_copy(REMOTE_URL, path='wc', update=False)

        # Test that the working tree was not updated
        run('tar -c -f wc_new.tar --exclude .git wc')
        new_md5 = md5sum('wc_new.tar')
        assert old_md5 == new_md5

    finally:
        run('rm -rf wc')
Example #8
0
def downloaded_file(url, filename=None, md5=None, use_sudo=False):
    """
    I can haz downloaded file
    """
    func = use_sudo and sudo or run

    if not filename:
        filename = os.path.basename(urlparse(url).path)

    download = True
    if is_file(filename):
        if (md5 is None) or (md5 == md5sum(filename)):
            download = False

    if download:
        func('wget --progress=dot %(url)s' % locals())
Example #9
0
def file(path=None, contents=None, source=None, url=None, md5=None,
         use_sudo=False, owner=None, group='', mode=None, verify_remote=True,
         temp_dir='/tmp'):
    """
    Require a file to exist and have specific contents and properties.

    You can provide either:

    - *contents*: the required contents of the file::

        from fabtools import require

        require.file('/tmp/hello.txt', contents='Hello, world')

    - *source*: the local path of a file to upload::

        from fabtools import require

        require.file('/tmp/hello.txt', source='files/hello.txt')

    - *url*: the URL of a file to download (*path* is then optional)::

        from fabric.api import cd
        from fabtools import require

        with cd('tmp'):
            require.file(url='http://example.com/files/hello.txt')

    If *verify_remote* is ``True`` (the default), then an MD5 comparison
    will be used to check whether the remote file is the same as the
    source. If this is ``False``, the file will be assumed to be the
    same if it is present. This is useful for very large files, where
    generating an MD5 sum may take a while.

    When providing either the *contents* or the *source* parameter, Fabric's
    ``put`` function will be used to upload the file to the remote host.
    When ``use_sudo`` is ``True``, the file will first be uploaded to a temporary
    directory, then moved to its final location. The default temporary
    directory is ``/tmp``, but can be overridden with the *temp_dir* parameter.
    If *temp_dir* is an empty string, then the user's home directory will
    be used.

    If `use_sudo` is `True`, then the remote file will be owned by root,
    and its mode will reflect root's default *umask*. The optional *owner*,
    *group* and *mode* parameters can be used to override these properties.

    .. note:: This function can be accessed directly from the
              ``fabtools.require`` module for convenience.

    """
    func = use_sudo and run_as_root or run

    # 1) Only a path is given
    if path and not (contents or source or url):
        assert path
        if not is_file(path):
            func('touch "%(path)s"' % locals())

    # 2) A URL is specified (path is optional)
    elif url:
        if not path:
            path = os.path.basename(urlparse(url).path)

        if not is_file(path) or md5 and md5sum(path) != md5:
            func('wget --progress=dot:mega "%(url)s" -O "%(path)s"' % locals())

    # 3) A local filename, or a content string, is specified
    else:
        if source:
            assert not contents
            t = None
        else:
            fd, source = mkstemp()
            t = os.fdopen(fd, 'w')
            t.write(contents)
            t.close()

        if verify_remote:
            # Avoid reading the whole file into memory at once
            digest = hashlib.md5()
            f = open(source, 'rb')
            try:
                while True:
                    d = f.read(BLOCKSIZE)
                    if not d:
                        break
                    digest.update(d)
            finally:
                f.close()
        else:
            digest = None

        if (not is_file(path, use_sudo=use_sudo) or
                (verify_remote and
                    md5sum(path, use_sudo=use_sudo) != digest.hexdigest())):
            with settings(hide('running')):
                put(source, path, use_sudo=use_sudo, temp_dir=temp_dir)

        if t is not None:
            os.unlink(source)

    # Ensure correct owner
    if use_sudo and owner is None:
        owner = 'root'
    if (owner and _owner(path, use_sudo) != owner) or \
       (group and _group(path, use_sudo) != group):
        func('chown %(owner)s:%(group)s "%(path)s"' % locals())

    # Ensure correct mode
    if use_sudo and mode is None:
        mode = oct(0666 & ~int(umask(use_sudo=True), base=8))
    if mode and _mode(path, use_sudo) != mode:
        func('chmod %(mode)s "%(path)s"' % locals())
Example #10
0
def file(path=None,
         contents=None,
         source=None,
         url=None,
         md5=None,
         use_sudo=False,
         owner=None,
         group='',
         mode=None,
         verify_remote=True,
         temp_dir='/tmp'):
    """
    Require a file to exist and have specific contents and properties.

    You can provide either:

    - *contents*: the required contents of the file::

        from fabtools import require

        require.file('/tmp/hello.txt', contents='Hello, world')

    - *source*: the local path of a file to upload::

        from fabtools import require

        require.file('/tmp/hello.txt', source='files/hello.txt')

    - *url*: the URL of a file to download (*path* is then optional)::

        from fabric.api import cd
        from fabtools import require

        with cd('tmp'):
            require.file(url='http://example.com/files/hello.txt')

    If *verify_remote* is ``True`` (the default), then an MD5 comparison
    will be used to check whether the remote file is the same as the
    source. If this is ``False``, the file will be assumed to be the
    same if it is present. This is useful for very large files, where
    generating an MD5 sum may take a while.

    When providing either the *contents* or the *source* parameter, Fabric's
    ``put`` function will be used to upload the file to the remote host.
    When ``use_sudo`` is ``True``, the file will first be uploaded to a temporary
    directory, then moved to its final location. The default temporary
    directory is ``/tmp``, but can be overridden with the *temp_dir* parameter.
    If *temp_dir* is an empty string, then the user's home directory will
    be used.

    If `use_sudo` is `True`, then the remote file will be owned by root,
    and its mode will reflect root's default *umask*. The optional *owner*,
    *group* and *mode* parameters can be used to override these properties.

    .. note:: This function can be accessed directly from the
              ``fabtools.require`` module for convenience.

    """
    func = use_sudo and run_as_root or run

    # 1) Only a path is given
    if path and not (contents or source or url):
        assert path
        if not is_file(path):
            func('touch "%(path)s"' % locals())

    # 2) A URL is specified (path is optional)
    elif url:
        if not path:
            path = os.path.basename(urlparse(url).path)

        if not is_file(path) or md5 and md5sum(path) != md5:
            func('wget --progress=dot:mega %(url)s -O %(path)s' % locals())

    # 3) A local filename, or a content string, is specified
    else:
        if source:
            assert not contents
            t = None
        else:
            fd, source = mkstemp()
            t = os.fdopen(fd, 'w')
            t.write(contents)
            t.close()

        if verify_remote:
            # Avoid reading the whole file into memory at once
            digest = hashlib.md5()
            f = open(source, 'rb')
            try:
                while True:
                    d = f.read(BLOCKSIZE)
                    if not d:
                        break
                    digest.update(d)
            finally:
                f.close()
        else:
            digest = None

        if (not is_file(path, use_sudo=use_sudo) or
            (verify_remote
             and md5sum(path, use_sudo=use_sudo) != digest.hexdigest())):
            with settings(hide('running')):
                put(source, path, use_sudo=use_sudo, temp_dir=temp_dir)

        if t is not None:
            os.unlink(source)

    # Ensure correct owner
    if use_sudo and owner is None:
        owner = 'root'
    if (owner and _owner(path, use_sudo) != owner) or \
       (group and _group(path, use_sudo) != group):
        func('chown %(owner)s:%(group)s "%(path)s"' % locals())

    # Ensure correct mode
    if use_sudo and mode is None:
        mode = oct(0666 & ~int(umask(use_sudo=True), base=8))
    if mode and _mode(path, use_sudo) != mode:
        func('chmod %(mode)s "%(path)s"' % locals())
Example #11
0
def file(path=None,
         contents=None,
         source=None,
         url=None,
         md5=None,
         use_sudo=False,
         owner=None,
         group='',
         mode=None,
         verify_remote=True):
    """
    Require a file

    You can provide either:
    - contents: the required contents of the file
    - source: the filename of a local file to upload
    - url: the address of a file to download (path is optional)

    If verify_remote is True (the default), then an MD5 comparison will be used
    to check whether the remote file is the same as the source. If this is
    False, the file will be assumed to be the same if it is present. This is
    useful for very large files, where generating an MD5 sum may take a while.
    """
    func = use_sudo and sudo or run

    # 1) Only a path is given
    if path and not (contents or source or url):
        assert path
        if not is_file(path):
            func('touch "%(path)s"' % locals())

    # 2) A URL is specified (path is optional)
    elif url:
        if not path:
            path = os.path.basename(urlparse(url).path)

        if not is_file(path) or md5 and md5sum(path) != md5:
            func('wget --progress=dot %(url)s' % locals())

    # 3) A local filename, or a content string, is specified
    else:
        if source:
            assert not contents
            t = None
        else:
            fd, source = mkstemp()
            t = os.fdopen(fd, 'w')
            t.write(contents)
            t.close()

        if verify_remote:
            # Avoid reading the whole file into memory at once
            digest = hashlib.md5()
            f = open(source, 'rb')
            try:
                while True:
                    d = f.read(BLOCKSIZE)
                    if not d:
                        break
                    digest.update(d)
            finally:
                f.close()
        else:
            digest = None

        if (not is_file(path, use_sudo=use_sudo) or
            (verify_remote
             and md5sum(path, use_sudo=use_sudo) != digest.hexdigest())):
            with settings(hide('running')):
                if source:
                    put(source, path, use_sudo=use_sudo)
                else:
                    put(tmp_file.name, path, use_sudo=use_sudo)
                    os.remove(tmp_file.name)

        if t is not None:
            os.unlink(source)

    # Ensure correct owner
    if owner:
        func('chown %(owner)s:%(group)s "%(path)s"' % locals())

    # Ensure correct mode
    if mode:
        func('chmod %(mode)s "%(path)s"' % locals())
Example #12
0
def file(path=None, contents=None, source=None, url=None, md5=None,
         use_sudo=False, owner=None, group='', mode=None, verify_remote=True):
    """
    Require a file to exist and have specific contents and properties.

    You can provide either:

    - *contents*: the required contents of the file::

        from fabtools import require

        require.file('/tmp/hello.txt', contents='Hello, world')

    - *source*: the local path of a file to upload::

        from fabtools import require

        require.file('/tmp/hello.txt', source='files/hello.txt')

    - *url*: the URL of a file to download (*path* is then optional)::

        from fabric.api import cd
        from fabtools import require

        with cd('tmp'):
            require.file(url='http://example.com/files/hello.txt')

    If *verify_remote* is ``True`` (the default), then an MD5 comparison
    will be used to check whether the remote file is the same as the
    source. If this is ``False``, the file will be assumed to be the
    same if it is present. This is useful for very large files, where
    generating an MD5 sum may take a while.

    .. note:: this function can be accessed directly from the
              ``fabtools.require`` module for convenience.

    """
    func = use_sudo and sudo or run

    # 1) Only a path is given
    if path and not (contents or source or url):
        assert path
        if not is_file(path):
            func('touch "%(path)s"' % locals())

    # 2) A URL is specified (path is optional)
    elif url:
        if not path:
            path = os.path.basename(urlparse(url).path)

        if not is_file(path) or md5 and md5sum(path) != md5:
            func('wget --progress=dot %(url)s' % locals())

    # 3) A local filename, or a content string, is specified
    else:
        if source:
            assert not contents
            t = None
        else:
            fd, source = mkstemp()
            t = os.fdopen(fd, 'w')
            t.write(contents)
            t.close()

        if verify_remote:
            # Avoid reading the whole file into memory at once
            digest = hashlib.md5()
            f = open(source, 'rb')
            try:
                while True:
                    d = f.read(BLOCKSIZE)
                    if not d:
                        break
                    digest.update(d)
            finally:
                f.close()
        else:
            digest = None

        if (not is_file(path, use_sudo=use_sudo) or
                (verify_remote and
                    md5sum(path, use_sudo=use_sudo) != digest.hexdigest())):
            with settings(hide('running')):
                if source:
                    put(source, path, use_sudo=use_sudo)
                else:
                    put(tmp_file.name, path, use_sudo=use_sudo)
                    os.remove(tmp_file.name)

        if t is not None:
            os.unlink(source)

    # Ensure correct owner
    if owner:
        func('chown %(owner)s:%(group)s "%(path)s"' % locals())

    # Ensure correct mode
    if mode:
        func('chmod %(mode)s "%(path)s"' % locals())
Example #13
0
def file(path=None, contents=None, source=None, url=None, md5=None,
         use_sudo=False, owner=None, group='', mode=None, verify_remote=True):
    """
    Require a file

    You can provide either:
    - contents: the required contents of the file
    - source: the filename of a local file to upload
    - url: the address of a file to download (path is optional)

    If verify_remote is True (the default), then an MD5 comparison will be used
    to check whether the remote file is the same as the source. If this is
    False, the file will be assumed to be the same if it is present. This is
    useful for very large files, where generating an MD5 sum may take a while.
    """
    func = use_sudo and sudo or run

    # 1) Only a path is given
    if path and not (contents or source or url):
        assert path
        if not is_file(path):
            func('touch "%(path)s"' % locals())

    # 2) A URL is specified (path is optional)
    elif url:
        if not path:
            path = os.path.basename(urlparse(url).path)

        if not is_file(path) or md5 and md5sum(path) != md5:
            func('wget --progress=dot %(url)s' % locals())

    # 3) A local filename, or a content string, is specified
    else:
        if source:
            assert not contents
            t = None
        else:
            fd, source = mkstemp()
            t = os.fdopen(fd, 'w')
            t.write(contents)
            t.close()

        if verify_remote:
            # Avoid reading the whole file into memory at once
            digest = hashlib.md5()
            f = open(source, 'rb')
            try:
                while True:
                    d = f.read(BLOCKSIZE)
                    if not d:
                        break
                    digest.update(d)
            finally:
                f.close()
        else:
            digest = None

        if (not is_file(path, use_sudo=use_sudo) or
                (verify_remote and
                    md5sum(path, use_sudo=use_sudo) != digest.hexdigest())):
            with settings(hide('running')):
                if source:
                    put(source, path, use_sudo=use_sudo)
                else:
                    put(tmp_file.name, path, use_sudo=use_sudo)
                    os.remove(tmp_file.name)

        if t is not None:
            os.unlink(source)

    # Ensure correct owner
    if owner:
        func('chown %(owner)s:%(group)s "%(path)s"' % locals())

    # Ensure correct mode
    if mode:
        func('chmod %(mode)s "%(path)s"' % locals())