def file(path=None, contents=None, source=None, url=None, md5=None, use_sudo=False, owner=None, group='', mode=None): """ Require a file You can provide either: - contents: the required contents of the file - source: the filename of a local file to upload - url: the address of a file to download (path is optional) """ func = use_sudo and sudo or run # 1) Only a path is given if path and not (contents or source or url): assert path if not is_file(path): func('touch "%(path)s"' % locals()) # 2) A URL is specified (path is optional) elif url: if not path: path = os.path.basename(urlparse(url).path) if not is_file(path) or md5 and md5sum(path) != md5: func('wget --progress=dot %(url)s' % locals()) # 3) A local filename, or a content string, is specified else: if source: assert not contents contents = open(source).read() else: tmp_file = NamedTemporaryFile(delete=False) tmp_file.write(contents) tmp_file.close() if not is_file( path) or md5sum(path) != hashlib.md5(contents).hexdigest(): with settings(hide('running')): if source: put(source, path, use_sudo=use_sudo) else: put(tmp_file.name, path, use_sudo=use_sudo) os.remove(tmp_file.name) # Ensure correct owner if owner: func('chown %(owner)s:%(group)s "%(path)s"' % locals()) # Ensure correct mode if mode: func('chmod %(mode)s "%(path)s"' % locals())
def file(path=None, contents=None, source=None, url=None, md5=None, use_sudo=False, owner=None, group='', mode=None): """ Require a file You can provide either: - contents: the required contents of the file - source: the filename of a local file to upload - url: the address of a file to download (path is optional) """ func = use_sudo and sudo or run # 1) Only a path is given if path and not (contents or source or url): assert path if not is_file(path): func('touch "%(path)s"' % locals()) # 2) A URL is specified (path is optional) elif url: if not path: path = os.path.basename(urlparse(url).path) if not is_file(path) or md5 and md5sum(path) != md5: func('wget --progress=dot %(url)s' % locals()) # 3) A local filename, or a content string, is specified else: if source: assert not contents contents = open(source).read() else: tmp_file = NamedTemporaryFile(delete=False) tmp_file.write(contents) tmp_file.close() if not is_file(path) or md5sum(path) != hashlib.md5(contents).hexdigest(): with settings(hide('running')): if source: put(source, path, use_sudo=use_sudo) else: put(tmp_file.name, path, use_sudo=use_sudo) os.remove(tmp_file.name) # Ensure correct owner if owner: func('chown %(owner)s:%(group)s "%(path)s"' % locals()) # Ensure correct mode if mode: func('chmod %(mode)s "%(path)s"' % locals())
def _upload_file(local_file, drop, md5_local, tries_left, user, group, use_sudo): remote_files = put(local_file, drop, mode="0444", use_sudo=use_sudo) if len(remote_files) == 1: rf = remote_files[0] if rf in remote_files.failed: if tries_left == 0: print "Upload failed... aborting..." return None else: print "Upload failed... trying again. " + str(tries_left) + " tries left." return _upload_file(local_file, drop, md5_local, tries_left - 1, user, group, use_sudo) else: md5_remote = md5sum(rf) if md5_local == md5_remote: print "MD5 match. Uploaded succeeded." sudo("chown {u}:{g} {f}".format(u=user, g=group, f=rf)) return rf else: print "MD5 mismatch." print "Local: " + md5_local print "Remote: " + md5_remote if tries_left == 0: print "Upload failed... aborting..." return None else: print "Upload failed... trying again. " + str(tries_left) + " tries left." return _upload_file(local_file, drop, md5_local, tries_left - 1, user, group, use_sudo)
def git_require_no_update(): """ Test working_copy() with update=False """ from fabric.api import run from fabtools.files import md5sum from fabtools import require run("tar -c -f wc_old.tar --exclude .git wc") old_md5 = md5sum("wc_old.tar") require.git.working_copy(REMOTE_URL, path="wc", update=False) # Test that the working tree was not updated run("tar -c -f wc_new.tar --exclude .git wc") new_md5 = md5sum("wc_new.tar") assert old_md5 == new_md5
def git_require_no_update(): """ Test working_copy() with update=False """ from fabric.api import run from fabtools.files import md5sum from fabtools import require run('tar -c -f wc_old.tar --exclude .git wc') old_md5 = md5sum('wc_old.tar') require.git.working_copy(REMOTE_URL, path='wc', update=False) # Test that the working tree was not updated run('tar -c -f wc_new.tar --exclude .git wc') new_md5 = md5sum('wc_new.tar') assert old_md5 == new_md5
def test_git_require_no_update(): """ Test working_copy() with update=False """ from fabtools.require.git import working_copy try: working_copy(REMOTE_URL, path='wc') run('tar -c -f wc_old.tar --exclude .git wc') old_md5 = md5sum('wc_old.tar') working_copy(REMOTE_URL, path='wc', update=False) # Test that the working tree was not updated run('tar -c -f wc_new.tar --exclude .git wc') new_md5 = md5sum('wc_new.tar') assert old_md5 == new_md5 finally: run('rm -rf wc')
def downloaded_file(url, filename=None, md5=None, use_sudo=False): """ I can haz downloaded file """ func = use_sudo and sudo or run if not filename: filename = os.path.basename(urlparse(url).path) download = True if is_file(filename): if (md5 is None) or (md5 == md5sum(filename)): download = False if download: func('wget --progress=dot %(url)s' % locals())
def file(path=None, contents=None, source=None, url=None, md5=None, use_sudo=False, owner=None, group='', mode=None, verify_remote=True, temp_dir='/tmp'): """ Require a file to exist and have specific contents and properties. You can provide either: - *contents*: the required contents of the file:: from fabtools import require require.file('/tmp/hello.txt', contents='Hello, world') - *source*: the local path of a file to upload:: from fabtools import require require.file('/tmp/hello.txt', source='files/hello.txt') - *url*: the URL of a file to download (*path* is then optional):: from fabric.api import cd from fabtools import require with cd('tmp'): require.file(url='http://example.com/files/hello.txt') If *verify_remote* is ``True`` (the default), then an MD5 comparison will be used to check whether the remote file is the same as the source. If this is ``False``, the file will be assumed to be the same if it is present. This is useful for very large files, where generating an MD5 sum may take a while. When providing either the *contents* or the *source* parameter, Fabric's ``put`` function will be used to upload the file to the remote host. When ``use_sudo`` is ``True``, the file will first be uploaded to a temporary directory, then moved to its final location. The default temporary directory is ``/tmp``, but can be overridden with the *temp_dir* parameter. If *temp_dir* is an empty string, then the user's home directory will be used. If `use_sudo` is `True`, then the remote file will be owned by root, and its mode will reflect root's default *umask*. The optional *owner*, *group* and *mode* parameters can be used to override these properties. .. note:: This function can be accessed directly from the ``fabtools.require`` module for convenience. """ func = use_sudo and run_as_root or run # 1) Only a path is given if path and not (contents or source or url): assert path if not is_file(path): func('touch "%(path)s"' % locals()) # 2) A URL is specified (path is optional) elif url: if not path: path = os.path.basename(urlparse(url).path) if not is_file(path) or md5 and md5sum(path) != md5: func('wget --progress=dot:mega "%(url)s" -O "%(path)s"' % locals()) # 3) A local filename, or a content string, is specified else: if source: assert not contents t = None else: fd, source = mkstemp() t = os.fdopen(fd, 'w') t.write(contents) t.close() if verify_remote: # Avoid reading the whole file into memory at once digest = hashlib.md5() f = open(source, 'rb') try: while True: d = f.read(BLOCKSIZE) if not d: break digest.update(d) finally: f.close() else: digest = None if (not is_file(path, use_sudo=use_sudo) or (verify_remote and md5sum(path, use_sudo=use_sudo) != digest.hexdigest())): with settings(hide('running')): put(source, path, use_sudo=use_sudo, temp_dir=temp_dir) if t is not None: os.unlink(source) # Ensure correct owner if use_sudo and owner is None: owner = 'root' if (owner and _owner(path, use_sudo) != owner) or \ (group and _group(path, use_sudo) != group): func('chown %(owner)s:%(group)s "%(path)s"' % locals()) # Ensure correct mode if use_sudo and mode is None: mode = oct(0666 & ~int(umask(use_sudo=True), base=8)) if mode and _mode(path, use_sudo) != mode: func('chmod %(mode)s "%(path)s"' % locals())
def file(path=None, contents=None, source=None, url=None, md5=None, use_sudo=False, owner=None, group='', mode=None, verify_remote=True, temp_dir='/tmp'): """ Require a file to exist and have specific contents and properties. You can provide either: - *contents*: the required contents of the file:: from fabtools import require require.file('/tmp/hello.txt', contents='Hello, world') - *source*: the local path of a file to upload:: from fabtools import require require.file('/tmp/hello.txt', source='files/hello.txt') - *url*: the URL of a file to download (*path* is then optional):: from fabric.api import cd from fabtools import require with cd('tmp'): require.file(url='http://example.com/files/hello.txt') If *verify_remote* is ``True`` (the default), then an MD5 comparison will be used to check whether the remote file is the same as the source. If this is ``False``, the file will be assumed to be the same if it is present. This is useful for very large files, where generating an MD5 sum may take a while. When providing either the *contents* or the *source* parameter, Fabric's ``put`` function will be used to upload the file to the remote host. When ``use_sudo`` is ``True``, the file will first be uploaded to a temporary directory, then moved to its final location. The default temporary directory is ``/tmp``, but can be overridden with the *temp_dir* parameter. If *temp_dir* is an empty string, then the user's home directory will be used. If `use_sudo` is `True`, then the remote file will be owned by root, and its mode will reflect root's default *umask*. The optional *owner*, *group* and *mode* parameters can be used to override these properties. .. note:: This function can be accessed directly from the ``fabtools.require`` module for convenience. """ func = use_sudo and run_as_root or run # 1) Only a path is given if path and not (contents or source or url): assert path if not is_file(path): func('touch "%(path)s"' % locals()) # 2) A URL is specified (path is optional) elif url: if not path: path = os.path.basename(urlparse(url).path) if not is_file(path) or md5 and md5sum(path) != md5: func('wget --progress=dot:mega %(url)s -O %(path)s' % locals()) # 3) A local filename, or a content string, is specified else: if source: assert not contents t = None else: fd, source = mkstemp() t = os.fdopen(fd, 'w') t.write(contents) t.close() if verify_remote: # Avoid reading the whole file into memory at once digest = hashlib.md5() f = open(source, 'rb') try: while True: d = f.read(BLOCKSIZE) if not d: break digest.update(d) finally: f.close() else: digest = None if (not is_file(path, use_sudo=use_sudo) or (verify_remote and md5sum(path, use_sudo=use_sudo) != digest.hexdigest())): with settings(hide('running')): put(source, path, use_sudo=use_sudo, temp_dir=temp_dir) if t is not None: os.unlink(source) # Ensure correct owner if use_sudo and owner is None: owner = 'root' if (owner and _owner(path, use_sudo) != owner) or \ (group and _group(path, use_sudo) != group): func('chown %(owner)s:%(group)s "%(path)s"' % locals()) # Ensure correct mode if use_sudo and mode is None: mode = oct(0666 & ~int(umask(use_sudo=True), base=8)) if mode and _mode(path, use_sudo) != mode: func('chmod %(mode)s "%(path)s"' % locals())
def file(path=None, contents=None, source=None, url=None, md5=None, use_sudo=False, owner=None, group='', mode=None, verify_remote=True): """ Require a file You can provide either: - contents: the required contents of the file - source: the filename of a local file to upload - url: the address of a file to download (path is optional) If verify_remote is True (the default), then an MD5 comparison will be used to check whether the remote file is the same as the source. If this is False, the file will be assumed to be the same if it is present. This is useful for very large files, where generating an MD5 sum may take a while. """ func = use_sudo and sudo or run # 1) Only a path is given if path and not (contents or source or url): assert path if not is_file(path): func('touch "%(path)s"' % locals()) # 2) A URL is specified (path is optional) elif url: if not path: path = os.path.basename(urlparse(url).path) if not is_file(path) or md5 and md5sum(path) != md5: func('wget --progress=dot %(url)s' % locals()) # 3) A local filename, or a content string, is specified else: if source: assert not contents t = None else: fd, source = mkstemp() t = os.fdopen(fd, 'w') t.write(contents) t.close() if verify_remote: # Avoid reading the whole file into memory at once digest = hashlib.md5() f = open(source, 'rb') try: while True: d = f.read(BLOCKSIZE) if not d: break digest.update(d) finally: f.close() else: digest = None if (not is_file(path, use_sudo=use_sudo) or (verify_remote and md5sum(path, use_sudo=use_sudo) != digest.hexdigest())): with settings(hide('running')): if source: put(source, path, use_sudo=use_sudo) else: put(tmp_file.name, path, use_sudo=use_sudo) os.remove(tmp_file.name) if t is not None: os.unlink(source) # Ensure correct owner if owner: func('chown %(owner)s:%(group)s "%(path)s"' % locals()) # Ensure correct mode if mode: func('chmod %(mode)s "%(path)s"' % locals())
def file(path=None, contents=None, source=None, url=None, md5=None, use_sudo=False, owner=None, group='', mode=None, verify_remote=True): """ Require a file to exist and have specific contents and properties. You can provide either: - *contents*: the required contents of the file:: from fabtools import require require.file('/tmp/hello.txt', contents='Hello, world') - *source*: the local path of a file to upload:: from fabtools import require require.file('/tmp/hello.txt', source='files/hello.txt') - *url*: the URL of a file to download (*path* is then optional):: from fabric.api import cd from fabtools import require with cd('tmp'): require.file(url='http://example.com/files/hello.txt') If *verify_remote* is ``True`` (the default), then an MD5 comparison will be used to check whether the remote file is the same as the source. If this is ``False``, the file will be assumed to be the same if it is present. This is useful for very large files, where generating an MD5 sum may take a while. .. note:: this function can be accessed directly from the ``fabtools.require`` module for convenience. """ func = use_sudo and sudo or run # 1) Only a path is given if path and not (contents or source or url): assert path if not is_file(path): func('touch "%(path)s"' % locals()) # 2) A URL is specified (path is optional) elif url: if not path: path = os.path.basename(urlparse(url).path) if not is_file(path) or md5 and md5sum(path) != md5: func('wget --progress=dot %(url)s' % locals()) # 3) A local filename, or a content string, is specified else: if source: assert not contents t = None else: fd, source = mkstemp() t = os.fdopen(fd, 'w') t.write(contents) t.close() if verify_remote: # Avoid reading the whole file into memory at once digest = hashlib.md5() f = open(source, 'rb') try: while True: d = f.read(BLOCKSIZE) if not d: break digest.update(d) finally: f.close() else: digest = None if (not is_file(path, use_sudo=use_sudo) or (verify_remote and md5sum(path, use_sudo=use_sudo) != digest.hexdigest())): with settings(hide('running')): if source: put(source, path, use_sudo=use_sudo) else: put(tmp_file.name, path, use_sudo=use_sudo) os.remove(tmp_file.name) if t is not None: os.unlink(source) # Ensure correct owner if owner: func('chown %(owner)s:%(group)s "%(path)s"' % locals()) # Ensure correct mode if mode: func('chmod %(mode)s "%(path)s"' % locals())