Beispiel #1
0
    def test_md5_matching(self):
        """
        Verify that md5 matching works properly
        """
        filepath = 'my/random/file'
        file_contents = 'Random file contents'
        checksum = hashlib.md5(file_contents).hexdigest()

        m = MagicMock(spec=file, return_value=io.BytesIO(file_contents))
        with patch('s3yum.util.open', m, create=True):
            self.assertTrue(md5_matches(filepath, checksum))

        m.assert_called_once_with(filepath, 'r')
        return
Beispiel #2
0
    def test_md5_matching(self):
        """
        Verify that md5 matching works properly
        """
        filepath = 'my/random/file'
        file_contents = 'Random file contents'
        checksum = hashlib.md5(file_contents).hexdigest()

        m = MagicMock(spec=file,
            return_value=io.BytesIO(file_contents))
        with patch('s3yum.util.open', m, create=True):
            self.assertTrue(md5_matches(filepath,checksum))

        m.assert_called_once_with(filepath, 'r')
        return
Beispiel #3
0
def should_upload(filepath, item, force_upload):
    """
    Return true if the file at filepath should be uploaded, false otherwise.

    We upload if any of the following are true:
     - force_upload is True
     - the remote item doesn't exist
     - the checksums differ and the local file is newer
    """
    if force_upload or not item:
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and local_mtime >= remote_mtime
Beispiel #4
0
def should_download(item, filepath, force_download):
    """
    Return true if item should be downloaded to filepath, false otherwise.

    We download if any of the following are true:
     - force_download is True
     - the file doesn't exist
     - the checksums differ and the remote file is newer
    """
    if force_download or not os.path.exists(filepath):
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and remote_mtime >= local_mtime
Beispiel #5
0
def should_upload(filepath, item, force_upload):
    """
    Return true if the file at filepath should be uploaded, false otherwise.

    We upload if any of the following are true:
     - force_upload is True
     - the remote item doesn't exist
     - the checksums differ and the local file is newer
    """
    if force_upload or not item:
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and local_mtime >= remote_mtime
Beispiel #6
0
def should_download(item, filepath, force_download):
    """
    Return true if item should be downloaded to filepath, false otherwise.

    We download if any of the following are true:
     - force_download is True
     - the file doesn't exist
     - the checksums differ and the remote file is newer
    """
    if force_download or not os.path.exists(filepath):
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and remote_mtime >= local_mtime
Beispiel #7
0
def download_items(context, items, dest_dir, force_download=False):
    """
    Download the s3 items given by 'items' into the destination directory
    given by 'dest_dir'. If force_download is true, download *everything* in
    the list. Otherwise, skip downloads for items which are already present
    in the working directory.
    """
    try:
        no_items = 0
        for item in items:
            # Skip folder keys:
            if item.name.find(FOLDER_SUFFIX) != -1:
                verbose("Not downloading: %s", item.name)
                continue

            filename = os.path.basename(item.name)
            filepath = os.path.join(dest_dir, filename)

            if should_download(item, filepath, force_download):
                f = open(filepath, 'w')

                item.get_file(f,
                              cb=get_progress_fn(context.opts.verbose,
                                                 "Downloading %s" % item.name))

                f.close()

                # Verify the checksum of the downloaded item:
                if not md5_matches(filepath, get_s3item_md5(item)):
                    raise ServiceError(
                        "\nDownload failed: md5 mismatch for %s" % (filename))
            else:
                verbose('File "%s" already exists in "%s" skipping download',
                        filename, dest_dir)
            no_items += 1

        return no_items
    except IOError as ex:
        err_msg = "Error opening %s: %s (%i)" % (ex.filename, ex.strerror,
                                                 ex.errno)
        raise ServiceError(err_msg)
    return
Beispiel #8
0
def download_items(context, items, dest_dir, force_download=False):
    """
    Download the s3 items given by 'items' into the destination directory
    given by 'dest_dir'. If force_download is true, download *everything* in
    the list. Otherwise, skip downloads for items which are already present
    in the working directory.
    """
    try:
        no_items = 0
        for item in items:
            # Skip folder keys:
            if item.name.find(FOLDER_SUFFIX) != -1:
                verbose("Not downloading: %s", item.name)
                continue

            filename = os.path.basename(item.name)
            filepath = os.path.join(dest_dir, filename)

            if should_download(item, filepath, force_download):
                f = open(filepath, 'w')

                item.get_file(f, cb=get_progress_fn(
                    context.opts.verbose, "Downloading %s" % item.name))

                f.close()

                # Verify the checksum of the downloaded item:
                if not md5_matches(filepath, get_s3item_md5(item)):
                    raise ServiceError(
                        "\nDownload failed: md5 mismatch for %s" % (filename))
            else:
                verbose('File "%s" already exists in "%s" skipping download',
                        filename, dest_dir)
            no_items += 1

        return no_items
    except IOError as ex:
        err_msg = "Error opening %s: %s (%i)" % (
            ex.filename, ex.strerror, ex.errno)
        raise ServiceError(err_msg)
    return