Esempio n. 1
0
 def test_s3md5(self):
     """
     Verify that the s3 item md5 is always available
     """
     test_checksum = 'FakeMD5'
     mock_item1 = MagicMock()
     mock_item1.md5 = test_checksum
     mock_item2 = MagicMock()
     mock_item2.md5 = None
     mock_item2.etag = test_checksum
     mock_item3 = MagicMock()
     mock_item3.md5 = None
     mock_item3.etag = '"{}"'.format(test_checksum)
     self.assertEqual(get_s3item_md5(mock_item1),test_checksum)
     self.assertEqual(get_s3item_md5(mock_item2),test_checksum)
     self.assertEqual(get_s3item_md5(mock_item3),test_checksum)
Esempio n. 2
0
 def test_s3md5(self):
     """
     Verify that the s3 item md5 is always available
     """
     test_checksum = 'FakeMD5'
     mock_item1 = MagicMock()
     mock_item1.md5 = test_checksum
     mock_item2 = MagicMock()
     mock_item2.md5 = None
     mock_item2.etag = test_checksum
     mock_item3 = MagicMock()
     mock_item3.md5 = None
     mock_item3.etag = '"{}"'.format(test_checksum)
     self.assertEqual(get_s3item_md5(mock_item1), test_checksum)
     self.assertEqual(get_s3item_md5(mock_item2), test_checksum)
     self.assertEqual(get_s3item_md5(mock_item3), test_checksum)
Esempio n. 3
0
def should_upload(filepath, item, force_upload):
    """
    Return true if the file at filepath should be uploaded, false otherwise.

    We upload if any of the following are true:
     - force_upload is True
     - the remote item doesn't exist
     - the checksums differ and the local file is newer
    """
    if force_upload or not item:
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and local_mtime >= remote_mtime
Esempio n. 4
0
def should_download(item, filepath, force_download):
    """
    Return true if item should be downloaded to filepath, false otherwise.

    We download if any of the following are true:
     - force_download is True
     - the file doesn't exist
     - the checksums differ and the remote file is newer
    """
    if force_download or not os.path.exists(filepath):
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and remote_mtime >= local_mtime
Esempio n. 5
0
def should_upload(filepath, item, force_upload):
    """
    Return true if the file at filepath should be uploaded, false otherwise.

    We upload if any of the following are true:
     - force_upload is True
     - the remote item doesn't exist
     - the checksums differ and the local file is newer
    """
    if force_upload or not item:
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and local_mtime >= remote_mtime
Esempio n. 6
0
def should_download(item, filepath, force_download):
    """
    Return true if item should be downloaded to filepath, false otherwise.

    We download if any of the following are true:
     - force_download is True
     - the file doesn't exist
     - the checksums differ and the remote file is newer
    """
    if force_download or not os.path.exists(filepath):
        return True

    local_mtime = mtime_as_datetime(filepath)
    remote_mtime = s3time_as_datetime(item.last_modified)
    files_differ = not md5_matches(filepath, get_s3item_md5(item))
    return files_differ and remote_mtime >= local_mtime
Esempio n. 7
0
def download_items(context, items, dest_dir, force_download=False):
    """
    Download the s3 items given by 'items' into the destination directory
    given by 'dest_dir'. If force_download is true, download *everything* in
    the list. Otherwise, skip downloads for items which are already present
    in the working directory.
    """
    try:
        no_items = 0
        for item in items:
            # Skip folder keys:
            if item.name.find(FOLDER_SUFFIX) != -1:
                verbose("Not downloading: %s", item.name)
                continue

            filename = os.path.basename(item.name)
            filepath = os.path.join(dest_dir, filename)

            if should_download(item, filepath, force_download):
                f = open(filepath, 'w')

                item.get_file(f,
                              cb=get_progress_fn(context.opts.verbose,
                                                 "Downloading %s" % item.name))

                f.close()

                # Verify the checksum of the downloaded item:
                if not md5_matches(filepath, get_s3item_md5(item)):
                    raise ServiceError(
                        "\nDownload failed: md5 mismatch for %s" % (filename))
            else:
                verbose('File "%s" already exists in "%s" skipping download',
                        filename, dest_dir)
            no_items += 1

        return no_items
    except IOError as ex:
        err_msg = "Error opening %s: %s (%i)" % (ex.filename, ex.strerror,
                                                 ex.errno)
        raise ServiceError(err_msg)
    return
Esempio n. 8
0
def download_items(context, items, dest_dir, force_download=False):
    """
    Download the s3 items given by 'items' into the destination directory
    given by 'dest_dir'. If force_download is true, download *everything* in
    the list. Otherwise, skip downloads for items which are already present
    in the working directory.
    """
    try:
        no_items = 0
        for item in items:
            # Skip folder keys:
            if item.name.find(FOLDER_SUFFIX) != -1:
                verbose("Not downloading: %s", item.name)
                continue

            filename = os.path.basename(item.name)
            filepath = os.path.join(dest_dir, filename)

            if should_download(item, filepath, force_download):
                f = open(filepath, 'w')

                item.get_file(f, cb=get_progress_fn(
                    context.opts.verbose, "Downloading %s" % item.name))

                f.close()

                # Verify the checksum of the downloaded item:
                if not md5_matches(filepath, get_s3item_md5(item)):
                    raise ServiceError(
                        "\nDownload failed: md5 mismatch for %s" % (filename))
            else:
                verbose('File "%s" already exists in "%s" skipping download',
                        filename, dest_dir)
            no_items += 1

        return no_items
    except IOError as ex:
        err_msg = "Error opening %s: %s (%i)" % (
            ex.filename, ex.strerror, ex.errno)
        raise ServiceError(err_msg)
    return