def test_s3md5(self):
"""
Verify that the s3 item md5 is always available
"""
test_checksum = 'FakeMD5'
mock_item1 = MagicMock()
mock_item1.md5 = test_checksum
mock_item2 = MagicMock()
mock_item2.md5 = None
mock_item2.etag = test_checksum
mock_item3 = MagicMock()
mock_item3.md5 = None
mock_item3.etag = '"{}"'.format(test_checksum)
self.assertEqual(get_s3item_md5(mock_item1),test_checksum)
self.assertEqual(get_s3item_md5(mock_item2),test_checksum)
self.assertEqual(get_s3item_md5(mock_item3),test_checksum)
def test_s3md5(self):
"""
Verify that the s3 item md5 is always available
"""
test_checksum = 'FakeMD5'
mock_item1 = MagicMock()
mock_item1.md5 = test_checksum
mock_item2 = MagicMock()
mock_item2.md5 = None
mock_item2.etag = test_checksum
mock_item3 = MagicMock()
mock_item3.md5 = None
mock_item3.etag = '"{}"'.format(test_checksum)
self.assertEqual(get_s3item_md5(mock_item1), test_checksum)
self.assertEqual(get_s3item_md5(mock_item2), test_checksum)
self.assertEqual(get_s3item_md5(mock_item3), test_checksum)
def should_upload(filepath, item, force_upload):
"""
Return true if the file at filepath should be uploaded, false otherwise.
We upload if any of the following are true:
- force_upload is True
- the remote item doesn't exist
- the checksums differ and the local file is newer
"""
if force_upload or not item:
return True
local_mtime = mtime_as_datetime(filepath)
remote_mtime = s3time_as_datetime(item.last_modified)
files_differ = not md5_matches(filepath, get_s3item_md5(item))
return files_differ and local_mtime >= remote_mtime
def should_download(item, filepath, force_download):
"""
Return true if item should be downloaded to filepath, false otherwise.
We download if any of the following are true:
- force_download is True
- the file doesn't exist
- the checksums differ and the remote file is newer
"""
if force_download or not os.path.exists(filepath):
return True
local_mtime = mtime_as_datetime(filepath)
remote_mtime = s3time_as_datetime(item.last_modified)
files_differ = not md5_matches(filepath, get_s3item_md5(item))
return files_differ and remote_mtime >= local_mtime
def should_upload(filepath, item, force_upload):
"""
Return true if the file at filepath should be uploaded, false otherwise.
We upload if any of the following are true:
- force_upload is True
- the remote item doesn't exist
- the checksums differ and the local file is newer
"""
if force_upload or not item:
return True
local_mtime = mtime_as_datetime(filepath)
remote_mtime = s3time_as_datetime(item.last_modified)
files_differ = not md5_matches(filepath, get_s3item_md5(item))
return files_differ and local_mtime >= remote_mtime
def should_download(item, filepath, force_download):
"""
Return true if item should be downloaded to filepath, false otherwise.
We download if any of the following are true:
- force_download is True
- the file doesn't exist
- the checksums differ and the remote file is newer
"""
if force_download or not os.path.exists(filepath):
return True
local_mtime = mtime_as_datetime(filepath)
remote_mtime = s3time_as_datetime(item.last_modified)
files_differ = not md5_matches(filepath, get_s3item_md5(item))
return files_differ and remote_mtime >= local_mtime
def download_items(context, items, dest_dir, force_download=False):
"""
Download the s3 items given by 'items' into the destination directory
given by 'dest_dir'. If force_download is true, download *everything* in
the list. Otherwise, skip downloads for items which are already present
in the working directory.
"""
try:
no_items = 0
for item in items:
# Skip folder keys:
if item.name.find(FOLDER_SUFFIX) != -1:
verbose("Not downloading: %s", item.name)
continue
filename = os.path.basename(item.name)
filepath = os.path.join(dest_dir, filename)
if should_download(item, filepath, force_download):
f = open(filepath, 'w')
item.get_file(f,
cb=get_progress_fn(context.opts.verbose,
"Downloading %s" % item.name))
f.close()
# Verify the checksum of the downloaded item:
if not md5_matches(filepath, get_s3item_md5(item)):
raise ServiceError(
"\nDownload failed: md5 mismatch for %s" % (filename))
else:
verbose('File "%s" already exists in "%s" skipping download',
filename, dest_dir)
no_items += 1
return no_items
except IOError as ex:
err_msg = "Error opening %s: %s (%i)" % (ex.filename, ex.strerror,
ex.errno)
raise ServiceError(err_msg)
return
def download_items(context, items, dest_dir, force_download=False):
"""
Download the s3 items given by 'items' into the destination directory
given by 'dest_dir'. If force_download is true, download *everything* in
the list. Otherwise, skip downloads for items which are already present
in the working directory.
"""
try:
no_items = 0
for item in items:
# Skip folder keys:
if item.name.find(FOLDER_SUFFIX) != -1:
verbose("Not downloading: %s", item.name)
continue
filename = os.path.basename(item.name)
filepath = os.path.join(dest_dir, filename)
if should_download(item, filepath, force_download):
f = open(filepath, 'w')
item.get_file(f, cb=get_progress_fn(
context.opts.verbose, "Downloading %s" % item.name))
f.close()
# Verify the checksum of the downloaded item:
if not md5_matches(filepath, get_s3item_md5(item)):
raise ServiceError(
"\nDownload failed: md5 mismatch for %s" % (filename))
else:
verbose('File "%s" already exists in "%s" skipping download',
filename, dest_dir)
no_items += 1
return no_items
except IOError as ex:
err_msg = "Error opening %s: %s (%i)" % (
ex.filename, ex.strerror, ex.errno)
raise ServiceError(err_msg)
return