def test_md5_matching(self): """ Verify that md5 matching works properly """ filepath = 'my/random/file' file_contents = 'Random file contents' checksum = hashlib.md5(file_contents).hexdigest() m = MagicMock(spec=file, return_value=io.BytesIO(file_contents)) with patch('s3yum.util.open', m, create=True): self.assertTrue(md5_matches(filepath, checksum)) m.assert_called_once_with(filepath, 'r') return
def test_md5_matching(self): """ Verify that md5 matching works properly """ filepath = 'my/random/file' file_contents = 'Random file contents' checksum = hashlib.md5(file_contents).hexdigest() m = MagicMock(spec=file, return_value=io.BytesIO(file_contents)) with patch('s3yum.util.open', m, create=True): self.assertTrue(md5_matches(filepath,checksum)) m.assert_called_once_with(filepath, 'r') return
def should_upload(filepath, item, force_upload): """ Return true if the file at filepath should be uploaded, false otherwise. We upload if any of the following are true: - force_upload is True - the remote item doesn't exist - the checksums differ and the local file is newer """ if force_upload or not item: return True local_mtime = mtime_as_datetime(filepath) remote_mtime = s3time_as_datetime(item.last_modified) files_differ = not md5_matches(filepath, get_s3item_md5(item)) return files_differ and local_mtime >= remote_mtime
def should_download(item, filepath, force_download): """ Return true if item should be downloaded to filepath, false otherwise. We download if any of the following are true: - force_download is True - the file doesn't exist - the checksums differ and the remote file is newer """ if force_download or not os.path.exists(filepath): return True local_mtime = mtime_as_datetime(filepath) remote_mtime = s3time_as_datetime(item.last_modified) files_differ = not md5_matches(filepath, get_s3item_md5(item)) return files_differ and remote_mtime >= local_mtime
def download_items(context, items, dest_dir, force_download=False): """ Download the s3 items given by 'items' into the destination directory given by 'dest_dir'. If force_download is true, download *everything* in the list. Otherwise, skip downloads for items which are already present in the working directory. """ try: no_items = 0 for item in items: # Skip folder keys: if item.name.find(FOLDER_SUFFIX) != -1: verbose("Not downloading: %s", item.name) continue filename = os.path.basename(item.name) filepath = os.path.join(dest_dir, filename) if should_download(item, filepath, force_download): f = open(filepath, 'w') item.get_file(f, cb=get_progress_fn(context.opts.verbose, "Downloading %s" % item.name)) f.close() # Verify the checksum of the downloaded item: if not md5_matches(filepath, get_s3item_md5(item)): raise ServiceError( "\nDownload failed: md5 mismatch for %s" % (filename)) else: verbose('File "%s" already exists in "%s" skipping download', filename, dest_dir) no_items += 1 return no_items except IOError as ex: err_msg = "Error opening %s: %s (%i)" % (ex.filename, ex.strerror, ex.errno) raise ServiceError(err_msg) return
def download_items(context, items, dest_dir, force_download=False): """ Download the s3 items given by 'items' into the destination directory given by 'dest_dir'. If force_download is true, download *everything* in the list. Otherwise, skip downloads for items which are already present in the working directory. """ try: no_items = 0 for item in items: # Skip folder keys: if item.name.find(FOLDER_SUFFIX) != -1: verbose("Not downloading: %s", item.name) continue filename = os.path.basename(item.name) filepath = os.path.join(dest_dir, filename) if should_download(item, filepath, force_download): f = open(filepath, 'w') item.get_file(f, cb=get_progress_fn( context.opts.verbose, "Downloading %s" % item.name)) f.close() # Verify the checksum of the downloaded item: if not md5_matches(filepath, get_s3item_md5(item)): raise ServiceError( "\nDownload failed: md5 mismatch for %s" % (filename)) else: verbose('File "%s" already exists in "%s" skipping download', filename, dest_dir) no_items += 1 return no_items except IOError as ex: err_msg = "Error opening %s: %s (%i)" % ( ex.filename, ex.strerror, ex.errno) raise ServiceError(err_msg) return