def test_checksum(s3fs, get_md5):
"""Test fixity."""
# Compute checksum of license file
with open('LICENSE', 'rb') as fp:
data = fp.read()
checksum = get_md5(data)
counter = dict(size=0)
def callback(total, size):
counter['size'] = size
# Now do it with storage interface
with open('LICENSE', 'rb') as fp:
uri, size, save_checksum = s3fs.save(fp,
size=os.path.getsize('LICENSE'))
assert checksum == save_checksum
assert checksum == s3fs.checksum(chunk_size=2, progress_callback=callback)
assert counter['size'] == size
assert counter['size'] == os.path.getsize('LICENSE')
# No size provided, means progress callback isn't called
counter['size'] = 0
s = S3FSFileStorage(s3fs.fileurl)
assert checksum == s.checksum(chunk_size=2, progress_callback=callback)
assert counter['size'] == 0
def test_copy(s3_bucket, s3fs):
"""Test copy file."""
data = b'test'
s3fs.save(BytesIO(data))
s3_copy_path = 's3://{}/path/to/copy/data'.format(s3_bucket.name)
s3fs_copy = S3FSFileStorage(s3_copy_path)
s3fs_copy.copy(s3fs)
assert s3fs_copy.open().read() == data
tmppath = tempfile.mkdtemp()
s = PyFSFileStorage(os.path.join(tmppath, 'anotherpath/data'))
data = b'othertest'
s.save(BytesIO(data))
s3fs_copy.copy(s)
assert s3fs_copy.open().read() == data
shutil.rmtree(tmppath)
def test_non_s3_path(tmpdir):
non_s3_path = os.path.join(tmpdir.dirname, 'test.txt')
s3fs = S3FSFileStorage(non_s3_path)
fs, path = s3fs._get_fs()
assert not isinstance(fs, S3FileSystem)
def s3fs(s3_bucket, s3fs_testpath):
"""Instance of S3FSFileStorage."""
s3_storage = S3FSFileStorage(s3fs_testpath)
return s3_storage