def get_file(self, local_path, remote_path):
from pcs_api.bytes_io import (FileByteSink, StdoutProgressListener)
from pcs_api.models import (CPath, CDownloadRequest)
bpath = CPath(remote_path)
download_request = CDownloadRequest(bpath, FileByteSink(local_path))
download_request.progress_listener(StdoutProgressListener())
self.storage.download(download_request)
def check(self):
from pcs_api.bytes_io import (MemoryByteSource, MemoryByteSink,
StdoutProgressListener)
from pcs_api.models import (CPath, CUploadRequest, CDownloadRequest)
msg = "Cloud storage user_id = " + repr(self.storage.get_user_id())
self.ctx.logger.info(msg)
msg = "Cloud storage quota = " + repr(self.storage.get_quota())
self.ctx.logger.info(msg)
self.ctx.logger.info("Cloud storage is ready")
fpath = CPath('/test_dir')
self.storage.create_folder(fpath)
bpath = fpath.add("test.txt")
file_contents_uploaded = b"Test file contents"
upload_request = CUploadRequest(
bpath, MemoryByteSource(file_contents_uploaded)).content_type(
'text/plain')
upload_request.progress_listener(StdoutProgressListener())
self.storage.upload(upload_request)
file_contents_downloaded = MemoryByteSink()
download_request = CDownloadRequest(bpath, file_contents_downloaded)
download_request.progress_listener(StdoutProgressListener())
self.storage.download(download_request)
self.storage.delete(fpath)
if file_contents_uploaded != file_contents_downloaded.get_bytes():
raise RemotePCSError
def get_bytes(self, remote_path):
from pcs_api.bytes_io import (MemoryByteSink, StdoutProgressListener)
from pcs_api.models import (CPath, CDownloadRequest)
remote_bytes = MemoryByteSink()
bpath = CPath(remote_path)
download_request = CDownloadRequest(bpath, remote_bytes)
download_request.progress_listener(StdoutProgressListener())
self.storage.download(download_request)
return remote_bytes.get_bytes()
def test_download_request_progress_listener():
mbs = MemoryByteSink()
dr = CDownloadRequest(CPath("/foo"), mbs)
assert dr.byte_sink() is mbs
# Now if we decorate:
pl = StdoutProgressListener()
dr.progress_listener(pl)
os = dr.byte_sink().open_stream()
os.write(b"a")
assert pl.current == 1
def upload_and_check_random_files(storage, root_path):
rnd = random.Random() # not shared (in case multi-thread)
tmp_path = CPath(root_path.path_name())
for i in range(0, rnd.randint(1, 5)):
c_path = miscutils.generate_test_path(tmp_path)
if rnd.randint(0, 2) == 0:
# Create folder:
storage.create_folder(c_path)
# And sometimes go inside:
if rnd.randint(0, 2) == 0:
tmp_path = c_path
else:
# Create file (deterministic content for later check):
rnd.seed(c_path)
file_size = rnd.randint(0, 1000) * rnd.randint(
0, 1000) # prefer small files
data = miscutils.generate_random_bytearray(file_size, rnd)
ur = CUploadRequest(c_path, MemoryByteSource(data))
storage.upload(ur)
# Check blob files content:
all_blobs = recursively_list_blobs(storage, root_path)
logger.info('All uploaded blobs = %s', all_blobs)
for blob in all_blobs:
rnd.seed(blob.path)
file_size = rnd.randint(0, 1000) * rnd.randint(
0, 1000) # same formula as above
assert file_size == blob.length
expected_data = miscutils.generate_random_bytearray(file_size, rnd)
bsink = MemoryByteSink()
dr = CDownloadRequest(blob.path, bsink)
storage.download(dr)
downloaded_data = bsink.get_bytes()
# In order to track strange download errors:
if expected_data != downloaded_data:
logger.error(
'Downloaded data differ (%d / %d bytes): will download again for disgnostic',
len(expected_data), len(downloaded_data))
bsink2 = MemoryByteSink()
dr2 = CDownloadRequest(blob.path, bsink2)
storage.download(dr2)
downloaded_data2 = bsink2.get_bytes()
logger.error(
'Second download: (%d bytes). Data are the same now?=%r / Same as first download?=%r',
len(downloaded_data2), expected_data == downloaded_data2,
downloaded_data == downloaded_data2)
# py.test cannot really compare bytearray and bytes, so we convert for nicer display:
assert bytes(expected_data) == downloaded_data
# Delete all files:
storage.delete(root_path)
def test_download_request_progress_listener():
mbs = MemoryByteSink()
dr = CDownloadRequest(CPath('/foo'), mbs)
assert dr.byte_sink() is mbs
# Now if we decorate:
pl = StdoutProgressListener()
dr.progress_listener(pl)
os = dr.byte_sink().open_stream()
os.write(b'a')
assert pl.current == 1
def test_abort_during_download(storage, tmpdir):
"""If download fails, check that abort is called on progress listener"""
# Upload a quite big file (for download):
file_size = 500000
c_path = miscutils.generate_test_path()
logger.info('Uploading blob with size %d bytes to %s', file_size, c_path)
try:
logger.info('Will upload a blob for download test (%d bytes) to %s',
file_size, c_path)
content = bytearray(os.urandom(file_size))
upload_request = CUploadRequest(c_path, MemoryByteSource(content))
storage.upload(upload_request)
# Now we download, asking for progress:
logger.info('Will download this blob but fail during download...')
local_pathname = tmpdir.join('/back_from_provider').strpath
fbs = FileByteSink(local_pathname,
temp_name_during_writes=False,
delete_on_abort=True)
pl = TestAbortProgressListener(
1, # abort once
file_size / 2, # abort at half download
False) # error is not temporary
pbs = ProgressByteSink(fbs, pl)
dr = CDownloadRequest(c_path, pbs)
with pytest.raises(RuntimeError) as rte:
storage.download(dr)
pytest.fail("Download should have failed !")
logger.info('Download has failed (as expected)')
# Check listener saw the abort:
assert pl.is_aborted
# check file does not exist (has been removed):
logger.info('Check destination file does not exist: %s',
local_pathname)
assert not os.path.exists(local_pathname)
finally:
if c_path:
storage.delete(c_path)
fpath = CPath('/pcs_api_new_folder')
storage.create_folder(fpath)
# Upload a local file in this folder :
bpath = fpath.add('pcs_api_new_file')
file_content = b'this is file content...'
upload_request = CUploadRequest(bpath, MemoryByteSource(file_content)).content_type('text/plain')
storage.upload(upload_request)
# Download back the file :
mbs = MemoryByteSink()
download_request = CDownloadRequest(bpath, mbs)
storage.download(download_request)
assert mbs.get_bytes() == file_content
# delete remote folder :
storage.delete(fpath)
# Example : we'll download a range of largest blob :
if largest_blob:
range_start = largest_blob.length / 2
range_length = min(largest_blob.length / 2, 1000000)
bs = FileByteSink('dest_file.txt')
print("Will download range from largest blob : %s to : %s" %
(largest_blob, bs))
dr = CDownloadRequest(largest_blob.path, bs).range(range_start, range_length)
# watch download progress :
dr.progress_listener(StdoutProgressListener())
storage.download(dr)
print()
def test_download_request_bytes_range():
dr = CDownloadRequest(CPath('/foo'), MemoryByteSink())
dr.range(None, None)
assert dr.get_http_headers() == {}
dr.range(None, 100)
assert dr.get_http_headers() == { 'Range': 'bytes=-100'}
dr.range(10, 100)
assert dr.get_http_headers() == { 'Range': 'bytes=10-109'}
dr.range(100, None)
assert dr.get_http_headers() == { 'Range': 'bytes=100-'}
def test_file_operations(storage):
# We'll use a temp folder for tests:
temp_root_path = miscutils.generate_test_path()
logger.info('Will use test folder: %s', temp_root_path)
# Create a sub-folder:
sub_path = temp_root_path.add('sub_folder')
logger.info('Creating sub_folder: %s', sub_path)
assert storage.create_folder(sub_path) is True # True because actually created
assert storage.create_folder(sub_path) is False # False because not created
# Check back:
sub_folder = storage.get_file(sub_path)
assert sub_folder.path == sub_path
assert sub_folder.is_folder()
assert not sub_folder.is_blob()
if sub_folder.modification_time: # Not all providers have a modif time on folders
miscutils.assert_datetime_is_almost_now(sub_folder.modification_time)
# Upload 2 files into this sub-folder:
fpath1 = sub_path.add('a_test_file1')
content_file1 = b'This is binary cont€nt of test file 1...'
logger.info('Uploading blob to: %s', fpath1)
upload_request = CUploadRequest(fpath1, MemoryByteSource(content_file1))
storage.upload(upload_request)
fpath2 = sub_path.add('a_test_file2')
# Generate a quite big random data:
content_file2 = bytearray(os.urandom(500000))
logger.info('Uploading blob to: %s', fpath2)
upload_request = CUploadRequest(fpath2, MemoryByteSource(content_file2))
storage.upload(upload_request)
# Check uploaded blobs informations:
# we check file2 first because has just been uploaded / for modif time check
cblob = storage.get_file(fpath2)
assert cblob.is_blob()
assert not cblob.is_folder()
assert cblob.length == len(content_file2)
miscutils.assert_datetime_is_almost_now(cblob.modification_time)
cblob = storage.get_file(fpath1)
assert cblob.is_blob()
assert not cblob.is_folder()
assert cblob.length == len(content_file1)
# Download data, and check:
logger.info('Downloading back and checking file: %s', fpath1)
mbs = MemoryByteSink()
download_request = CDownloadRequest(fpath1, mbs)
storage.download(download_request)
assert mbs.get_bytes() == content_file1
logger.info('Downloading back and checking file ranges: %s', fpath1)
download_request.range(5, None) # starting at offset 5
storage.download(download_request)
assert mbs.get_bytes() == content_file1[5:]
if storage.provider_name() != 'rapidshare':
# rapidshare does not support such requests:
download_request.range(None, 5) # last 5 bytes
storage.download(download_request);
assert mbs.get_bytes() == content_file1[-5:]
download_request.range(2, 5); # 5 bytes at offset 2
storage.download(download_request);
assert mbs.get_bytes() == content_file1[2:7]
logger.info('Downloading back and checking file: %s', fpath2)
download_request = CDownloadRequest(fpath2, mbs)
storage.download(download_request)
assert mbs.get_bytes() == content_file2
# Check that if we upload again, blob is replaced:
logger.info('Checking file overwrite: %s', fpath2)
content_file2 = bytearray(os.urandom(300000)) # 300kB file
upload_request = CUploadRequest(fpath2, MemoryByteSource(content_file2))
storage.upload(upload_request)
storage.download(download_request)
assert mbs.get_bytes() == content_file2
# Check that we can replace existing blob with empty content:
logger.info('Checking file overwrite with empty file: %s', fpath2)
content_file2 = b''
upload_request = CUploadRequest(fpath2, MemoryByteSource(content_file2))
storage.upload(upload_request)
storage.download(download_request)
assert mbs.get_bytes() == content_file2
# Create a sub_sub_folder:
sub_sub_path = sub_path.add('a_sub_sub_folder')
logger.info('Creating a_sub_sub_folder: %s', sub_sub_path)
created = storage.create_folder(sub_sub_path)
assert created
logger.info('Check uploaded blobs and sub_sub_folder all appear in folder list')
folder_content = storage.list_folder(sub_folder)
logger.info('sub_folder contains files: %r', folder_content)
# It happened once here that hubic did not list fpath1 'a_test_file1' in folder_content:
# only 2 files were present ?!
assert len(folder_content) == 3
assert fpath1 in folder_content
assert folder_content[fpath1].is_blob()
assert not folder_content[fpath1].is_folder()
assert fpath2 in folder_content
assert folder_content[fpath2].is_blob()
assert not folder_content[fpath2].is_folder()
assert sub_sub_path in folder_content
assert not folder_content[sub_sub_path].is_blob()
assert folder_content[sub_sub_path].is_folder()
logger.info('Check that list of sub_sub folder is empty: %s', sub_sub_path)
assert storage.list_folder(sub_sub_path) == {}
logger.info('Check that listing content of a blob raises: %s', fpath1)
try:
storage.list_folder(fpath1)
pytest.fail('Listing a blob should raise')
except CInvalidFileTypeError as e:
assert e.path == fpath1
assert e.expected_blob is False
logger.info('Delete file1: %s', fpath1)
assert storage.delete(fpath1) is True # We have deleted the file
assert storage.delete(fpath1) is False # We have not deleted anything
logger.info('Check file1 does not appear anymore in folder: %s', sub_folder)
assert fpath1 not in storage.list_folder(sub_folder)
tmp = storage.get_file(fpath1)
assert tmp is None
logger.info('Delete whole test folder: %s', temp_root_path)
ret = storage.delete(temp_root_path)
assert ret is True # We have deleted at least one file
logger.info('Deleting again returns False')
ret = storage.delete(temp_root_path)
assert ret is False # We have not deleted anything
logger.info('Listing a deleted folder returns None: %s', temp_root_path)
assert storage.list_folder(temp_root_path) is None
assert storage.get_file(temp_root_path) is None
def test_file_with_special_chars(storage):
temp_root_path = miscutils.generate_test_path()
try:
safe_name = _filter_name_for_provider(
storage.provider_name(),
"hum...\u00a0',;.:\u00a0!*%&~#{[|`_ç^@ £€")
folder_path = temp_root_path.add(safe_name)
storage.create_folder(folder_path)
fback = storage.get_file(folder_path)
assert fback.path == folder_path
assert fback.is_folder()
assert not fback.is_blob()
# Folder must appear in test root folder list:
root_test_content = storage.list_folder(temp_root_path)
assert folder_path in root_test_content
assert root_test_content[folder_path].path == folder_path
assert root_test_content[folder_path].is_folder()
assert not root_test_content[folder_path].is_blob()
# Generate a random blob name (ensure it does not start nor end with a space)
nameb = list()
nameb.append('b')
for i in range(0, 30):
nameb.append(
_generate_random_blob_name_char(storage.provider_name()))
nameb.append('e')
blob_path = None
last_blob_path = None
for i in range(0, 10):
last_blob_path = blob_path
# slightly change blob name, so that we get similar but different names:
index = int(random.random() * (len(nameb) - 2)) + 1
nameb[index] = _generate_random_blob_name_char(
storage.provider_name())
blob_name = ''.join(nameb)
blob_path = folder_path.add(blob_name)
logger.info("Will upload file to path: %r", blob_path)
content_file = b'This is cont€nt of test file: ' + blob_name.encode(
'UTF-8')
upload_request = CUploadRequest(blob_path,
MemoryByteSource(content_file))
upload_request.content_type('text/plain; charset=UTF-8')
storage.upload(upload_request)
bback = storage.get_file(blob_path)
if bback is None:
logger.error('Test failed with blob_path=%r', blob_path)
logger.error(' last_blob_path=%r', last_blob_path)
assert bback is not None
assert bback.path == blob_path
assert bback.is_blob()
assert not bback.is_folder()
# Download and check content:
mbs = MemoryByteSink()
download_request = CDownloadRequest(blob_path, mbs)
# It has been seen once a 404 error below with google drive:
# This may be caused by different servers that are not yet informed of file creation ?
storage.download(download_request)
mbs.get_bytes() == content_file
# blob must appear in folder list:
folder_content = storage.list_folder(folder_path)
assert blob_path in folder_content
assert folder_content[blob_path].path == blob_path
assert folder_content[blob_path].is_blob()
assert not folder_content[blob_path].is_folder()
finally:
storage.delete(temp_root_path)
def test_invalid_file_operations(storage):
# We'll use a temp folder for tests:
temp_root_path = miscutils.generate_test_path()
logger.info('Will use test folder: %s', temp_root_path)
# Upload 1 file into this folder:
fpath1 = temp_root_path.add('a_test_file1')
content_file1 = b'This is binary cont€nt of test file 1...'
upload_request = CUploadRequest(fpath1, MemoryByteSource(content_file1))
storage.upload(upload_request)
logger.info('Created blob: %s', fpath1)
# Create a sub_folder:
sub_folder = temp_root_path.add('sub_folder')
storage.create_folder(sub_folder)
logger.info('Check that listing content of a blob raises: %s', fpath1)
try:
storage.list_folder(fpath1)
pytest.fail('Listing a blob should raise')
except CInvalidFileTypeError as e:
assert e.path == fpath1
assert e.expected_blob is False
logger.info('Check that trying to download a folder raises: %s',
sub_folder)
mbs = MemoryByteSink()
download_request = CDownloadRequest(sub_folder, mbs)
try:
storage.download(download_request)
pytest.fail('Downloading a folder should raise')
except CInvalidFileTypeError as e:
assert e.path == sub_folder
assert e.expected_blob is True
logger.info('Check that we cannot create a folder over a blob: %s', fpath1)
try:
storage.create_folder(fpath1)
pytest.fail('Creating a folder over a blob should raise')
except CInvalidFileTypeError as e:
assert e.path == fpath1
assert e.expected_blob is False
logger.info('Check we cannot upload over an existing folder: %s',
sub_folder)
try:
content = b'some data...'
upload_request = CUploadRequest(sub_folder, MemoryByteSource(content))
storage.upload(upload_request)
pytest.fail('Uploading over a folder should raise')
except CInvalidFileTypeError as e:
assert e.path == sub_folder
assert e.expected_blob is True
logger.info('Check that content of a never existed folder is None')
c_path = CPath('/hope i did never exist (even for tests) !')
assert storage.list_folder(c_path) is None
logger.info('Check that get_file() returns None is file does not exist')
assert storage.get_file(c_path) is None
logger.info('Check that downloading a non-existing file raises')
dr = CDownloadRequest(c_path, MemoryByteSink())
try:
storage.download(dr)
pytest.fail('Downlad a non-existing blob should raise')
except CFileNotFoundError as e:
logger.info('Expected exception: %s', e)
assert e.path == dr.path
storage.delete(temp_root_path)
def test_file_operations(storage):
# We'll use a temp folder for tests:
temp_root_path = miscutils.generate_test_path()
logger.info('Will use test folder: %s', temp_root_path)
# Create a sub-folder:
sub_path = temp_root_path.add('sub_folder')
logger.info('Creating sub_folder: %s', sub_path)
assert storage.create_folder(
sub_path) is True # True because actually created
assert storage.create_folder(
sub_path) is False # False because not created
# Check back:
sub_folder = storage.get_file(sub_path)
assert sub_folder.path == sub_path
assert sub_folder.is_folder()
assert not sub_folder.is_blob()
if sub_folder.modification_time: # Not all providers have a modif time on folders
miscutils.assert_datetime_is_almost_now(sub_folder.modification_time)
# Upload 2 files into this sub-folder:
fpath1 = sub_path.add('a_test_file1')
content_file1 = b'This is binary cont€nt of test file 1...'
logger.info('Uploading blob to: %s', fpath1)
upload_request = CUploadRequest(fpath1, MemoryByteSource(content_file1))
storage.upload(upload_request)
fpath2 = sub_path.add('a_test_file2')
# Generate a quite big random data:
content_file2 = bytearray(os.urandom(500000))
logger.info('Uploading blob to: %s', fpath2)
upload_request = CUploadRequest(fpath2, MemoryByteSource(content_file2))
storage.upload(upload_request)
# Check uploaded blobs informations:
# we check file2 first because has just been uploaded / for modif time check
cblob = storage.get_file(fpath2)
assert cblob.is_blob()
assert not cblob.is_folder()
assert cblob.length == len(content_file2)
miscutils.assert_datetime_is_almost_now(cblob.modification_time)
cblob = storage.get_file(fpath1)
assert cblob.is_blob()
assert not cblob.is_folder()
assert cblob.length == len(content_file1)
# Download data, and check:
logger.info('Downloading back and checking file: %s', fpath1)
mbs = MemoryByteSink()
download_request = CDownloadRequest(fpath1, mbs)
storage.download(download_request)
assert mbs.get_bytes() == content_file1
logger.info('Downloading back and checking file ranges: %s', fpath1)
download_request.range(5, None) # starting at offset 5
storage.download(download_request)
assert mbs.get_bytes() == content_file1[5:]
if storage.provider_name() != 'rapidshare':
# rapidshare does not support such requests:
download_request.range(None, 5) # last 5 bytes
storage.download(download_request)
assert mbs.get_bytes() == content_file1[-5:]
download_request.range(2, 5)
# 5 bytes at offset 2
storage.download(download_request)
assert mbs.get_bytes() == content_file1[2:7]
logger.info('Downloading back and checking file: %s', fpath2)
download_request = CDownloadRequest(fpath2, mbs)
storage.download(download_request)
assert mbs.get_bytes() == content_file2
# Check that if we upload again, blob is replaced:
logger.info('Checking file overwrite: %s', fpath2)
content_file2 = bytearray(os.urandom(300000)) # 300kB file
upload_request = CUploadRequest(fpath2, MemoryByteSource(content_file2))
storage.upload(upload_request)
storage.download(download_request)
assert mbs.get_bytes() == content_file2
# Check that we can replace existing blob with empty content:
logger.info('Checking file overwrite with empty file: %s', fpath2)
content_file2 = b''
upload_request = CUploadRequest(fpath2, MemoryByteSource(content_file2))
storage.upload(upload_request)
storage.download(download_request)
assert mbs.get_bytes() == content_file2
# Create a sub_sub_folder:
sub_sub_path = sub_path.add('a_sub_sub_folder')
logger.info('Creating a_sub_sub_folder: %s', sub_sub_path)
created = storage.create_folder(sub_sub_path)
assert created
logger.info(
'Check uploaded blobs and sub_sub_folder all appear in folder list')
folder_content = storage.list_folder(sub_folder)
logger.info('sub_folder contains files: %r', folder_content)
# It happened once here that hubic did not list fpath1 'a_test_file1' in folder_content:
# only 2 files were present ?!
assert len(folder_content) == 3
assert fpath1 in folder_content
assert folder_content[fpath1].is_blob()
assert not folder_content[fpath1].is_folder()
assert fpath2 in folder_content
assert folder_content[fpath2].is_blob()
assert not folder_content[fpath2].is_folder()
assert sub_sub_path in folder_content
assert not folder_content[sub_sub_path].is_blob()
assert folder_content[sub_sub_path].is_folder()
logger.info('Check that list of sub_sub folder is empty: %s', sub_sub_path)
assert storage.list_folder(sub_sub_path) == {}
logger.info('Check that listing content of a blob raises: %s', fpath1)
try:
storage.list_folder(fpath1)
pytest.fail('Listing a blob should raise')
except CInvalidFileTypeError as e:
assert e.path == fpath1
assert e.expected_blob is False
logger.info('Delete file1: %s', fpath1)
assert storage.delete(fpath1) is True # We have deleted the file
assert storage.delete(fpath1) is False # We have not deleted anything
logger.info('Check file1 does not appear anymore in folder: %s',
sub_folder)
assert fpath1 not in storage.list_folder(sub_folder)
tmp = storage.get_file(fpath1)
assert tmp is None
logger.info('Delete whole test folder: %s', temp_root_path)
ret = storage.delete(temp_root_path)
assert ret is True # We have deleted at least one file
logger.info('Deleting again returns False')
ret = storage.delete(temp_root_path)
assert ret is False # We have not deleted anything
logger.info('Listing a deleted folder returns None: %s', temp_root_path)
assert storage.list_folder(temp_root_path) is None
assert storage.get_file(temp_root_path) is None
def test_download_request_bytes_range():
dr = CDownloadRequest(CPath("/foo"), MemoryByteSink())
dr.range(None, None)
assert dr.get_http_headers() == {}
dr.range(None, 100)
assert dr.get_http_headers() == {"Range": "bytes=-100"}
dr.range(10, 100)
assert dr.get_http_headers() == {"Range": "bytes=10-109"}
dr.range(100, None)
assert dr.get_http_headers() == {"Range": "bytes=100-"}
print('root folder content :', root_folder_content)
# Create a new folder :
fpath = CPath('/pcs_api_new_folder')
storage.create_folder(fpath)
# Upload a local file in this folder :
bpath = fpath.add('pcs_api_new_file')
file_content = b'this is file content...'
upload_request = CUploadRequest(
bpath, MemoryByteSource(file_content)).content_type('text/plain')
storage.upload(upload_request)
# Download back the file :
mbs = MemoryByteSink()
download_request = CDownloadRequest(bpath, mbs)
storage.download(download_request)
assert mbs.get_bytes() == file_content
# delete remote folder :
storage.delete(fpath)
# Example : we'll download a range of largest blob :
if largest_blob:
range_start = largest_blob.length / 2
range_length = min(largest_blob.length / 2, 1000000)
bs = FileByteSink('dest_file.txt')
print("Will download range from largest blob : %s to : %s" %
(largest_blob, bs))
dr = CDownloadRequest(largest_blob.path, bs).range(range_start,
range_length)
