def check(self):
from pcs_api.bytes_io import (MemoryByteSource, MemoryByteSink,
StdoutProgressListener)
from pcs_api.models import (CPath, CUploadRequest, CDownloadRequest)
msg = "Cloud storage user_id = " + repr(self.storage.get_user_id())
self.ctx.logger.info(msg)
msg = "Cloud storage quota = " + repr(self.storage.get_quota())
self.ctx.logger.info(msg)
self.ctx.logger.info("Cloud storage is ready")
fpath = CPath('/test_dir')
self.storage.create_folder(fpath)
bpath = fpath.add("test.txt")
file_contents_uploaded = b"Test file contents"
upload_request = CUploadRequest(
bpath, MemoryByteSource(file_contents_uploaded)).content_type(
'text/plain')
upload_request.progress_listener(StdoutProgressListener())
self.storage.upload(upload_request)
file_contents_downloaded = MemoryByteSink()
download_request = CDownloadRequest(bpath, file_contents_downloaded)
download_request.progress_listener(StdoutProgressListener())
self.storage.download(download_request)
self.storage.delete(fpath)
if file_contents_uploaded != file_contents_downloaded.get_bytes():
raise RemotePCSError
def put_file(self, local_path, remote_path):
from pcs_api.bytes_io import (FileByteSource, StdoutProgressListener)
from pcs_api.models import (CPath, CUploadRequest)
bpath = CPath(remote_path)
upload_request = CUploadRequest(bpath, FileByteSource(local_path))
upload_request.progress_listener(StdoutProgressListener())
self.storage.upload(upload_request)
def get_file(self, local_path, remote_path):
from pcs_api.bytes_io import (FileByteSink, StdoutProgressListener)
from pcs_api.models import (CPath, CDownloadRequest)
bpath = CPath(remote_path)
download_request = CDownloadRequest(bpath, FileByteSink(local_path))
download_request.progress_listener(StdoutProgressListener())
self.storage.download(download_request)
def test_cpath_as_key():
adict = {CPath('/a'): 'file_a', CPath('/a/b'): 'file_b'}
assert CPath('/a') in adict
assert adict[CPath('a')] == 'file_a'
assert CPath('/a/b') in adict
assert adict[CPath('/a/b')] == 'file_b'
assert CPath('/b') not in adict
def get_bytes(self, remote_path):
from pcs_api.bytes_io import (MemoryByteSink, StdoutProgressListener)
from pcs_api.models import (CPath, CDownloadRequest)
remote_bytes = MemoryByteSink()
bpath = CPath(remote_path)
download_request = CDownloadRequest(bpath, remote_bytes)
download_request.progress_listener(StdoutProgressListener())
self.storage.download(download_request)
return remote_bytes.get_bytes()
def put_bytes(self, local_bytes, remote_path):
from pcs_api.bytes_io import (MemoryByteSource, StdoutProgressListener)
from pcs_api.models import (CPath, CUploadRequest)
byte_source = MemoryByteSource(local_bytes)
bpath = CPath(remote_path)
upload_request = CUploadRequest(bpath,
byte_source).content_type('text/plain')
upload_request.progress_listener(StdoutProgressListener())
self.storage.upload(upload_request)
def test_download_request_bytes_range():
dr = CDownloadRequest(CPath('/foo'), MemoryByteSink())
dr.range(None, None)
assert dr.get_http_headers() == {}
dr.range(None, 100)
assert dr.get_http_headers() == { 'Range': 'bytes=-100'}
dr.range(10, 100)
assert dr.get_http_headers() == { 'Range': 'bytes=10-109'}
dr.range(100, None)
assert dr.get_http_headers() == { 'Range': 'bytes=100-'}
def generate_test_path(c_root_path=None):
"""Generate a temp folder for tests."""
temp_pathname = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(6))
if c_root_path is None or c_root_path.is_root():
temp_path = CPath(TEST_FOLDER_PREFIX + temp_pathname)
else:
temp_path = c_root_path.add(temp_pathname)
return temp_path
def test_invalid_cpath():
for pathname in ['\\no anti-slash is allowed',
'This is an inv\\u001Flid pathname !',
'This is an \t invalid pathname !'
'This/ is/an invalid pathname !',
'This/is /also an invalid pathname !',
' bad', 'bad ', '\u00a0bad', 'bad\u00a0']:
with pytest.raises(ValueError) as e:
logger.info("Checking CPath is invalid: %s", pathname)
p = CPath(pathname)
logger.info("CPath validation failed as expected: %s", e)
def test_upload_request_progress_listener():
mbs = MemoryByteSource(b'content')
ur = CUploadRequest(CPath('/foo'), mbs)
assert ur.byte_source() is mbs
# Now if we decorate:
pl = StdoutProgressListener()
ur.progress_listener(pl)
istream = ur.byte_source().open_stream()
data = istream.read(1)
assert pl.current == 1
def test_download_request_progress_listener():
mbs = MemoryByteSink()
dr = CDownloadRequest(CPath('/foo'), mbs)
assert dr.byte_sink() is mbs
# Now if we decorate:
pl = StdoutProgressListener()
dr.progress_listener(pl)
os = dr.byte_sink().open_stream()
os.write(b'a')
assert pl.current == 1
def upload_and_check_random_files(storage, root_path):
rnd = random.Random() # not shared (in case multi-thread)
tmp_path = CPath(root_path.path_name())
for i in range(0, rnd.randint(1, 5)):
c_path = miscutils.generate_test_path(tmp_path)
if rnd.randint(0, 2) == 0:
# Create folder:
storage.create_folder(c_path)
# And sometimes go inside:
if rnd.randint(0, 2) == 0:
tmp_path = c_path
else:
# Create file (deterministic content for later check):
rnd.seed(c_path)
file_size = rnd.randint(0, 1000) * rnd.randint(
0, 1000) # prefer small files
data = miscutils.generate_random_bytearray(file_size, rnd)
ur = CUploadRequest(c_path, MemoryByteSource(data))
storage.upload(ur)
# Check blob files content:
all_blobs = recursively_list_blobs(storage, root_path)
logger.info('All uploaded blobs = %s', all_blobs)
for blob in all_blobs:
rnd.seed(blob.path)
file_size = rnd.randint(0, 1000) * rnd.randint(
0, 1000) # same formula as above
assert file_size == blob.length
expected_data = miscutils.generate_random_bytearray(file_size, rnd)
bsink = MemoryByteSink()
dr = CDownloadRequest(blob.path, bsink)
storage.download(dr)
downloaded_data = bsink.get_bytes()
# In order to track strange download errors:
if expected_data != downloaded_data:
logger.error(
'Downloaded data differ (%d / %d bytes): will download again for disgnostic',
len(expected_data), len(downloaded_data))
bsink2 = MemoryByteSink()
dr2 = CDownloadRequest(blob.path, bsink2)
storage.download(dr2)
downloaded_data2 = bsink2.get_bytes()
logger.error(
'Second download: (%d bytes). Data are the same now?=%r / Same as first download?=%r',
len(downloaded_data2), expected_data == downloaded_data2,
downloaded_data == downloaded_data2)
# py.test cannot really compare bytearray and bytes, so we convert for nicer display:
assert bytes(expected_data) == downloaded_data
# Delete all files:
storage.delete(root_path)
print(
'Refer to documentation and class UserCredentialsFileRepository to setup pcs_api for a quick test'
)
exit(1)
user_credentials_repo = UserCredentialsFileRepository(user_creds_pathname)
storage = StorageFacade.for_provider(cli_args.provider_name) \
.app_info_repository(apps_repo, cli_args.app_name) \
.user_credentials_repository(user_credentials_repo, cli_args.user_id) \
.build()
print("user_id = ", storage.get_user_id())
print("quota = ", storage.get_quota())
# Recursively list all regular files (blobs in folders) :
folders_to_process = [CPath('/')]
largest_blob = None
while folders_to_process:
f = folders_to_process.pop()
# list folder :
print("\nContent of %s :" % f)
content = storage.list_folder(f)
blobs = [f for f in content.values() if f.is_blob()]
folders = [f for f in content.values() if f.is_folder()]
# print blobs of this folder :
for b in blobs:
print(" ", b)
if (not largest_blob) or b.length > largest_blob.length:
largest_blob = b
for f in folders:
print(" ", f)
print(" ", b)
if (not largest_blob) or b.length > largest_blob.length:
largest_blob = b
for f in folders:
print(" ", f)
# Add folders to list :
folders_to_process += folders
# Create a folder and upload a test file :
root_folder_content = storage.list_root_folder()
# root_folder_content is a dict
# keys are files paths, values are CFolder or CBlob providing details (length, content-type...)
print('root folder content :', root_folder_content)
# Create a new folder :
fpath = CPath('/pcs_api_new_folder')
storage.create_folder(fpath)
# Upload a local file in this folder :
bpath = fpath.add('pcs_api_new_file')
file_content = b'this is file content...'
upload_request = CUploadRequest(bpath, MemoryByteSource(file_content)).content_type('text/plain')
storage.upload(upload_request)
# Download back the file :
mbs = MemoryByteSink()
download_request = CDownloadRequest(bpath, mbs)
storage.download(download_request)
assert mbs.get_bytes() == file_content
# delete remote folder :
def test_invalid_file_operations(storage):
# We'll use a temp folder for tests:
temp_root_path = miscutils.generate_test_path()
logger.info('Will use test folder: %s', temp_root_path)
# Upload 1 file into this folder:
fpath1 = temp_root_path.add('a_test_file1')
content_file1 = b'This is binary cont€nt of test file 1...'
upload_request = CUploadRequest(fpath1, MemoryByteSource(content_file1))
storage.upload(upload_request)
logger.info('Created blob: %s', fpath1)
# Create a sub_folder:
sub_folder = temp_root_path.add('sub_folder')
storage.create_folder(sub_folder)
logger.info('Check that listing content of a blob raises: %s', fpath1)
try:
storage.list_folder(fpath1)
pytest.fail('Listing a blob should raise')
except CInvalidFileTypeError as e:
assert e.path == fpath1
assert e.expected_blob is False
logger.info('Check that trying to download a folder raises: %s',
sub_folder)
mbs = MemoryByteSink()
download_request = CDownloadRequest(sub_folder, mbs)
try:
storage.download(download_request)
pytest.fail('Downloading a folder should raise')
except CInvalidFileTypeError as e:
assert e.path == sub_folder
assert e.expected_blob is True
logger.info('Check that we cannot create a folder over a blob: %s', fpath1)
try:
storage.create_folder(fpath1)
pytest.fail('Creating a folder over a blob should raise')
except CInvalidFileTypeError as e:
assert e.path == fpath1
assert e.expected_blob is False
logger.info('Check we cannot upload over an existing folder: %s',
sub_folder)
try:
content = b'some data...'
upload_request = CUploadRequest(sub_folder, MemoryByteSource(content))
storage.upload(upload_request)
pytest.fail('Uploading over a folder should raise')
except CInvalidFileTypeError as e:
assert e.path == sub_folder
assert e.expected_blob is True
logger.info('Check that content of a never existed folder is None')
c_path = CPath('/hope i did never exist (even for tests) !')
assert storage.list_folder(c_path) is None
logger.info('Check that get_file() returns None is file does not exist')
assert storage.get_file(c_path) is None
logger.info('Check that downloading a non-existing file raises')
dr = CDownloadRequest(c_path, MemoryByteSink())
try:
storage.download(dr)
pytest.fail('Downlad a non-existing blob should raise')
except CFileNotFoundError as e:
logger.info('Expected exception: %s', e)
assert e.path == dr.path
storage.delete(temp_root_path)
def test_cpath():
p = CPath('/foo//bar€/')
assert p.path_name() == '/foo/bar€'
assert p.url_encoded() == '/foo/bar%E2%82%AC'
assert p.base_name() == 'bar€'
assert p.parent() == CPath('/foo')
assert p.add('a,file...') == CPath('/foo/bar€/a,file...')
assert p.add('/a,file...') == CPath('/foo/bar€/a,file...')
assert p.add('a,file.../') == CPath('/foo/bar€/a,file...')
assert p.add('/several//folders/he re/') == CPath('/foo/bar€/several/folders/he re')
assert p.is_root() is False
assert p.parent().is_root() is False
root = p.parent().parent()
assert root.is_root() is True
assert root.parent().is_root() is True
assert root == CPath('/')
assert root == CPath('')
assert root.base_name() == ''
assert root.split() == []
assert CPath('').split() == []
assert CPath('/a').split() == ['a']
assert CPath('/alpha/"beta').split() == ['alpha', '"beta']
def test_cpath_url_encoded():
assert CPath('/a +%b/c').url_encoded() == '/a%20%2B%25b/c'
assert CPath('/a:b').url_encoded() == '/a%3Ab'
assert CPath('/€').url_encoded() == '/%E2%82%AC'
def mkdir(self, remote_path):
from pcs_api.models import CPath
fpath = CPath(remote_path)
self.storage.create_folder(fpath)
def test_cpath():
p = CPath("/foo//bar€/")
assert p.path_name() == "/foo/bar€"
assert p.url_encoded() == "/foo/bar%E2%82%AC"
assert p.base_name() == "bar€"
assert p.parent() == CPath("/foo")
assert p.add("a,file...") == CPath("/foo/bar€/a,file...")
assert p.add("/a,file...") == CPath("/foo/bar€/a,file...")
assert p.add("a,file.../") == CPath("/foo/bar€/a,file...")
assert p.add("/several//folders/he re/") == CPath("/foo/bar€/several/folders/he re")
assert p.is_root() is False
assert p.parent().is_root() is False
root = p.parent().parent()
assert root.is_root() is True
assert root.parent().is_root() is True
assert root == CPath("/")
assert root == CPath("")
assert root.base_name() == ""
assert root.split() == []
assert CPath("").split() == []
assert CPath("/a").split() == ["a"]
assert CPath('/alpha/"beta').split() == ["alpha", '"beta']