Ejemplo n.º 1
0
 def check(self):
     from pcs_api.bytes_io import (MemoryByteSource, MemoryByteSink,
                                   StdoutProgressListener)
     from pcs_api.models import (CPath, CUploadRequest, CDownloadRequest)
     msg = "Cloud storage user_id = " + repr(self.storage.get_user_id())
     self.ctx.logger.info(msg)
     msg = "Cloud storage quota = " + repr(self.storage.get_quota())
     self.ctx.logger.info(msg)
     self.ctx.logger.info("Cloud storage is  ready")
     fpath = CPath('/test_dir')
     self.storage.create_folder(fpath)
     bpath = fpath.add("test.txt")
     file_contents_uploaded = b"Test file contents"
     upload_request = CUploadRequest(
         bpath, MemoryByteSource(file_contents_uploaded)).content_type(
             'text/plain')
     upload_request.progress_listener(StdoutProgressListener())
     self.storage.upload(upload_request)
     file_contents_downloaded = MemoryByteSink()
     download_request = CDownloadRequest(bpath, file_contents_downloaded)
     download_request.progress_listener(StdoutProgressListener())
     self.storage.download(download_request)
     self.storage.delete(fpath)
     if file_contents_uploaded != file_contents_downloaded.get_bytes():
         raise RemotePCSError
Ejemplo n.º 2
0
 def put_file(self, local_path, remote_path):
     from pcs_api.bytes_io import (FileByteSource, StdoutProgressListener)
     from pcs_api.models import (CPath, CUploadRequest)
     bpath = CPath(remote_path)
     upload_request = CUploadRequest(bpath, FileByteSource(local_path))
     upload_request.progress_listener(StdoutProgressListener())
     self.storage.upload(upload_request)
Ejemplo n.º 3
0
 def get_file(self, local_path, remote_path):
     from pcs_api.bytes_io import (FileByteSink, StdoutProgressListener)
     from pcs_api.models import (CPath, CDownloadRequest)
     bpath = CPath(remote_path)
     download_request = CDownloadRequest(bpath, FileByteSink(local_path))
     download_request.progress_listener(StdoutProgressListener())
     self.storage.download(download_request)
Ejemplo n.º 4
0
def test_cpath_as_key():
    adict = {CPath('/a'): 'file_a', CPath('/a/b'): 'file_b'}
    assert CPath('/a') in adict
    assert adict[CPath('a')] == 'file_a'
    assert CPath('/a/b') in adict
    assert adict[CPath('/a/b')] == 'file_b'
    assert CPath('/b') not in adict
Ejemplo n.º 5
0
 def get_bytes(self, remote_path):
     from pcs_api.bytes_io import (MemoryByteSink, StdoutProgressListener)
     from pcs_api.models import (CPath, CDownloadRequest)
     remote_bytes = MemoryByteSink()
     bpath = CPath(remote_path)
     download_request = CDownloadRequest(bpath, remote_bytes)
     download_request.progress_listener(StdoutProgressListener())
     self.storage.download(download_request)
     return remote_bytes.get_bytes()
Ejemplo n.º 6
0
 def put_bytes(self, local_bytes, remote_path):
     from pcs_api.bytes_io import (MemoryByteSource, StdoutProgressListener)
     from pcs_api.models import (CPath, CUploadRequest)
     byte_source = MemoryByteSource(local_bytes)
     bpath = CPath(remote_path)
     upload_request = CUploadRequest(bpath,
                                     byte_source).content_type('text/plain')
     upload_request.progress_listener(StdoutProgressListener())
     self.storage.upload(upload_request)
Ejemplo n.º 7
0
def test_download_request_bytes_range():
    dr = CDownloadRequest(CPath('/foo'), MemoryByteSink())
    dr.range(None, None)
    assert dr.get_http_headers() == {}
    dr.range(None, 100)
    assert dr.get_http_headers() == { 'Range': 'bytes=-100'}
    dr.range(10, 100)
    assert dr.get_http_headers() == { 'Range': 'bytes=10-109'}
    dr.range(100, None)
    assert dr.get_http_headers() == { 'Range': 'bytes=100-'}
Ejemplo n.º 8
0
def generate_test_path(c_root_path=None):
    """Generate a temp folder for tests."""
    temp_pathname = ''.join(
        random.choice(string.ascii_uppercase + string.digits)
        for x in range(6))
    if c_root_path is None or c_root_path.is_root():
        temp_path = CPath(TEST_FOLDER_PREFIX + temp_pathname)
    else:
        temp_path = c_root_path.add(temp_pathname)
    return temp_path
Ejemplo n.º 9
0
def test_invalid_cpath():
    for pathname in ['\\no anti-slash is allowed',
                     'This is an inv\\u001Flid pathname !',
                     'This is an \t invalid pathname !'
                     'This/ is/an invalid pathname !',
                     'This/is /also an invalid pathname !',
                     ' bad', 'bad ', '\u00a0bad', 'bad\u00a0']:
        with pytest.raises(ValueError) as e:
            logger.info("Checking CPath is invalid: %s", pathname)
            p = CPath(pathname)
        logger.info("CPath validation failed as expected: %s", e)
Ejemplo n.º 10
0
def test_upload_request_progress_listener():
    mbs = MemoryByteSource(b'content')
    ur = CUploadRequest(CPath('/foo'), mbs)
    assert ur.byte_source() is mbs

    # Now if we decorate:
    pl = StdoutProgressListener()
    ur.progress_listener(pl)
    istream = ur.byte_source().open_stream()
    data = istream.read(1)
    assert pl.current == 1
Ejemplo n.º 11
0
def test_download_request_progress_listener():
    mbs = MemoryByteSink()
    dr = CDownloadRequest(CPath('/foo'), mbs)
    assert dr.byte_sink() is mbs

    # Now if we decorate:
    pl = StdoutProgressListener()
    dr.progress_listener(pl)
    os = dr.byte_sink().open_stream()
    os.write(b'a')
    assert pl.current == 1
Ejemplo n.º 12
0
def upload_and_check_random_files(storage, root_path):
    rnd = random.Random()  # not shared (in case multi-thread)
    tmp_path = CPath(root_path.path_name())
    for i in range(0, rnd.randint(1, 5)):
        c_path = miscutils.generate_test_path(tmp_path)

        if rnd.randint(0, 2) == 0:
            # Create folder:
            storage.create_folder(c_path)
            # And sometimes go inside:
            if rnd.randint(0, 2) == 0:
                tmp_path = c_path
        else:
            # Create file (deterministic content for later check):
            rnd.seed(c_path)
            file_size = rnd.randint(0, 1000) * rnd.randint(
                0, 1000)  # prefer small files
            data = miscutils.generate_random_bytearray(file_size, rnd)
            ur = CUploadRequest(c_path, MemoryByteSource(data))
            storage.upload(ur)

    # Check blob files content:
    all_blobs = recursively_list_blobs(storage, root_path)
    logger.info('All uploaded blobs = %s', all_blobs)
    for blob in all_blobs:
        rnd.seed(blob.path)
        file_size = rnd.randint(0, 1000) * rnd.randint(
            0, 1000)  # same formula as above
        assert file_size == blob.length
        expected_data = miscutils.generate_random_bytearray(file_size, rnd)
        bsink = MemoryByteSink()
        dr = CDownloadRequest(blob.path, bsink)
        storage.download(dr)
        downloaded_data = bsink.get_bytes()
        # In order to track strange download errors:
        if expected_data != downloaded_data:
            logger.error(
                'Downloaded data differ (%d / %d bytes): will download again for disgnostic',
                len(expected_data), len(downloaded_data))
            bsink2 = MemoryByteSink()
            dr2 = CDownloadRequest(blob.path, bsink2)
            storage.download(dr2)
            downloaded_data2 = bsink2.get_bytes()
            logger.error(
                'Second download: (%d bytes). Data are the same now?=%r / Same as first download?=%r',
                len(downloaded_data2), expected_data == downloaded_data2,
                downloaded_data == downloaded_data2)
        # py.test cannot really compare bytearray and bytes, so we convert for nicer display:
        assert bytes(expected_data) == downloaded_data

    # Delete all files:
    storage.delete(root_path)
Ejemplo n.º 13
0
    print(
        'Refer to documentation and class UserCredentialsFileRepository to setup pcs_api for a quick test'
    )
    exit(1)
user_credentials_repo = UserCredentialsFileRepository(user_creds_pathname)

storage = StorageFacade.for_provider(cli_args.provider_name) \
                       .app_info_repository(apps_repo, cli_args.app_name) \
                       .user_credentials_repository(user_credentials_repo, cli_args.user_id) \
                       .build()

print("user_id = ", storage.get_user_id())
print("quota = ", storage.get_quota())

# Recursively list all regular files (blobs in folders) :
folders_to_process = [CPath('/')]
largest_blob = None
while folders_to_process:
    f = folders_to_process.pop()
    # list folder :
    print("\nContent of %s :" % f)
    content = storage.list_folder(f)
    blobs = [f for f in content.values() if f.is_blob()]
    folders = [f for f in content.values() if f.is_folder()]
    # print blobs of this folder :
    for b in blobs:
        print("  ", b)
        if (not largest_blob) or b.length > largest_blob.length:
            largest_blob = b
    for f in folders:
        print("  ", f)
Ejemplo n.º 14
0
        print("  ", b)
        if (not largest_blob) or b.length > largest_blob.length:
            largest_blob = b
    for f in folders:
        print("  ", f)
    # Add folders to list :
    folders_to_process += folders

# Create a folder and upload a test file :
root_folder_content = storage.list_root_folder()
# root_folder_content is a dict
# keys are files paths, values are CFolder or CBlob providing details (length, content-type...)
print('root folder content :', root_folder_content)

# Create a new folder :
fpath = CPath('/pcs_api_new_folder')
storage.create_folder(fpath)

# Upload a local file in this folder :
bpath = fpath.add('pcs_api_new_file')
file_content = b'this is file content...'
upload_request = CUploadRequest(bpath, MemoryByteSource(file_content)).content_type('text/plain')
storage.upload(upload_request)

# Download back the file :
mbs = MemoryByteSink()
download_request = CDownloadRequest(bpath, mbs)
storage.download(download_request)
assert mbs.get_bytes() == file_content

# delete remote folder :
Ejemplo n.º 15
0
def test_invalid_file_operations(storage):
    # We'll use a temp folder for tests:
    temp_root_path = miscutils.generate_test_path()
    logger.info('Will use test folder: %s', temp_root_path)

    # Upload 1 file into this folder:
    fpath1 = temp_root_path.add('a_test_file1')
    content_file1 = b'This is binary cont€nt of test file 1...'
    upload_request = CUploadRequest(fpath1, MemoryByteSource(content_file1))
    storage.upload(upload_request)
    logger.info('Created blob: %s', fpath1)

    # Create a sub_folder:
    sub_folder = temp_root_path.add('sub_folder')
    storage.create_folder(sub_folder)

    logger.info('Check that listing content of a blob raises: %s', fpath1)
    try:
        storage.list_folder(fpath1)
        pytest.fail('Listing a blob should raise')
    except CInvalidFileTypeError as e:
        assert e.path == fpath1
        assert e.expected_blob is False

    logger.info('Check that trying to download a folder raises: %s',
                sub_folder)
    mbs = MemoryByteSink()
    download_request = CDownloadRequest(sub_folder, mbs)
    try:
        storage.download(download_request)
        pytest.fail('Downloading a folder should raise')
    except CInvalidFileTypeError as e:
        assert e.path == sub_folder
        assert e.expected_blob is True

    logger.info('Check that we cannot create a folder over a blob: %s', fpath1)
    try:
        storage.create_folder(fpath1)
        pytest.fail('Creating a folder over a blob should raise')
    except CInvalidFileTypeError as e:
        assert e.path == fpath1
        assert e.expected_blob is False

    logger.info('Check we cannot upload over an existing folder: %s',
                sub_folder)
    try:
        content = b'some data...'
        upload_request = CUploadRequest(sub_folder, MemoryByteSource(content))
        storage.upload(upload_request)
        pytest.fail('Uploading over a folder should raise')
    except CInvalidFileTypeError as e:
        assert e.path == sub_folder
        assert e.expected_blob is True

    logger.info('Check that content of a never existed folder is None')
    c_path = CPath('/hope i did never exist (even for tests) !')
    assert storage.list_folder(c_path) is None
    logger.info('Check that get_file() returns None is file does not exist')
    assert storage.get_file(c_path) is None

    logger.info('Check that downloading a non-existing file raises')
    dr = CDownloadRequest(c_path, MemoryByteSink())
    try:
        storage.download(dr)
        pytest.fail('Downlad a non-existing blob should raise')
    except CFileNotFoundError as e:
        logger.info('Expected exception: %s', e)
        assert e.path == dr.path

    storage.delete(temp_root_path)
Ejemplo n.º 16
0
def test_cpath():
    p = CPath('/foo//bar€/')
    assert p.path_name() == '/foo/bar€'
    assert p.url_encoded() == '/foo/bar%E2%82%AC'
    assert p.base_name() == 'bar€'
    assert p.parent() == CPath('/foo')
    assert p.add('a,file...') == CPath('/foo/bar€/a,file...')
    assert p.add('/a,file...') == CPath('/foo/bar€/a,file...')
    assert p.add('a,file.../') == CPath('/foo/bar€/a,file...')
    assert p.add('/several//folders/he re/') == CPath('/foo/bar€/several/folders/he re')
    assert p.is_root() is False
    assert p.parent().is_root() is False
    root = p.parent().parent()
    assert root.is_root() is True
    assert root.parent().is_root() is True
    assert root == CPath('/')
    assert root == CPath('')
    assert root.base_name() == ''
    assert root.split() == []
    assert CPath('').split() == []
    assert CPath('/a').split() == ['a']
    assert CPath('/alpha/"beta').split() == ['alpha', '"beta']
Ejemplo n.º 17
0
def test_cpath_url_encoded():
    assert CPath('/a +%b/c').url_encoded() == '/a%20%2B%25b/c'
    assert CPath('/a:b').url_encoded() == '/a%3Ab'
    assert CPath('/€').url_encoded() == '/%E2%82%AC'
Ejemplo n.º 18
0
 def mkdir(self, remote_path):
     from pcs_api.models import CPath
     fpath = CPath(remote_path)
     self.storage.create_folder(fpath)
Ejemplo n.º 19
0
def test_cpath():
    p = CPath("/foo//bar€/")
    assert p.path_name() == "/foo/bar€"
    assert p.url_encoded() == "/foo/bar%E2%82%AC"
    assert p.base_name() == "bar€"
    assert p.parent() == CPath("/foo")
    assert p.add("a,file...") == CPath("/foo/bar€/a,file...")
    assert p.add("/a,file...") == CPath("/foo/bar€/a,file...")
    assert p.add("a,file.../") == CPath("/foo/bar€/a,file...")
    assert p.add("/several//folders/he re/") == CPath("/foo/bar€/several/folders/he re")
    assert p.is_root() is False
    assert p.parent().is_root() is False
    root = p.parent().parent()
    assert root.is_root() is True
    assert root.parent().is_root() is True
    assert root == CPath("/")
    assert root == CPath("")
    assert root.base_name() == ""
    assert root.split() == []
    assert CPath("").split() == []
    assert CPath("/a").split() == ["a"]
    assert CPath('/alpha/"beta').split() == ["alpha", '"beta']