def test_download_object_success_not_found(self):
CloudFilesMockRawResponse.type = 'NOT_FOUND'
container = Container(name='foo_bar_container', extra={}, driver=self)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=CloudFilesStorageDriver)
destination_path = os.path.abspath(__file__) + '.temp'
try:
self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
except ObjectDoesNotExistError:
pass
else:
self.fail('Object does not exist but an exception was not thrown')
def test_download_object_not_found(self):
self.mock_response_klass.type = "not_found"
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
destination_path = os.path.abspath(__file__) + ".temp"
self.assertRaises(
ObjectDoesNotExistError,
self.driver.download_object,
obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True,
)
def test_upload_object_nonexistent_file(self):
def dummy_content_type(name):
return 'application/zip', None
old_func = libcloud.utils.guess_file_mime_type
libcloud.utils.guess_file_mime_type = dummy_content_type
file_path = os.path.abspath(__file__ + '.inexistent')
container = Container(name='fbc', extra={}, driver=self)
object_name = 'ftu'
try:
self.driver.upload_object(
file_path=file_path,
container=container,
object_name=object_name)
except OSError:
pass
else:
self.fail('Inesitent but an exception was not thrown')
finally:
libcloud.utils.guess_file_mime_type = old_func
def test_upload_small_object_via_stream(self):
if self.driver.supports_s3_multipart_upload:
self.mock_raw_response_klass.type = 'MULTIPART'
self.mock_response_klass.type = 'MULTIPART'
else:
self.mock_raw_response_klass.type = None
self.mock_response_klass.type = None
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
object_name = 'foo_test_stream_data'
iterator = DummyIterator(data=['2', '3', '5'])
extra = {'content_type': 'text/plain'}
obj = self.driver.upload_object_via_stream(container=container,
object_name=object_name,
iterator=iterator,
extra=extra)
self.assertEqual(obj.name, object_name)
self.assertEqual(obj.size, 3)
def test_list_container_objects_with_prefix(self):
self.mock_response_klass.type = None
AzureBlobsStorageDriver.RESPONSES_PER_REQUEST = 2
container = Container(name="test_container", extra={}, driver=self.driver)
objects = self.driver.list_container_objects(
container=container, prefix="test_prefix"
)
self.assertEqual(len(objects), 4)
obj = objects[1]
self.assertEqual(obj.name, "object2.txt")
self.assertEqual(obj.hash, "0x8CFB90F1BA8CD8F")
self.assertEqual(obj.size, 1048576)
self.assertEqual(obj.container.name, "test_container")
self.assertTrue("meta1" in obj.meta_data)
self.assertTrue("meta2" in obj.meta_data)
self.assertTrue("last_modified" in obj.extra)
self.assertTrue("content_type" in obj.extra)
self.assertTrue("content_encoding" in obj.extra)
self.assertTrue("content_language" in obj.extra)
def test_upload_small_block_object_success_with_lease(self):
self.mock_response_klass.use_param = "comp"
file_path = os.path.abspath(__file__)
file_size = os.stat(file_path).st_size
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
object_name = "foo_test_upload"
extra = {"meta_data": {"some-value": "foobar"}}
obj = self.driver.upload_object(
file_path=file_path,
container=container,
object_name=object_name,
extra=extra,
verify_hash=False,
ex_use_lease=True,
)
self.assertEqual(obj.name, "foo_test_upload")
self.assertEqual(obj.size, file_size)
self.assertTrue("some-value" in obj.meta_data)
self.mock_response_klass.use_param = None
def test__upload_object_manifest_wrong_hash(self):
fake_response = type('CloudFilesResponse', (),
{'headers': {
'etag': '0000000'
}})
_request = self.driver.connection.request
mocked_request = mock.Mock(return_value=fake_response)
self.driver.connection.request = mocked_request
container = Container(name='foo_bar_container', extra={}, driver=self)
object_name = "test_object"
try:
self.driver._upload_object_manifest(container, object_name)
except ObjectHashMismatchError:
pass
else:
self.fail('Exception was not thrown')
finally:
self.driver.connection.request = _request
def test_upload_object_invalid_md5(self):
# Invalid md5 is returned by azure
self.mock_response_klass.type = "INVALID_HASH"
container = Container(name="foo_bar_container",
extra={},
driver=self.driver)
object_name = "foo_test_upload"
file_path = os.path.abspath(__file__)
try:
self.driver.upload_object(
file_path=file_path,
container=container,
object_name=object_name,
verify_hash=True,
)
except ObjectHashMismatchError:
pass
else:
self.fail(
"Invalid hash was returned but an exception was not thrown")
def test_download_object_success(self):
container = Container(name="foo_bar_container",
extra={},
driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
destination_path = os.path.abspath(__file__) + ".temp"
result = self.driver.download_object(
obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True,
)
self.assertTrue(result)
def test_upload_object_via_stream_no_content_type(self):
def no_content_type(name):
return None, None
old_func = libcloud.storage.drivers.atmos.guess_file_mime_type
libcloud.storage.drivers.atmos.guess_file_mime_type = no_content_type
container = Container(name='fbc', extra={}, driver=self)
object_name = 'ftsdct'
iterator = DummyIterator(data=['2', '3', '5'])
try:
self.driver.upload_object_via_stream(container=container,
object_name=object_name,
iterator=iterator)
except AttributeError:
pass
else:
self.fail('File content type not provided'
' but an exception was not thrown')
finally:
libcloud.storage.drivers.atmos.guess_file_mime_type = old_func
def test_upload_object_success(self):
def upload_file(self, response, file_path, chunked=False,
calculate_hash=True):
return True, '0cc175b9c0f1b6a831c399e269772661', 1000
old_func = S3StorageDriver._upload_file
S3StorageDriver._upload_file = upload_file
file_path = os.path.abspath(__file__)
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
object_name = 'foo_test_upload'
extra = {'meta_data': {'some-value': 'foobar'}}
obj = self.driver.upload_object(file_path=file_path,
container=container,
object_name=object_name,
extra=extra,
verify_hash=True)
self.assertEqual(obj.name, 'foo_test_upload')
self.assertEqual(obj.size, 1000)
self.assertTrue('some-value' in obj.meta_data)
S3StorageDriver._upload_file = old_func
def test_upload_object_invalid_hash1(self):
def upload_file(
self,
object_name=None,
content_type=None,
request_path=None,
request_method=None,
headers=None,
file_path=None,
stream=None,
):
return {
"response": make_response(200, headers={"etag": "2345"}),
"bytes_transferred": 1000,
"data_hash": "hash343hhash89h932439jsaa89",
}
self.mock_response_klass.type = "INVALID_HASH1"
old_func = self.driver_type._upload_object
self.driver_type._upload_object = upload_file
file_path = os.path.abspath(__file__)
container = Container(name="foo_bar_container",
extra={},
driver=self.driver)
object_name = "foo_test_upload"
try:
self.driver.upload_object(
file_path=file_path,
container=container,
object_name=object_name,
verify_hash=True,
)
except ObjectHashMismatchError:
pass
else:
self.fail(
"Invalid hash was returned but an exception was not thrown")
finally:
self.driver_type._upload_object = old_func
def test_upload_object_invalid_hash2(self):
# Invalid hash is detected when comparing hash provided in the response
# ETag header
def upload_file(self,
object_name=None,
content_type=None,
request_path=None,
request_method=None,
headers=None,
file_path=None,
stream=None):
return {
'response': make_response(200,
headers={'etag': 'woopwoopwoop'}),
'bytes_transferred': 1000,
'data_hash': '0cc175b9c0f1b6a831c399e269772661'
}
self.mock_response_klass.type = 'INVALID_HASH2'
old_func = self.driver_type._upload_object
self.driver_type._upload_object = upload_file
file_path = os.path.abspath(__file__)
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
object_name = 'foo_test_upload'
try:
self.driver.upload_object(file_path=file_path,
container=container,
object_name=object_name,
verify_hash=True)
except ObjectHashMismatchError:
pass
else:
self.fail(
'Invalid hash was returned but an exception was not thrown')
finally:
self.driver_type._upload_object = old_func
def test_download_object_as_stream_data_is_not_buffered_in_memory(self):
# Test case which verifies that response.response attribute is not accessed
# and as such, whole body response is not buffered into RAM
# If content is consumed and response.content attribute accessed exception
# will be thrown and test will fail
mock_response = Mock(name='mock response')
mock_response.headers = {}
mock_response.status = 200
msg1 = '"response" attribute was accessed but it shouldn\'t have been'
msg2 = '"content" attribute was accessed but it shouldn\'t have been'
type(mock_response).response = PropertyMock(
name='mock response attribute', side_effect=Exception(msg1))
type(mock_response).content = PropertyMock(
name='mock content attribute', side_effect=Exception(msg2))
mock_response.iter_content.return_value = StringIO('a' * 1000)
self.driver.connection.request = Mock()
self.driver.connection.request.return_value = mock_response
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object_NO_BUFFER',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type)
destination_path = self._file_path
result = self.driver.download_object_as_stream(obj=obj)
result = exhaust_iterator(result)
if PY3:
result = result.decode('utf-8')
self.assertEqual(result, 'a' * 1000)
def test_download_object_invalid_file_size(self):
self.mock_response_klass.type = "INVALID_SIZE"
container = Container(name="foo_bar_container",
extra={},
driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
destination_path = os.path.abspath(__file__) + ".temp"
result = self.driver.download_object(
obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True,
)
self.assertFalse(result)
def test_upload_small_block_object_success_with_lease(self):
self.mock_response_klass.use_param = 'comp'
file_path = os.path.abspath(__file__)
file_size = os.stat(file_path).st_size
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
object_name = 'foo_test_upload'
extra = {'meta_data': {'some-value': 'foobar'}}
obj = self.driver.upload_object(file_path=file_path,
container=container,
object_name=object_name,
extra=extra,
verify_hash=False,
ex_blob_type='BlockBlob',
ex_use_lease=True)
self.assertEqual(obj.name, 'foo_test_upload')
self.assertEqual(obj.size, file_size)
self.assertTrue('some-value' in obj.meta_data)
self.mock_response_klass.use_param = None
def test_list_container_objects_success(self):
self.mock_response_klass.type = None
AzureBlobsStorageDriver.RESPONSES_PER_REQUEST = 2
container = Container(name='test_container',
extra={},
driver=self.driver)
objects = self.driver.list_container_objects(container=container)
self.assertEqual(len(objects), 4)
obj = objects[1]
self.assertEqual(obj.name, 'object2.txt')
self.assertEqual(obj.hash, '0x8CFB90F1BA8CD8F')
self.assertEqual(obj.size, 1048576)
self.assertEqual(obj.container.name, 'test_container')
self.assertTrue('meta1' in obj.meta_data)
self.assertTrue('meta2' in obj.meta_data)
self.assertTrue('last_modified' in obj.extra)
self.assertTrue('content_type' in obj.extra)
self.assertTrue('content_encoding' in obj.extra)
self.assertTrue('content_language' in obj.extra)
def test_download_object_invalid_file_already_exists(self):
self.mock_response_klass.type = 'INVALID_SIZE'
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type)
destination_path = os.path.abspath(__file__)
try:
self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
except LibcloudError:
pass
else:
self.fail('Exception was not thrown')
def _response_to_container(self, container_name, response):
"""
Converts a HTTP response to a container instance
:param container_name: Name of the container
:type container_name: ``str``
:param response: HTTP Response
:type node: L{}
:return: A container instance
:rtype: :class:`Container`
"""
headers = response.headers
scheme = "https" if self.secure else "http"
extra = {
"url":
"%s://%s%s" %
(scheme, response.connection.host, response.connection.action),
"etag":
headers["etag"],
"last_modified":
headers["last-modified"],
"lease": {
"status": headers.get("x-ms-lease-status", None),
"state": headers.get("x-ms-lease-state", None),
"duration": headers.get("x-ms-lease-duration", None),
},
"meta_data": {},
}
for key, value in response.headers.items():
if key.startswith("x-ms-meta-"):
key = key.split("x-ms-meta-")[1]
extra["meta_data"][key] = value
return Container(name=container_name, extra=extra, driver=self)
def test_upload_object_success(self):
def upload_file(self,
response,
file_path,
chunked=False,
calculate_hash=True):
return True, 'hash343hhash89h932439jsaa89', 1000
old_func = CloudFilesStorageDriver._upload_file
CloudFilesStorageDriver._upload_file = upload_file
file_path = os.path.abspath(__file__)
container = Container(name='foo_bar_container', extra={}, driver=self)
object_name = 'foo_test_upload'
extra = {'meta_data': {'some-value': 'foobar'}}
obj = self.driver.upload_object(file_path=file_path,
container=container,
extra=extra,
object_name=object_name)
self.assertEqual(obj.name, 'foo_test_upload')
self.assertEqual(obj.size, 1000)
self.assertTrue('some-value' in obj.meta_data)
CloudFilesStorageDriver._upload_file = old_func
def create_container(self, container_name):
req = RequestWithMethod(
"PUT",
"%s/%s?restype=container" % (self.get_base_url(), container_name))
req.add_header("Content-Length", "0")
self._credentials.sign_request(req)
try:
response = urlopen(req)
print("create_container ret code :" + str(response.code))
if response.code == 201:
container = Container(name=container_name,
extra=None,
driver=self)
return container
else:
raise LibcloudError("Unexpected status code %s" %
(response.status),
driver=self)
except HTTPError, e:
raise LibcloudError("Unexpected HTTPError %s " % (e.code),
driver=self)
def create_container(self, container_name, ex_location=None):
"""
@inherits :class:`StorageDriver.create_container`
:keyword ex_location: The desired location where to create container
:type keyword: ``str``
"""
extra = None
if ex_location:
root = Element('CreateBucketConfiguration')
child = SubElement(root, 'LocationConstraint')
child.text = ex_location
data = tostring(root)
extra = {'location': ex_location}
else:
data = ''
container = Container(name=container_name, extra=extra, driver=self)
response = self.connection.request('/',
data=data,
method='PUT',
container=container)
if response.status == httplib.OK:
return container
elif response.status == httplib.CONFLICT:
raise InvalidContainerNameError(
value='Container with this name already exists. The name must '
'be unique among all the containers in the system',
container_name=container_name, driver=self)
elif response.status == httplib.BAD_REQUEST:
raise ContainerError(
value='Bad request when creating container: %s' %
response.body,
container_name=container_name, driver=self)
raise LibcloudError('Unexpected status code: %s' % (response.status),
driver=self)
def test_upload_object_with_acl(self):
def upload_file(
self,
object_name=None,
content_type=None,
request_path=None,
request_method=None,
headers=None,
file_path=None,
stream=None,
):
return {
"response": make_response(
200, headers={"etag": "0cc175b9c0f1b6a831c399e269772661"}
),
"bytes_transferred": 1000,
"data_hash": "0cc175b9c0f1b6a831c399e269772661",
}
self.mock_response_klass.type = None
old_func = self.driver_type._upload_object
self.driver_type._upload_object = upload_file
file_path = os.path.abspath(__file__)
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
object_name = "foo_test_upload"
extra = {"acl": "public-read"}
obj = self.driver.upload_object(
file_path=file_path,
container=container,
object_name=object_name,
extra=extra,
verify_hash=True,
)
self.assertEqual(obj.name, "foo_test_upload")
self.assertEqual(obj.size, 1000)
self.assertEqual(obj.extra["acl"], "public-read")
self.driver_type._upload_object = old_func
def test_download_object_data_is_not_buffered_in_memory(self):
# Test case which verifies that response.body attribute is not accessed
# and as such, whole body response is not buffered into RAM
# If content is consumed and response.content attribute accessed execption
# will be thrown and test will fail
mock_response = Mock(name="mock response")
mock_response.headers = {}
mock_response.status_code = 200
msg = '"content" attribute was accessed but it shouldn\'t have been'
type(mock_response).content = PropertyMock(
name="mock content attribute", side_effect=Exception(msg))
mock_response.iter_content.return_value = StringIO("a" * 1000)
self.driver.connection.connection.getresponse = Mock()
self.driver.connection.connection.getresponse.return_value = mock_response
container = Container(name="foo_bar_container",
extra={},
driver=self.driver)
obj = Object(
name="foo_bar_object_NO_BUFFER",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
destination_path = self._file_path
result = self.driver.download_object(
obj=obj,
destination_path=destination_path,
overwrite_existing=True,
delete_on_failure=True,
)
self.assertTrue(result)
def test_upload_object_with_acl(self):
def upload_file(self,
object_name=None,
content_type=None,
request_path=None,
request_method=None,
headers=None,
file_path=None,
stream=None):
return {
'response':
make_response(
200, headers={'etag': '0cc175b9c0f1b6a831c399e269772661'}),
'bytes_transferred':
1000,
'data_hash':
'0cc175b9c0f1b6a831c399e269772661'
}
self.mock_response_klass.type = None
old_func = self.driver_type._upload_object
self.driver_type._upload_object = upload_file
file_path = os.path.abspath(__file__)
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
object_name = 'foo_test_upload'
extra = {'acl': 'public-read'}
obj = self.driver.upload_object(file_path=file_path,
container=container,
object_name=object_name,
extra=extra,
verify_hash=True)
self.assertEqual(obj.name, 'foo_test_upload')
self.assertEqual(obj.size, 1000)
self.assertEqual(obj.extra['acl'], 'public-read')
self.driver_type._upload_object = old_func
def test_upload_object_invalid_hash1(self):
# Invalid hash is detected on the amazon side and BAD_REQUEST is
# returned
def upload_file(self,
object_name=None,
content_type=None,
request_path=None,
request_method=None,
headers=None,
file_path=None,
stream=None):
return {
'response': make_response(200),
'bytes_transferred': 1000,
'data_hash': 'hash343hhash89h932439jsaa89'
}
self.mock_response_klass.type = 'INVALID_HASH1'
old_func = self.driver_type._upload_object
self.driver_type._upload_object = upload_file
file_path = os.path.abspath(__file__)
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
object_name = 'foo_test_upload'
try:
self.driver.upload_object(file_path=file_path,
container=container,
object_name=object_name,
verify_hash=True)
except ObjectHashMismatchError:
pass
else:
self.fail(
'Invalid hash was returned but an exception was not thrown')
finally:
self.driver_type._upload_object = old_func
def test_upload_big_block_object_success(self):
file_path = tempfile.mktemp(suffix='.jpg')
file_size = AZURE_UPLOAD_CHUNK_SIZE + 1
with open(file_path, 'w') as file_hdl:
file_hdl.write('0' * file_size)
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
object_name = 'foo_test_upload'
extra = {'meta_data': {'some-value': 'foobar'}}
obj = self.driver.upload_object(file_path=file_path,
container=container,
object_name=object_name,
extra=extra,
verify_hash=False)
self.assertEqual(obj.name, 'foo_test_upload')
self.assertEqual(obj.size, file_size)
self.assertTrue('some-value' in obj.meta_data)
os.remove(file_path)
def test_upload_object_invalid_hash(self):
def upload_file(self, response, file_path, chunked=False,
calculate_hash=True):
return True, 'hash343hhash89h932439jsaa89', 1000
CloudFilesMockRawResponse.type = 'INVALID_HASH'
old_func = CloudFilesStorageDriver._upload_file
CloudFilesStorageDriver._upload_file = upload_file
file_path = os.path.abspath(__file__)
container = Container(name='foo_bar_container', extra={}, driver=self)
object_name = 'foo_test_upload'
try:
self.driver.upload_object(file_path=file_path, container=container,
object_name=object_name,
verify_hash=True)
except ObjectHashMismatchError:
pass
else:
self.fail(
'Invalid hash was returned but an exception was not thrown')
finally:
CloudFilesStorageDriver._upload_file = old_func
def test_ex_multipart_upload_object_success(self):
_upload_object_part = CloudFilesStorageDriver._upload_object_part
_upload_object_manifest = CloudFilesStorageDriver._upload_object_manifest
mocked__upload_object_part = mock.Mock(return_value="test_part")
mocked__upload_object_manifest = mock.Mock(return_value="test_manifest")
CloudFilesStorageDriver._upload_object_part = mocked__upload_object_part
CloudFilesStorageDriver._upload_object_manifest = mocked__upload_object_manifest
parts = 5
file_path = os.path.abspath(__file__)
chunk_size = int(math.ceil(float(os.path.getsize(file_path)) / parts))
container = Container(name='foo_bar_container', extra={}, driver=self)
object_name = 'foo_test_upload'
self.driver.ex_multipart_upload_object(file_path=file_path, container=container,
object_name=object_name, chunk_size=chunk_size)
CloudFilesStorageDriver._upload_object_part = _upload_object_part
CloudFilesStorageDriver._upload_object_manifest = _upload_object_manifest
self.assertEqual(mocked__upload_object_part.call_count, parts)
self.assertTrue(mocked__upload_object_manifest.call_count, 1)
def test_upload_object_invalid_hash_kms_encryption(self):
# Hash check should be skipped when AWS KMS server side encryption is
# used
def upload_file(self, object_name=None, content_type=None,
request_path=None, request_method=None,
headers=None, file_path=None, stream=None):
headers = {'etag': 'blahblah', 'x-amz-server-side-encryption': 'aws:kms'}
return {'response': make_response(200, headers=headers),
'bytes_transferred': 1000,
'data_hash': 'hash343hhash89h932439jsaa81'}
old_func = self.driver_type._upload_object
self.driver_type._upload_object = upload_file
file_path = os.path.abspath(__file__)
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
object_name = 'foo_test_upload'
try:
self.driver.upload_object(file_path=file_path, container=container,
object_name=object_name,
verify_hash=True)
finally:
self.driver_type._upload_object = old_func
