def test_ex_get_object_temp_url(self, time):
time.return_value = 0
self.driver.ex_get_meta_data = mock.Mock()
self.driver.ex_get_meta_data.return_value = {'container_count': 1,
'object_count': 1,
'bytes_used': 1,
'temp_url_key': 'foo'}
container = Container(name='foo_bar_container', extra={}, driver=self)
obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
container=container, meta_data=None,
driver=self)
hmac_body = "%s\n%s\n%s" % ('GET', 60,
"/v1/MossoCloudFS/foo_bar_container/foo_bar_object")
sig = hmac.new(b('foo'), b(hmac_body), sha1).hexdigest()
ret = self.driver.ex_get_object_temp_url(obj, 'GET')
temp_url = 'https://storage101.%s1.clouddrive.com/v1/MossoCloudFS/foo_bar_container/foo_bar_object?temp_url_expires=60&temp_url_sig=%s' % (self.region, sig)
self.assertEquals(''.join(sorted(ret)), ''.join(sorted(temp_url)))
def test_download_object_as_stream_success(self):
container = Container(name="foo_bar_container",
extra={},
driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
stream = self.driver.download_object_as_stream(obj=obj,
chunk_size=None)
self.assertTrue(hasattr(stream, "__iter__"))
def test_download_object_invalid_file_size(self):
self.mock_raw_response_klass.type = 'INVALID_SIZE'
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type)
destination_path = os.path.abspath(__file__) + '.temp'
result = self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
self.assertFalse(result)
def test_delete_object_not_found(self):
self.mock_response_klass.type = 'NOT_FOUND'
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object',
size=1234,
hash=None,
extra=None,
meta_data=None,
container=container,
driver=self.driver)
try:
self.driver.delete_object(obj=obj)
except ObjectDoesNotExistError:
pass
else:
self.fail('Exception was not thrown')
def test_delete_object_not_found(self):
AtmosMockHttp.type = 'NOT_FOUND'
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver)
try:
self.driver.delete_object(obj=obj)
except ObjectDoesNotExistError:
pass
else:
self.fail('Object does not exist but an exception was not thrown')
def test_ex_get_object_temp_url_no_key_raises_key_error(self):
self.driver.ex_get_meta_data = mock.Mock()
self.driver.ex_get_meta_data.return_value = {
'container_count': 1,
'object_count': 1,
'bytes_used': 1,
'temp_url_key': None
}
container = Container(name='foo_bar_container', extra={}, driver=self)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self)
self.assertRaises(KeyError, self.driver.ex_get_object_temp_url, obj,
'GET')
def test_download_object(self):
url = self.url_tmpl % ('/'.join(
['containers', 'foo_bar_container', 'objects', 'foo_bar_object']))
container = Container(name='foo_bar_container', extra={}, driver=None)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=None)
with patch.object(CloudFilesUSStorageDriver,
'get_object',
mocksignature=True) as get_object_mock:
get_object_mock.return_value = obj
resp = self.client.get(url, headers=self.headers)
self.assertEqual(resp.status_code, httplib.OK)
def make_blob(self, blob_name, blob_date):
extra = {
'creation_time': blob_date,
'access_time': blob_date,
'modify_time': blob_date
}
checksum = hashlib.md5()
checksum.update(os.urandom(4))
return Object(
name=blob_name,
size=randrange(100),
extra=extra,
driver=self,
container=None,
hash=checksum.hexdigest(),
meta_data=None
)
def test_download_object_not_found(self):
self.mock_raw_response_klass.type = 'not_found'
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type)
destination_path = os.path.abspath(__file__) + '.temp'
self.assertRaises(ObjectDoesNotExistError,
self.driver.download_object,
obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
def test_download_object_success_not_found(self):
CloudFilesMockRawResponse.type = 'NOT_FOUND'
container = Container(name='foo_bar_container', extra={}, driver=self)
obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
container=container,
meta_data=None,
driver=CloudFilesStorageDriver)
destination_path = os.path.abspath(__file__) + '.temp'
try:
self.driver.download_object(
obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
except ObjectDoesNotExistError:
pass
else:
self.fail('Object does not exist but an exception was not thrown')
def _add_object(self, container, object_name, size, extra=None):
container = self.get_container(container.name)
extra = extra or {}
meta_data = extra.get('meta_data', {})
meta_data.update({
'cdn_url':
'http://www.test.com/object/%s' % (object_name.replace(' ', '_'))
})
obj = Object(name=object_name,
size=size,
extra=extra,
hash=None,
meta_data=meta_data,
container=container,
driver=self)
self._containers[container.name]['objects'][object_name] = obj
return obj
def test_download_object_success(self):
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
destination_path = os.path.abspath(__file__) + ".temp"
result = self.driver.download_object(
obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True,
)
self.assertTrue(result)
def test_download_object_range_success(self):
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
obj = Object(name='foo_bar_object_range', size=19, hash=None, extra={},
container=container, meta_data=None,
driver=self.driver_type)
destination_path = self._file_path
result = self.driver.download_object_range(obj=obj,
destination_path=destination_path,
start_bytes=5,
end_bytes=7,
overwrite_existing=True,
delete_on_failure=True)
self.assertTrue(result)
with open(self._file_path, 'r') as fp:
content = fp.read()
self.assertEqual(content, '56')
def test_download_object_range_as_stream_success(self):
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
obj = Object(
name="foo_bar_object_range_stream",
size=2,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
stream = self.driver.download_object_range_as_stream(
obj=obj, start_bytes=4, end_bytes=6, chunk_size=None
)
consumed_stream = "".join(chunk.decode("utf-8") for chunk in stream)
self.assertEqual(consumed_stream, "45")
self.assertEqual(len(consumed_stream), obj.size)
def test_download_object_invalid_file_already_exists(self):
S3MockRawResponse.type = 'INVALID_SIZE'
container = Container(name='foo_bar_container', extra={}, driver=self)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=S3StorageDriver)
destination_path = os.path.abspath(__file__)
try:
self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
except LibcloudError:
pass
else:
self.fail('Exception was not thrown')
def test_download_object_as_stream_success(self):
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
container=container, meta_data=None,
driver=self.driver_type)
def mock_get_object(self, obj, callback, callback_kwargs, response,
success_status_code=None):
return response._response.iter_content(1024)
old_func = self.driver_type._get_object
self.driver_type._get_object = mock_get_object
try:
stream = self.driver.download_object_as_stream(obj=obj,
chunk_size=1024)
self.assertTrue(hasattr(stream, '__iter__'))
finally:
self.driver_type._get_object = old_func
def _to_object_list(self, response, container):
objects = []
for obj in response:
name = obj['name']
size = int(obj['bytes'])
hash = obj['hash']
extra = {
'content_type': obj['content_type'],
'last_modified': obj['last_modified']
}
objects.append(
Object(name=name,
size=size,
hash=hash,
extra=extra,
meta_data=None,
container=container,
driver=self))
return objects
def _add_object(self, container, object_name, size, extra=None):
container = self.get_container(container.name)
extra = extra or {}
meta_data = extra.get("meta_data", {})
meta_data.update({
"cdn_url":
"http://www.test.com/object/%s" % (object_name.replace(" ", "_"))
})
obj = Object(
name=object_name,
size=size,
extra=extra,
hash=None,
meta_data=meta_data,
container=container,
driver=self,
)
self._containers[container.name]["objects"][object_name] = obj
return obj
def list_container_objects(self, container):
ret = None
try:
res = self.list_bucket(container.name)
body = res.read()
h = GetBucketXml(body).list()
if len(h[0]) == 0:
ret = []
return ret
ret = [
Object(i[0],
size=None,
hash=None,
extra=None,
meta_data=None,
container=container,
driver=self) for i in h[0]
]
return ret
except:
return ret
def upload_object(self,
file_path,
container,
object_name,
extra=None,
verify_hash=True,
ex_storage_class=None):
try:
res = self.put_object_from_file(container.name, object_name,
file_path, 'text/HTML', {})
if (res.status / 100) == 2:
obj = Object(object_name, None, None, None, None, container,
self)
return obj
else:
raise LibcloudError("Unexpected status code %s" % (res.status),
driver=self)
except HTTPError, e:
raise LibcloudError("Unexpected HTTPError %s " % (e.code),
driver=self)
def _make_object(self, container, object_name):
"""
Create an object instance
:param container: Container.
:type container: :class:`Container`
:param object_name: Object name.
:type object_name: ``str``
:return: Object instance.
:rtype: :class:`Object`
"""
full_path = os.path.join(self.base_path, container.name, object_name)
if os.path.isdir(full_path):
raise ObjectError(value=None, driver=self, object_name=object_name)
try:
stat = os.stat(full_path)
except Exception:
raise ObjectDoesNotExistError(value=None, driver=self,
object_name=object_name)
# Make a hash for the file based on the metadata. We can safely
# use only the mtime attribute here. If the file contents change,
# the underlying file-system will change mtime
data_hash = self._get_hash_function()
data_hash.update(u(stat.st_mtime).encode('ascii'))
data_hash = data_hash.hexdigest()
extra = {}
extra['creation_time'] = stat.st_ctime
extra['access_time'] = stat.st_atime
extra['modify_time'] = stat.st_mtime
return Object(name=object_name, size=stat.st_size, extra=extra,
driver=self, container=container, hash=data_hash,
meta_data=None)
def _headers_to_object(self, name, container, headers):
size = int(headers.pop('content-length', 0))
last_modified = headers.pop('last-modified', None)
etag = headers.pop('etag', None)
content_type = headers.pop('content-type', None)
meta_data = {}
for key, value in list(headers.items()):
if key.find('x-object-meta-') != -1:
key = key.replace('x-object-meta-', '')
meta_data[key] = value
extra = {'content_type': content_type, 'last_modified': last_modified}
obj = Object(name=name,
size=size,
hash=etag,
extra=extra,
meta_data=meta_data,
container=container,
driver=self)
return obj
def _headers_to_object(self, object_name, container, headers):
hash = headers.get("etag", "").replace('"', "")
extra = {"content_type": headers["content-type"]}
if "etag" in headers:
extra["etag"] = headers["etag"]
meta_data = {}
if "content-encoding" in headers:
extra["content_encoding"] = headers["content-encoding"]
if "last-modified" in headers:
extra["last_modified"] = headers["last-modified"]
for key, value in headers.items():
if not key.lower().startswith(self.http_vendor_prefix + "-meta-"):
continue
key = key.replace(self.http_vendor_prefix + "-meta-", "")
meta_data[key] = value
content_length = self._get_content_length_from_headers(headers=headers)
if content_length is None:
raise KeyError(
"Can not deduce object size from headers for "
"object %s" % (object_name)
)
obj = Object(
name=object_name,
size=int(content_length),
hash=hash,
extra=extra,
meta_data=meta_data,
container=container,
driver=self,
)
return obj
def _headers_to_object(self, object_name, container, headers):
hash = headers['etag'].replace('"', '')
extra = {'content_type': headers['content-type'],
'etag': headers['etag']}
meta_data = {}
if 'last-modified' in headers:
extra['last_modified'] = headers['last-modified']
for key, value in headers.items():
if not key.lower().startswith(self.http_vendor_prefix + '-meta-'):
continue
key = key.replace(self.http_vendor_prefix + '-meta-', '')
meta_data[key] = value
obj = Object(name=object_name, size=headers['content-length'],
hash=hash, extra=extra,
meta_data=meta_data,
container=container,
driver=self)
return obj
def _to_obj(self, element, container):
owner_id = findtext(element=element, xpath='Owner/ID',
namespace=self.namespace)
owner_display_name = findtext(element=element,
xpath='Owner/DisplayName',
namespace=self.namespace)
meta_data = {'owner': {'id': owner_id,
'display_name': owner_display_name}}
obj = Object(name=findtext(element=element, xpath='Key',
namespace=self.namespace),
size=int(findtext(element=element, xpath='Size',
namespace=self.namespace)),
hash=findtext(element=element, xpath='ETag',
namespace=self.namespace).replace('"', ''),
extra=None,
meta_data=meta_data,
container=container,
driver=self
)
return obj
def _upload_object_manifest(self, container, object_name, extra=None,
verify_hash=True):
extra = extra or {}
meta_data = extra.get('meta_data')
container_name_encoded = self._encode_container_name(container.name)
object_name_encoded = self._encode_object_name(object_name)
request_path = '/%s/%s' % (container_name_encoded, object_name_encoded)
# pylint: disable=no-member
headers = {'X-Auth-Token': self.connection.auth_token,
'X-Object-Manifest': '%s/%s/' %
(container_name_encoded,
object_name_encoded)}
data = ''
response = self.connection.request(request_path,
method='PUT', data=data,
headers=headers, raw=True)
object_hash = None
if verify_hash:
hash_function = self._get_hash_function()
hash_function.update(b(data))
data_hash = hash_function.hexdigest()
object_hash = response.headers.get('etag')
if object_hash != data_hash:
raise ObjectHashMismatchError(
value=('MD5 hash checksum does not match (expected=%s, ' +
'actual=%s)') %
(data_hash, object_hash),
object_name=object_name, driver=self)
obj = Object(name=object_name, size=0, hash=object_hash, extra=None,
meta_data=meta_data, container=container, driver=self)
return obj
def test_download_object_as_stream_data_is_not_buffered_in_memory(self):
# Test case which verifies that response.response attribute is not accessed
# and as such, whole body response is not buffered into RAM
# If content is consumed and response.content attribute accessed exception
# will be thrown and test will fail
mock_response = Mock(name='mock response')
mock_response.headers = {}
mock_response.status = 200
msg1 = '"response" attribute was accessed but it shouldn\'t have been'
msg2 = '"content" attribute was accessed but it shouldn\'t have been'
type(mock_response).response = PropertyMock(
name='mock response attribute', side_effect=Exception(msg1))
type(mock_response).content = PropertyMock(
name='mock content attribute', side_effect=Exception(msg2))
mock_response.iter_content.return_value = StringIO('a' * 1000)
self.driver.connection.request = Mock()
self.driver.connection.request.return_value = mock_response
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object_NO_BUFFER',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type)
destination_path = self._file_path
result = self.driver.download_object_as_stream(obj=obj)
result = exhaust_iterator(result)
if PY3:
result = result.decode('utf-8')
self.assertEqual(result, 'a' * 1000)
def test_download_object_data_is_not_buffered_in_memory(self):
# Test case which verifies that response.body attribute is not accessed
# and as such, whole body response is not buffered into RAM
# If content is consumed and response.content attribute accessed execption
# will be thrown and test will fail
mock_response = Mock(name="mock response")
mock_response.headers = {}
mock_response.status_code = 200
msg = '"content" attribute was accessed but it shouldn\'t have been'
type(mock_response).content = PropertyMock(
name="mock content attribute", side_effect=Exception(msg))
mock_response.iter_content.return_value = StringIO("a" * 1000)
self.driver.connection.connection.getresponse = Mock()
self.driver.connection.connection.getresponse.return_value = mock_response
container = Container(name="foo_bar_container",
extra={},
driver=self.driver)
obj = Object(
name="foo_bar_object_NO_BUFFER",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
destination_path = self._file_path
result = self.driver.download_object(
obj=obj,
destination_path=destination_path,
overwrite_existing=True,
delete_on_failure=True,
)
self.assertTrue(result)
def _make_object(self, container, object_name):
"""
Create an object instance
@param container: Container.
@type container: L{Container}
@param object_name: Object name.
@type object_name: C{str}
@return: Object instance.
@rtype: L{Object}
"""
full_path = os.path.join(self.base_path, container.name, object_name)
if os.path.isdir(full_path):
raise ObjectError(value=None, driver=self, object_name=object_name)
try:
stat = os.stat(full_path)
except Exception:
raise ObjectDoesNotExistError(value=None,
driver=self,
object_name=object_name)
extra = {}
extra['creation_time'] = stat.st_ctime
extra['access_time'] = stat.st_atime
extra['modify_time'] = stat.st_mtime
return Object(name=object_name,
size=stat.st_size,
extra=extra,
driver=self,
container=container,
hash=None,
meta_data=None)
def test_download_object_range_invalid_values(self):
obj = Object('a', 500, '', {}, {}, None, None)
tmppath = self.make_tmp_file(content='')
expected_msg = 'start_bytes must be greater than 0'
self.assertRaisesRegex(ValueError, expected_msg,
self.driver.download_object_range, obj=obj,
destination_path=tmppath,
start_bytes=-1)
expected_msg = 'start_bytes must be smaller than end_bytes'
self.assertRaisesRegex(ValueError, expected_msg,
self.driver.download_object_range, obj=obj,
destination_path=tmppath,
start_bytes=5,
end_bytes=4)
expected_msg = 'start_bytes and end_bytes can\'t be the same'
self.assertRaisesRegex(ValueError, expected_msg,
self.driver.download_object_range, obj=obj,
destination_path=tmppath,
start_bytes=5,
end_bytes=5)