def test_delete_object_not_found(self):
AtmosMockHttp.type = 'NOT_FOUND'
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
container=container, meta_data=None,
driver=self.driver)
try:
self.driver.delete_object(obj=obj)
except ObjectDoesNotExistError:
pass
else:
self.fail('Object does not exist but an exception was not thrown')
def test_delete_object_not_found(self):
self.mock_response_klass.type = "not_found"
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1234,
hash=None,
extra=None,
meta_data=None,
container=container,
driver=self.driver,
)
self.assertRaises(ObjectDoesNotExistError, self.driver.delete_object, obj=obj)
def test_download_object_as_stream(self):
container = Container(name='foo_bar_container', extra={}, driver=self)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=CloudFilesStorageDriver)
stream = self.driver.download_object_as_stream(obj=obj,
chunk_size=None)
self.assertTrue(hasattr(stream, '__iter__'))
def test_download_object_invalid_file_size(self):
self.mock_raw_response_klass.type = 'INVALID_SIZE'
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
container=container, meta_data=None,
driver=S3StorageDriver)
destination_path = os.path.abspath(__file__) + '.temp'
result = self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
self.assertFalse(result)
def __init__(self): # pylint: disable=W0231
self._TEST_CONTAINER = Container(name="test_container",
extra={},
driver=self)
self._TEST_OBJECT = Object(
name="test_obj",
size=1234,
hash="123sdfsdf",
extra={},
meta_data={"key": "value"},
container=self._TEST_CONTAINER,
driver=self,
)
def _upload_object_manifest(self,
container,
object_name,
extra=None,
verify_hash=True):
extra = extra or {}
meta_data = extra.get('meta_data')
container_name_encoded = self._encode_container_name(container.name)
object_name_encoded = self._encode_object_name(object_name)
request_path = '/%s/%s' % (container_name_encoded, object_name_encoded)
# pylint: disable=no-member
headers = {
'X-Auth-Token':
self.connection.auth_token,
'X-Object-Manifest':
'%s/%s/' % (container_name_encoded, object_name_encoded)
}
data = ''
response = self.connection.request(request_path,
method='PUT',
data=data,
headers=headers,
raw=True)
object_hash = None
if verify_hash:
hash_function = self._get_hash_function()
hash_function.update(b(data))
data_hash = hash_function.hexdigest()
object_hash = response.headers.get('etag')
if object_hash != data_hash:
raise ObjectHashMismatchError(
value=('MD5 hash checksum does not match (expected=%s, ' +
'actual=%s)') % (data_hash, object_hash),
object_name=object_name,
driver=self)
obj = Object(name=object_name,
size=0,
hash=object_hash,
extra=None,
meta_data=meta_data,
container=container,
driver=self)
return obj
def _response_to_object(self, object_name, container, response):
"""
Converts a HTTP response to an object (from headers)
:param object_name: Name of the object
:type object_name: ``str``
:param container: Instance of the container holding the blob
:type: :class:`Container`
:param response: HTTP Response
:type node: L{}
:return: An object instance
:rtype: :class:`Object`
"""
headers = response.headers
size = int(headers['content-length'])
etag = headers['etag']
extra = {
'url': 'http://%s%s' % (response.connection.host,
response.connection.action),
'etag': etag,
'md5_hash': headers.get('content-md5', None),
'content_type': headers.get('content-type', None),
'content_language': headers.get('content-language', None),
'content_encoding': headers.get('content-encoding', None),
'last_modified': headers['last-modified'],
'lease': {
'status': headers.get('x-ms-lease-status', None),
'state': headers.get('x-ms-lease-state', None),
'duration': headers.get('x-ms-lease-duration', None),
},
'blob_type': headers['x-ms-blob-type']
}
if extra['md5_hash']:
value = binascii.hexlify(base64.b64decode(b(extra['md5_hash'])))
value = value.decode('ascii')
extra['md5_hash'] = value
meta_data = {}
for key, value in response.headers.items():
if key.startswith('x-ms-meta-'):
key = key.split('x-ms-meta-')[1]
meta_data[key] = value
return Object(name=object_name, size=size, hash=etag, extra=extra,
meta_data=meta_data, container=container, driver=self)
def _xml_to_object(self, container, blob):
"""
Converts a BLOB XML node to an object instance
:param container: Instance of the container holding the blob
:type: :class:`Container`
:param blob: XML info of the blob
:type blob: L{}
:return: An object instance
:rtype: :class:`Object`
"""
name = blob.findtext(fixxpath(xpath='Name'))
props = blob.find(fixxpath(xpath='Properties'))
metadata = blob.find(fixxpath(xpath='Metadata'))
etag = props.findtext(fixxpath(xpath='Etag'))
size = int(props.findtext(fixxpath(xpath='Content-Length')))
extra = {
'content_type': props.findtext(fixxpath(xpath='Content-Type')),
'etag': etag,
'md5_hash': props.findtext(fixxpath(xpath='Content-MD5')),
'last_modified': props.findtext(fixxpath(xpath='Last-Modified')),
'url': blob.findtext(fixxpath(xpath='Url')),
'hash': props.findtext(fixxpath(xpath='Etag')),
'lease': {
'status': props.findtext(fixxpath(xpath='LeaseStatus')),
'state': props.findtext(fixxpath(xpath='LeaseState')),
'duration': props.findtext(fixxpath(xpath='LeaseDuration')),
},
'content_encoding': props.findtext(fixxpath(
xpath='Content-Encoding')),
'content_language': props.findtext(fixxpath(
xpath='Content-Language')),
'blob_type': props.findtext(fixxpath(xpath='BlobType'))
}
if extra['md5_hash']:
value = binascii.hexlify(base64.b64decode(b(extra['md5_hash'])))
value = value.decode('ascii')
extra['md5_hash'] = value
meta_data = {}
if metadata is not None:
for meta in list(metadata):
meta_data[meta.tag] = meta.text
return Object(name=name, size=size, hash=etag, meta_data=meta_data,
extra=extra, container=container, driver=self)
def upload_object(self,
file_path,
container,
object_name,
extra=None,
verify_hash=True):
method = 'PUT'
extra = extra or {}
object_name_cleaned = self._clean_object_name(object_name)
request_path = self._namespace_path(container.name) + '/' +\
object_name_cleaned
content_type = extra.get('content_type', None)
try:
self.connection.request(request_path + '?metadata/system')
except AtmosError:
e = sys.exc_info()[1]
if e.code != 1003:
raise
method = 'POST'
result_dict = self._upload_object(object_name=object_name,
content_type=content_type,
request_path=request_path,
request_method=method,
headers={},
file_path=file_path)
bytes_transferred = result_dict['bytes_transferred']
if extra is None:
meta_data = {}
else:
meta_data = extra.get('meta_data', {})
meta_data['md5'] = result_dict['data_hash']
user_meta = ', '.join(
[k + '=' + str(v) for k, v in list(meta_data.items())])
self.connection.request(request_path + '?metadata/user',
method='POST',
headers={'x-emc-meta': user_meta})
result = self.connection.request(request_path + '?metadata/system')
meta = self._emc_meta(result)
del meta_data['md5']
extra = {
'object_id': meta['objectid'],
'meta_data': meta_data,
}
return Object(object_name, bytes_transferred, result_dict['data_hash'],
extra, meta_data, container, self)
def _put_object(self, container, object_name, upload_func,
upload_func_kwargs, extra=None, file_path=None,
iterator=None, verify_hash=True):
extra = extra or {}
container_name_cleaned = self._clean_container_name(container.name)
object_name_cleaned = self._clean_object_name(object_name)
content_type = extra.get('content_type', None)
meta_data = extra.get('meta_data', None)
headers = {}
if meta_data:
for key, value in meta_data.iteritems():
key = 'X-Object-Meta-%s' % (key)
headers[key] = value
request_path = '/%s/%s' % (container_name_cleaned, object_name_cleaned)
result_dict = self._upload_object(object_name=object_name,
content_type=content_type,
upload_func=upload_func,
upload_func_kwargs=upload_func_kwargs,
request_path=request_path,
request_method='PUT',
headers=headers, file_path=file_path,
iterator=iterator)
response = result_dict['response'].response
bytes_transferred = result_dict['bytes_transferred']
server_hash = result_dict['response'].headers.get('etag', None)
if response.status == httplib.EXPECTATION_FAILED:
raise LibcloudError(value='Missing content-type header',
driver=self)
elif verify_hash and not server_hash:
raise LibcloudError(value='Server didn\'t return etag',
driver=self)
elif (verify_hash and result_dict['data_hash'] != server_hash):
raise ObjectHashMismatchError(
value=('MD5 hash checksum does not match (expected=%s, ' +
'actual=%s)') % (result_dict['data_hash'], server_hash),
object_name=object_name, driver=self)
elif response.status == httplib.CREATED:
obj = Object(
name=object_name, size=bytes_transferred, hash=server_hash,
extra=None, meta_data=meta_data, container=container,
driver=self)
return obj
else:
# @TODO: Add test case for this condition (probably 411)
raise LibcloudError('status_code=%s' % (response.status),
driver=self)
def test_delete_object_escaped_success(self):
AtmosMockHttp.type = 'DELETE'
container = Container(name='foo & bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo & bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver)
status = self.driver.delete_object(obj=obj)
self.assertTrue(status)
def _to_object_list(self, response, container):
objects = []
for obj in response:
name = obj['name']
size = int(obj['bytes'])
hash = obj['hash']
extra = {'content_type': obj['content_type'],
'last_modified': obj['last_modified']}
objects.append(Object(
name=name, size=size, hash=hash, extra=extra,
meta_data=None, container=container, driver=self))
return objects
def test_ex_purge_object_from_cdn(self):
CloudFilesMockHttp.type = 'PURGE_SUCCESS'
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self)
self.assertTrue(self.driver.ex_purge_object_from_cdn(obj=obj))
def test_delete_object_success(self):
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object',
size=1234,
hash=None,
extra=None,
meta_data=None,
container=container,
driver=self.driver)
result = self.driver.delete_object(obj=obj)
self.assertTrue(result)
def test_download_object_not_found(self):
self.mock_response_klass.type = 'not_found'
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
container=container, meta_data=None,
driver=self.driver_type)
destination_path = os.path.abspath(__file__) + '.temp'
self.assertRaises(ObjectDoesNotExistError,
self.driver.download_object,
obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
def test_download_object_as_stream_success(self):
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None)
self.assertTrue(hasattr(stream, "__iter__"))
def _to_object(self, item, container=None):
extra = {}
extra['fileId'] = item['fileId']
extra['uploadTimestamp'] = item.get('uploadTimestamp', None)
size = item.get('size', item.get('contentLength', None))
hash = item.get('contentSha1', None)
meta_data = item.get('fileInfo', {})
obj = Object(name=item['fileName'],
size=size,
hash=hash,
extra=extra,
meta_data=meta_data,
container=container,
driver=self)
return obj
def get_object(self, container_name, object_name):
container = self.get_container(container_name)
if self.object_exists(container_name, object_name):
obj = Object(object_name,
size=None,
hash=None,
extra=None,
meta_data=None,
container=container,
driver=self)
return obj
else:
raise ObjectDoesNotExistError(value=None,
driver=self,
object_name=object_name)
def _add_object(self, container, object_name, size, extra=None):
container = self.get_container(container.name)
extra = extra or {}
meta_data = extra.get('meta_data', {})
obj = Object(name=object_name,
size=size,
extra=extra,
hash=None,
meta_data=meta_data,
container=container,
driver=self)
self._containers[container.name]['objects'][object_name] = obj
return obj
def test_delete_object_success(self):
self.mock_response_klass.type = "delete"
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1234,
hash=None,
extra=None,
meta_data=None,
container=container,
driver=self.driver,
)
result = self.driver.delete_object(obj=obj)
self.assertTrue(result)
def test_download_object_success(self):
container = Container(name='foo_bar_container', extra={}, driver=self)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=CloudFilesStorageDriver)
destination_path = os.path.abspath(__file__) + '.temp'
result = self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
self.assertTrue(result)
def _to_obj(self, irods_data_obj, container):
extra = {
'path': irods_data_obj.path,
'create_time': irods_data_obj.create_time,
'irods': irods_data_obj,
'last_modified': irods_data_obj.modify_time
}
obj = Object(name=irods_data_obj.name,
size=irods_data_obj.size,
hash=irods_data_obj.checksum,
extra=extra,
meta_data={},
container=container,
driver=self)
return obj
def test_download_object_range_as_stream_success(self):
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object_range_stream',
size=19,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type)
iterator = self.driver.download_object_range_as_stream(obj=obj,
start_bytes=4,
end_bytes=7)
content = exhaust_iterator(iterator)
self.assertEqual(content, b'456')
def make_blob(self, blob_name, blob_date):
extra = {
'creation_time': blob_date,
'access_time': blob_date,
'modify_time': blob_date
}
checksum = hashlib.md5()
checksum.update(os.urandom(4))
return Object(name=blob_name,
size=randrange(100),
extra=extra,
driver=self,
container=None,
hash=checksum.hexdigest(),
meta_data=None)
def _headers_to_object(self, name, container, headers):
size = int(headers.pop('content-length', 0))
last_modified = headers.pop('last-modified', None)
etag = headers.pop('etag', None)
content_type = headers.pop('content-type', None)
meta_data = {}
for key, value in list(headers.items()):
if key.find('x-object-meta-') != -1:
key = key.replace('x-object-meta-', '')
meta_data[key] = value
extra = {'content_type': content_type, 'last_modified': last_modified}
obj = Object(name=name, size=size, hash=etag, extra=extra,
meta_data=meta_data, container=container, driver=self)
return obj
def test_download_object_as_stream_success(self):
container = Container(name="foo_bar_container", extra={}, driver=self.driver)
obj = Object(
name="foo_bar_object",
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type,
)
stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None)
consumed_stream = "".join(chunk.decode("utf-8") for chunk in stream)
self.assertEqual(len(consumed_stream), obj.size)
def test_download_object_success(self):
container = Container(name='foo_bar_container',
extra={},
driver=self.driver)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=self.driver_type)
destination_path = self._file_path
result = self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=True,
delete_on_failure=True)
self.assertTrue(result)
def test_delete_object_success(self):
url = self.url_tmpl % ('/'.join(
['containers', 'foo_bar_container', 'objects', 'foo_bar_object']))
container = Container(name='foo_bar_container', extra={}, driver=None)
obj = Object(name='foo_bar_object',
size=1000,
hash=None,
extra={},
container=container,
meta_data=None,
driver=None)
with patch.object(CloudFilesUSStorageDriver,
'get_object',
mocksignature=True) as get_object_mock:
get_object_mock.return_value = obj
resp = self.client.delete(url, headers=self.headers)
self.assertEqual(resp.status_code, httplib.OK)
def test_download_object_invalid_file_already_exists(self):
self.mock_response_klass.type = 'INVALID_SIZE'
container = Container(name='foo_bar_container', extra={},
driver=self.driver)
obj = Object(name='foo_bar_object', size=1000, hash=None, extra={},
container=container, meta_data=None,
driver=self.driver_type)
destination_path = os.path.abspath(__file__)
try:
self.driver.download_object(obj=obj,
destination_path=destination_path,
overwrite_existing=False,
delete_on_failure=True)
except LibcloudError:
pass
else:
self.fail('Exception was not thrown')
def _make_object(self, container, object_name):
"""
Create an object instance
@param container: Container.
@type container: L{Container}
@param object_name: Object name.
@type object_name: C{str}
@return: Object instance.
@rtype: L{Object}
"""
full_path = os.path.join(self.base_path, container.name, object_name)
if os.path.isdir(full_path):
raise ObjectError(value=None, driver=self, object_name=object_name)
try:
stat = os.stat(full_path)
except Exception:
raise ObjectDoesNotExistError(value=None,
driver=self,
object_name=object_name)
# Make a hash for the file based on the metadata. We can safely
# use only the mtime attribute here. If the file contents change,
# the underlying file-system will change mtime
data_hash = self._get_hash_function()
data_hash.update(str(stat.st_mtime))
data_hash = data_hash.hexdigest()
extra = {}
extra['creation_time'] = stat.st_ctime
extra['access_time'] = stat.st_atime
extra['modify_time'] = stat.st_mtime
return Object(name=object_name,
size=stat.st_size,
extra=extra,
driver=self,
container=container,
hash=data_hash,
meta_data=None)
def test_object_with_chinese_name(self):
driver = OSSStorageDriver(*STORAGE_OSS_PARAMS)
obj = Object(name='中文', size=0, hash=None, extra=None,
meta_data=None, container=None, driver=driver)
self.assertTrue(obj.__repr__() is not None)
