def test_delete_object_not_found(self): AtmosMockHttp.type = 'NOT_FOUND' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver) try: self.driver.delete_object(obj=obj) except ObjectDoesNotExistError: pass else: self.fail('Object does not exist but an exception was not thrown')
def test_delete_object_not_found(self): self.mock_response_klass.type = "not_found" container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object", size=1234, hash=None, extra=None, meta_data=None, container=container, driver=self.driver, ) self.assertRaises(ObjectDoesNotExistError, self.driver.delete_object, obj=obj)
def test_download_object_as_stream(self): container = Container(name='foo_bar_container', extra={}, driver=self) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=CloudFilesStorageDriver) stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) self.assertTrue(hasattr(stream, '__iter__'))
def test_download_object_invalid_file_size(self): self.mock_raw_response_klass.type = 'INVALID_SIZE' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=S3StorageDriver) destination_path = os.path.abspath(__file__) + '.temp' result = self.driver.download_object(obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True) self.assertFalse(result)
def __init__(self): # pylint: disable=W0231 self._TEST_CONTAINER = Container(name="test_container", extra={}, driver=self) self._TEST_OBJECT = Object( name="test_obj", size=1234, hash="123sdfsdf", extra={}, meta_data={"key": "value"}, container=self._TEST_CONTAINER, driver=self, )
def _upload_object_manifest(self, container, object_name, extra=None, verify_hash=True): extra = extra or {} meta_data = extra.get('meta_data') container_name_encoded = self._encode_container_name(container.name) object_name_encoded = self._encode_object_name(object_name) request_path = '/%s/%s' % (container_name_encoded, object_name_encoded) # pylint: disable=no-member headers = { 'X-Auth-Token': self.connection.auth_token, 'X-Object-Manifest': '%s/%s/' % (container_name_encoded, object_name_encoded) } data = '' response = self.connection.request(request_path, method='PUT', data=data, headers=headers, raw=True) object_hash = None if verify_hash: hash_function = self._get_hash_function() hash_function.update(b(data)) data_hash = hash_function.hexdigest() object_hash = response.headers.get('etag') if object_hash != data_hash: raise ObjectHashMismatchError( value=('MD5 hash checksum does not match (expected=%s, ' + 'actual=%s)') % (data_hash, object_hash), object_name=object_name, driver=self) obj = Object(name=object_name, size=0, hash=object_hash, extra=None, meta_data=meta_data, container=container, driver=self) return obj
def _response_to_object(self, object_name, container, response): """ Converts a HTTP response to an object (from headers) :param object_name: Name of the object :type object_name: ``str`` :param container: Instance of the container holding the blob :type: :class:`Container` :param response: HTTP Response :type node: L{} :return: An object instance :rtype: :class:`Object` """ headers = response.headers size = int(headers['content-length']) etag = headers['etag'] extra = { 'url': 'http://%s%s' % (response.connection.host, response.connection.action), 'etag': etag, 'md5_hash': headers.get('content-md5', None), 'content_type': headers.get('content-type', None), 'content_language': headers.get('content-language', None), 'content_encoding': headers.get('content-encoding', None), 'last_modified': headers['last-modified'], 'lease': { 'status': headers.get('x-ms-lease-status', None), 'state': headers.get('x-ms-lease-state', None), 'duration': headers.get('x-ms-lease-duration', None), }, 'blob_type': headers['x-ms-blob-type'] } if extra['md5_hash']: value = binascii.hexlify(base64.b64decode(b(extra['md5_hash']))) value = value.decode('ascii') extra['md5_hash'] = value meta_data = {} for key, value in response.headers.items(): if key.startswith('x-ms-meta-'): key = key.split('x-ms-meta-')[1] meta_data[key] = value return Object(name=object_name, size=size, hash=etag, extra=extra, meta_data=meta_data, container=container, driver=self)
def _xml_to_object(self, container, blob): """ Converts a BLOB XML node to an object instance :param container: Instance of the container holding the blob :type: :class:`Container` :param blob: XML info of the blob :type blob: L{} :return: An object instance :rtype: :class:`Object` """ name = blob.findtext(fixxpath(xpath='Name')) props = blob.find(fixxpath(xpath='Properties')) metadata = blob.find(fixxpath(xpath='Metadata')) etag = props.findtext(fixxpath(xpath='Etag')) size = int(props.findtext(fixxpath(xpath='Content-Length'))) extra = { 'content_type': props.findtext(fixxpath(xpath='Content-Type')), 'etag': etag, 'md5_hash': props.findtext(fixxpath(xpath='Content-MD5')), 'last_modified': props.findtext(fixxpath(xpath='Last-Modified')), 'url': blob.findtext(fixxpath(xpath='Url')), 'hash': props.findtext(fixxpath(xpath='Etag')), 'lease': { 'status': props.findtext(fixxpath(xpath='LeaseStatus')), 'state': props.findtext(fixxpath(xpath='LeaseState')), 'duration': props.findtext(fixxpath(xpath='LeaseDuration')), }, 'content_encoding': props.findtext(fixxpath( xpath='Content-Encoding')), 'content_language': props.findtext(fixxpath( xpath='Content-Language')), 'blob_type': props.findtext(fixxpath(xpath='BlobType')) } if extra['md5_hash']: value = binascii.hexlify(base64.b64decode(b(extra['md5_hash']))) value = value.decode('ascii') extra['md5_hash'] = value meta_data = {} if metadata is not None: for meta in list(metadata): meta_data[meta.tag] = meta.text return Object(name=name, size=size, hash=etag, meta_data=meta_data, extra=extra, container=container, driver=self)
def upload_object(self, file_path, container, object_name, extra=None, verify_hash=True): method = 'PUT' extra = extra or {} object_name_cleaned = self._clean_object_name(object_name) request_path = self._namespace_path(container.name) + '/' +\ object_name_cleaned content_type = extra.get('content_type', None) try: self.connection.request(request_path + '?metadata/system') except AtmosError: e = sys.exc_info()[1] if e.code != 1003: raise method = 'POST' result_dict = self._upload_object(object_name=object_name, content_type=content_type, request_path=request_path, request_method=method, headers={}, file_path=file_path) bytes_transferred = result_dict['bytes_transferred'] if extra is None: meta_data = {} else: meta_data = extra.get('meta_data', {}) meta_data['md5'] = result_dict['data_hash'] user_meta = ', '.join( [k + '=' + str(v) for k, v in list(meta_data.items())]) self.connection.request(request_path + '?metadata/user', method='POST', headers={'x-emc-meta': user_meta}) result = self.connection.request(request_path + '?metadata/system') meta = self._emc_meta(result) del meta_data['md5'] extra = { 'object_id': meta['objectid'], 'meta_data': meta_data, } return Object(object_name, bytes_transferred, result_dict['data_hash'], extra, meta_data, container, self)
def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, iterator=None, verify_hash=True): extra = extra or {} container_name_cleaned = self._clean_container_name(container.name) object_name_cleaned = self._clean_object_name(object_name) content_type = extra.get('content_type', None) meta_data = extra.get('meta_data', None) headers = {} if meta_data: for key, value in meta_data.iteritems(): key = 'X-Object-Meta-%s' % (key) headers[key] = value request_path = '/%s/%s' % (container_name_cleaned, object_name_cleaned) result_dict = self._upload_object(object_name=object_name, content_type=content_type, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, request_path=request_path, request_method='PUT', headers=headers, file_path=file_path, iterator=iterator) response = result_dict['response'].response bytes_transferred = result_dict['bytes_transferred'] server_hash = result_dict['response'].headers.get('etag', None) if response.status == httplib.EXPECTATION_FAILED: raise LibcloudError(value='Missing content-type header', driver=self) elif verify_hash and not server_hash: raise LibcloudError(value='Server didn\'t return etag', driver=self) elif (verify_hash and result_dict['data_hash'] != server_hash): raise ObjectHashMismatchError( value=('MD5 hash checksum does not match (expected=%s, ' + 'actual=%s)') % (result_dict['data_hash'], server_hash), object_name=object_name, driver=self) elif response.status == httplib.CREATED: obj = Object( name=object_name, size=bytes_transferred, hash=server_hash, extra=None, meta_data=meta_data, container=container, driver=self) return obj else: # @TODO: Add test case for this condition (probably 411) raise LibcloudError('status_code=%s' % (response.status), driver=self)
def test_delete_object_escaped_success(self): AtmosMockHttp.type = 'DELETE' container = Container(name='foo & bar_container', extra={}, driver=self.driver) obj = Object(name='foo & bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver) status = self.driver.delete_object(obj=obj) self.assertTrue(status)
def _to_object_list(self, response, container): objects = [] for obj in response: name = obj['name'] size = int(obj['bytes']) hash = obj['hash'] extra = {'content_type': obj['content_type'], 'last_modified': obj['last_modified']} objects.append(Object( name=name, size=size, hash=hash, extra=extra, meta_data=None, container=container, driver=self)) return objects
def test_ex_purge_object_from_cdn(self): CloudFilesMockHttp.type = 'PURGE_SUCCESS' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self) self.assertTrue(self.driver.ex_purge_object_from_cdn(obj=obj))
def test_delete_object_success(self): container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1234, hash=None, extra=None, meta_data=None, container=container, driver=self.driver) result = self.driver.delete_object(obj=obj) self.assertTrue(result)
def test_download_object_not_found(self): self.mock_response_klass.type = 'not_found' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) destination_path = os.path.abspath(__file__) + '.temp' self.assertRaises(ObjectDoesNotExistError, self.driver.download_object, obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True)
def test_download_object_as_stream_success(self): container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object", size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type, ) stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) self.assertTrue(hasattr(stream, "__iter__"))
def _to_object(self, item, container=None): extra = {} extra['fileId'] = item['fileId'] extra['uploadTimestamp'] = item.get('uploadTimestamp', None) size = item.get('size', item.get('contentLength', None)) hash = item.get('contentSha1', None) meta_data = item.get('fileInfo', {}) obj = Object(name=item['fileName'], size=size, hash=hash, extra=extra, meta_data=meta_data, container=container, driver=self) return obj
def get_object(self, container_name, object_name): container = self.get_container(container_name) if self.object_exists(container_name, object_name): obj = Object(object_name, size=None, hash=None, extra=None, meta_data=None, container=container, driver=self) return obj else: raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name)
def _add_object(self, container, object_name, size, extra=None): container = self.get_container(container.name) extra = extra or {} meta_data = extra.get('meta_data', {}) obj = Object(name=object_name, size=size, extra=extra, hash=None, meta_data=meta_data, container=container, driver=self) self._containers[container.name]['objects'][object_name] = obj return obj
def test_delete_object_success(self): self.mock_response_klass.type = "delete" container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object", size=1234, hash=None, extra=None, meta_data=None, container=container, driver=self.driver, ) result = self.driver.delete_object(obj=obj) self.assertTrue(result)
def test_download_object_success(self): container = Container(name='foo_bar_container', extra={}, driver=self) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=CloudFilesStorageDriver) destination_path = os.path.abspath(__file__) + '.temp' result = self.driver.download_object(obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True) self.assertTrue(result)
def _to_obj(self, irods_data_obj, container): extra = { 'path': irods_data_obj.path, 'create_time': irods_data_obj.create_time, 'irods': irods_data_obj, 'last_modified': irods_data_obj.modify_time } obj = Object(name=irods_data_obj.name, size=irods_data_obj.size, hash=irods_data_obj.checksum, extra=extra, meta_data={}, container=container, driver=self) return obj
def test_download_object_range_as_stream_success(self): container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object_range_stream', size=19, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) iterator = self.driver.download_object_range_as_stream(obj=obj, start_bytes=4, end_bytes=7) content = exhaust_iterator(iterator) self.assertEqual(content, b'456')
def make_blob(self, blob_name, blob_date): extra = { 'creation_time': blob_date, 'access_time': blob_date, 'modify_time': blob_date } checksum = hashlib.md5() checksum.update(os.urandom(4)) return Object(name=blob_name, size=randrange(100), extra=extra, driver=self, container=None, hash=checksum.hexdigest(), meta_data=None)
def _headers_to_object(self, name, container, headers): size = int(headers.pop('content-length', 0)) last_modified = headers.pop('last-modified', None) etag = headers.pop('etag', None) content_type = headers.pop('content-type', None) meta_data = {} for key, value in list(headers.items()): if key.find('x-object-meta-') != -1: key = key.replace('x-object-meta-', '') meta_data[key] = value extra = {'content_type': content_type, 'last_modified': last_modified} obj = Object(name=name, size=size, hash=etag, extra=extra, meta_data=meta_data, container=container, driver=self) return obj
def test_download_object_as_stream_success(self): container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object", size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type, ) stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) consumed_stream = "".join(chunk.decode("utf-8") for chunk in stream) self.assertEqual(len(consumed_stream), obj.size)
def test_download_object_success(self): container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) destination_path = self._file_path result = self.driver.download_object(obj=obj, destination_path=destination_path, overwrite_existing=True, delete_on_failure=True) self.assertTrue(result)
def test_delete_object_success(self): url = self.url_tmpl % ('/'.join( ['containers', 'foo_bar_container', 'objects', 'foo_bar_object'])) container = Container(name='foo_bar_container', extra={}, driver=None) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=None) with patch.object(CloudFilesUSStorageDriver, 'get_object', mocksignature=True) as get_object_mock: get_object_mock.return_value = obj resp = self.client.delete(url, headers=self.headers) self.assertEqual(resp.status_code, httplib.OK)
def test_download_object_invalid_file_already_exists(self): self.mock_response_klass.type = 'INVALID_SIZE' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) destination_path = os.path.abspath(__file__) try: self.driver.download_object(obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True) except LibcloudError: pass else: self.fail('Exception was not thrown')
def _make_object(self, container, object_name): """ Create an object instance @param container: Container. @type container: L{Container} @param object_name: Object name. @type object_name: C{str} @return: Object instance. @rtype: L{Object} """ full_path = os.path.join(self.base_path, container.name, object_name) if os.path.isdir(full_path): raise ObjectError(value=None, driver=self, object_name=object_name) try: stat = os.stat(full_path) except Exception: raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) # Make a hash for the file based on the metadata. We can safely # use only the mtime attribute here. If the file contents change, # the underlying file-system will change mtime data_hash = self._get_hash_function() data_hash.update(str(stat.st_mtime)) data_hash = data_hash.hexdigest() extra = {} extra['creation_time'] = stat.st_ctime extra['access_time'] = stat.st_atime extra['modify_time'] = stat.st_mtime return Object(name=object_name, size=stat.st_size, extra=extra, driver=self, container=container, hash=data_hash, meta_data=None)
def test_object_with_chinese_name(self): driver = OSSStorageDriver(*STORAGE_OSS_PARAMS) obj = Object(name='中文', size=0, hash=None, extra=None, meta_data=None, container=None, driver=driver) self.assertTrue(obj.__repr__() is not None)