def test_ex_get_object_temp_url(self, time): time.return_value = 0 self.driver.ex_get_meta_data = mock.Mock() self.driver.ex_get_meta_data.return_value = {'container_count': 1, 'object_count': 1, 'bytes_used': 1, 'temp_url_key': 'foo'} container = Container(name='foo_bar_container', extra={}, driver=self) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self) hmac_body = "%s\n%s\n%s" % ('GET', 60, "/v1/MossoCloudFS/foo_bar_container/foo_bar_object") sig = hmac.new(b('foo'), b(hmac_body), sha1).hexdigest() ret = self.driver.ex_get_object_temp_url(obj, 'GET') temp_url = 'https://storage101.%s1.clouddrive.com/v1/MossoCloudFS/foo_bar_container/foo_bar_object?temp_url_expires=60&temp_url_sig=%s' % (self.region, sig) self.assertEquals(''.join(sorted(ret)), ''.join(sorted(temp_url)))
def test_download_object_as_stream_success(self): container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object", size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type, ) stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) self.assertTrue(hasattr(stream, "__iter__"))
def test_download_object_invalid_file_size(self): self.mock_raw_response_klass.type = 'INVALID_SIZE' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) destination_path = os.path.abspath(__file__) + '.temp' result = self.driver.download_object(obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True) self.assertFalse(result)
def test_delete_object_not_found(self): self.mock_response_klass.type = 'NOT_FOUND' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1234, hash=None, extra=None, meta_data=None, container=container, driver=self.driver) try: self.driver.delete_object(obj=obj) except ObjectDoesNotExistError: pass else: self.fail('Exception was not thrown')
def test_delete_object_not_found(self): AtmosMockHttp.type = 'NOT_FOUND' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver) try: self.driver.delete_object(obj=obj) except ObjectDoesNotExistError: pass else: self.fail('Object does not exist but an exception was not thrown')
def test_ex_get_object_temp_url_no_key_raises_key_error(self): self.driver.ex_get_meta_data = mock.Mock() self.driver.ex_get_meta_data.return_value = { 'container_count': 1, 'object_count': 1, 'bytes_used': 1, 'temp_url_key': None } container = Container(name='foo_bar_container', extra={}, driver=self) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self) self.assertRaises(KeyError, self.driver.ex_get_object_temp_url, obj, 'GET')
def test_download_object(self): url = self.url_tmpl % ('/'.join( ['containers', 'foo_bar_container', 'objects', 'foo_bar_object'])) container = Container(name='foo_bar_container', extra={}, driver=None) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=None) with patch.object(CloudFilesUSStorageDriver, 'get_object', mocksignature=True) as get_object_mock: get_object_mock.return_value = obj resp = self.client.get(url, headers=self.headers) self.assertEqual(resp.status_code, httplib.OK)
def make_blob(self, blob_name, blob_date): extra = { 'creation_time': blob_date, 'access_time': blob_date, 'modify_time': blob_date } checksum = hashlib.md5() checksum.update(os.urandom(4)) return Object( name=blob_name, size=randrange(100), extra=extra, driver=self, container=None, hash=checksum.hexdigest(), meta_data=None )
def test_download_object_not_found(self): self.mock_raw_response_klass.type = 'not_found' container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) destination_path = os.path.abspath(__file__) + '.temp' self.assertRaises(ObjectDoesNotExistError, self.driver.download_object, obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True)
def test_download_object_success_not_found(self): CloudFilesMockRawResponse.type = 'NOT_FOUND' container = Container(name='foo_bar_container', extra={}, driver=self) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=CloudFilesStorageDriver) destination_path = os.path.abspath(__file__) + '.temp' try: self.driver.download_object( obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True) except ObjectDoesNotExistError: pass else: self.fail('Object does not exist but an exception was not thrown')
def _add_object(self, container, object_name, size, extra=None): container = self.get_container(container.name) extra = extra or {} meta_data = extra.get('meta_data', {}) meta_data.update({ 'cdn_url': 'http://www.test.com/object/%s' % (object_name.replace(' ', '_')) }) obj = Object(name=object_name, size=size, extra=extra, hash=None, meta_data=meta_data, container=container, driver=self) self._containers[container.name]['objects'][object_name] = obj return obj
def test_download_object_success(self): container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object", size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type, ) destination_path = os.path.abspath(__file__) + ".temp" result = self.driver.download_object( obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True, ) self.assertTrue(result)
def test_download_object_range_success(self): container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object_range', size=19, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) destination_path = self._file_path result = self.driver.download_object_range(obj=obj, destination_path=destination_path, start_bytes=5, end_bytes=7, overwrite_existing=True, delete_on_failure=True) self.assertTrue(result) with open(self._file_path, 'r') as fp: content = fp.read() self.assertEqual(content, '56')
def test_download_object_range_as_stream_success(self): container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object_range_stream", size=2, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type, ) stream = self.driver.download_object_range_as_stream( obj=obj, start_bytes=4, end_bytes=6, chunk_size=None ) consumed_stream = "".join(chunk.decode("utf-8") for chunk in stream) self.assertEqual(consumed_stream, "45") self.assertEqual(len(consumed_stream), obj.size)
def test_download_object_invalid_file_already_exists(self): S3MockRawResponse.type = 'INVALID_SIZE' container = Container(name='foo_bar_container', extra={}, driver=self) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=S3StorageDriver) destination_path = os.path.abspath(__file__) try: self.driver.download_object(obj=obj, destination_path=destination_path, overwrite_existing=False, delete_on_failure=True) except LibcloudError: pass else: self.fail('Exception was not thrown')
def test_download_object_as_stream_success(self): container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) def mock_get_object(self, obj, callback, callback_kwargs, response, success_status_code=None): return response._response.iter_content(1024) old_func = self.driver_type._get_object self.driver_type._get_object = mock_get_object try: stream = self.driver.download_object_as_stream(obj=obj, chunk_size=1024) self.assertTrue(hasattr(stream, '__iter__')) finally: self.driver_type._get_object = old_func
def _to_object_list(self, response, container): objects = [] for obj in response: name = obj['name'] size = int(obj['bytes']) hash = obj['hash'] extra = { 'content_type': obj['content_type'], 'last_modified': obj['last_modified'] } objects.append( Object(name=name, size=size, hash=hash, extra=extra, meta_data=None, container=container, driver=self)) return objects
def _add_object(self, container, object_name, size, extra=None): container = self.get_container(container.name) extra = extra or {} meta_data = extra.get("meta_data", {}) meta_data.update({ "cdn_url": "http://www.test.com/object/%s" % (object_name.replace(" ", "_")) }) obj = Object( name=object_name, size=size, extra=extra, hash=None, meta_data=meta_data, container=container, driver=self, ) self._containers[container.name]["objects"][object_name] = obj return obj
def list_container_objects(self, container): ret = None try: res = self.list_bucket(container.name) body = res.read() h = GetBucketXml(body).list() if len(h[0]) == 0: ret = [] return ret ret = [ Object(i[0], size=None, hash=None, extra=None, meta_data=None, container=container, driver=self) for i in h[0] ] return ret except: return ret
def upload_object(self, file_path, container, object_name, extra=None, verify_hash=True, ex_storage_class=None): try: res = self.put_object_from_file(container.name, object_name, file_path, 'text/HTML', {}) if (res.status / 100) == 2: obj = Object(object_name, None, None, None, None, container, self) return obj else: raise LibcloudError("Unexpected status code %s" % (res.status), driver=self) except HTTPError, e: raise LibcloudError("Unexpected HTTPError %s " % (e.code), driver=self)
def _make_object(self, container, object_name): """ Create an object instance :param container: Container. :type container: :class:`Container` :param object_name: Object name. :type object_name: ``str`` :return: Object instance. :rtype: :class:`Object` """ full_path = os.path.join(self.base_path, container.name, object_name) if os.path.isdir(full_path): raise ObjectError(value=None, driver=self, object_name=object_name) try: stat = os.stat(full_path) except Exception: raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) # Make a hash for the file based on the metadata. We can safely # use only the mtime attribute here. If the file contents change, # the underlying file-system will change mtime data_hash = self._get_hash_function() data_hash.update(u(stat.st_mtime).encode('ascii')) data_hash = data_hash.hexdigest() extra = {} extra['creation_time'] = stat.st_ctime extra['access_time'] = stat.st_atime extra['modify_time'] = stat.st_mtime return Object(name=object_name, size=stat.st_size, extra=extra, driver=self, container=container, hash=data_hash, meta_data=None)
def _headers_to_object(self, name, container, headers): size = int(headers.pop('content-length', 0)) last_modified = headers.pop('last-modified', None) etag = headers.pop('etag', None) content_type = headers.pop('content-type', None) meta_data = {} for key, value in list(headers.items()): if key.find('x-object-meta-') != -1: key = key.replace('x-object-meta-', '') meta_data[key] = value extra = {'content_type': content_type, 'last_modified': last_modified} obj = Object(name=name, size=size, hash=etag, extra=extra, meta_data=meta_data, container=container, driver=self) return obj
def _headers_to_object(self, object_name, container, headers): hash = headers.get("etag", "").replace('"', "") extra = {"content_type": headers["content-type"]} if "etag" in headers: extra["etag"] = headers["etag"] meta_data = {} if "content-encoding" in headers: extra["content_encoding"] = headers["content-encoding"] if "last-modified" in headers: extra["last_modified"] = headers["last-modified"] for key, value in headers.items(): if not key.lower().startswith(self.http_vendor_prefix + "-meta-"): continue key = key.replace(self.http_vendor_prefix + "-meta-", "") meta_data[key] = value content_length = self._get_content_length_from_headers(headers=headers) if content_length is None: raise KeyError( "Can not deduce object size from headers for " "object %s" % (object_name) ) obj = Object( name=object_name, size=int(content_length), hash=hash, extra=extra, meta_data=meta_data, container=container, driver=self, ) return obj
def _headers_to_object(self, object_name, container, headers): hash = headers['etag'].replace('"', '') extra = {'content_type': headers['content-type'], 'etag': headers['etag']} meta_data = {} if 'last-modified' in headers: extra['last_modified'] = headers['last-modified'] for key, value in headers.items(): if not key.lower().startswith(self.http_vendor_prefix + '-meta-'): continue key = key.replace(self.http_vendor_prefix + '-meta-', '') meta_data[key] = value obj = Object(name=object_name, size=headers['content-length'], hash=hash, extra=extra, meta_data=meta_data, container=container, driver=self) return obj
def _to_obj(self, element, container): owner_id = findtext(element=element, xpath='Owner/ID', namespace=self.namespace) owner_display_name = findtext(element=element, xpath='Owner/DisplayName', namespace=self.namespace) meta_data = {'owner': {'id': owner_id, 'display_name': owner_display_name}} obj = Object(name=findtext(element=element, xpath='Key', namespace=self.namespace), size=int(findtext(element=element, xpath='Size', namespace=self.namespace)), hash=findtext(element=element, xpath='ETag', namespace=self.namespace).replace('"', ''), extra=None, meta_data=meta_data, container=container, driver=self ) return obj
def _upload_object_manifest(self, container, object_name, extra=None, verify_hash=True): extra = extra or {} meta_data = extra.get('meta_data') container_name_encoded = self._encode_container_name(container.name) object_name_encoded = self._encode_object_name(object_name) request_path = '/%s/%s' % (container_name_encoded, object_name_encoded) # pylint: disable=no-member headers = {'X-Auth-Token': self.connection.auth_token, 'X-Object-Manifest': '%s/%s/' % (container_name_encoded, object_name_encoded)} data = '' response = self.connection.request(request_path, method='PUT', data=data, headers=headers, raw=True) object_hash = None if verify_hash: hash_function = self._get_hash_function() hash_function.update(b(data)) data_hash = hash_function.hexdigest() object_hash = response.headers.get('etag') if object_hash != data_hash: raise ObjectHashMismatchError( value=('MD5 hash checksum does not match (expected=%s, ' + 'actual=%s)') % (data_hash, object_hash), object_name=object_name, driver=self) obj = Object(name=object_name, size=0, hash=object_hash, extra=None, meta_data=meta_data, container=container, driver=self) return obj
def test_download_object_as_stream_data_is_not_buffered_in_memory(self): # Test case which verifies that response.response attribute is not accessed # and as such, whole body response is not buffered into RAM # If content is consumed and response.content attribute accessed exception # will be thrown and test will fail mock_response = Mock(name='mock response') mock_response.headers = {} mock_response.status = 200 msg1 = '"response" attribute was accessed but it shouldn\'t have been' msg2 = '"content" attribute was accessed but it shouldn\'t have been' type(mock_response).response = PropertyMock( name='mock response attribute', side_effect=Exception(msg1)) type(mock_response).content = PropertyMock( name='mock content attribute', side_effect=Exception(msg2)) mock_response.iter_content.return_value = StringIO('a' * 1000) self.driver.connection.request = Mock() self.driver.connection.request.return_value = mock_response container = Container(name='foo_bar_container', extra={}, driver=self.driver) obj = Object(name='foo_bar_object_NO_BUFFER', size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type) destination_path = self._file_path result = self.driver.download_object_as_stream(obj=obj) result = exhaust_iterator(result) if PY3: result = result.decode('utf-8') self.assertEqual(result, 'a' * 1000)
def test_download_object_data_is_not_buffered_in_memory(self): # Test case which verifies that response.body attribute is not accessed # and as such, whole body response is not buffered into RAM # If content is consumed and response.content attribute accessed execption # will be thrown and test will fail mock_response = Mock(name="mock response") mock_response.headers = {} mock_response.status_code = 200 msg = '"content" attribute was accessed but it shouldn\'t have been' type(mock_response).content = PropertyMock( name="mock content attribute", side_effect=Exception(msg)) mock_response.iter_content.return_value = StringIO("a" * 1000) self.driver.connection.connection.getresponse = Mock() self.driver.connection.connection.getresponse.return_value = mock_response container = Container(name="foo_bar_container", extra={}, driver=self.driver) obj = Object( name="foo_bar_object_NO_BUFFER", size=1000, hash=None, extra={}, container=container, meta_data=None, driver=self.driver_type, ) destination_path = self._file_path result = self.driver.download_object( obj=obj, destination_path=destination_path, overwrite_existing=True, delete_on_failure=True, ) self.assertTrue(result)
def _make_object(self, container, object_name): """ Create an object instance @param container: Container. @type container: L{Container} @param object_name: Object name. @type object_name: C{str} @return: Object instance. @rtype: L{Object} """ full_path = os.path.join(self.base_path, container.name, object_name) if os.path.isdir(full_path): raise ObjectError(value=None, driver=self, object_name=object_name) try: stat = os.stat(full_path) except Exception: raise ObjectDoesNotExistError(value=None, driver=self, object_name=object_name) extra = {} extra['creation_time'] = stat.st_ctime extra['access_time'] = stat.st_atime extra['modify_time'] = stat.st_mtime return Object(name=object_name, size=stat.st_size, extra=extra, driver=self, container=container, hash=None, meta_data=None)
def test_download_object_range_invalid_values(self): obj = Object('a', 500, '', {}, {}, None, None) tmppath = self.make_tmp_file(content='') expected_msg = 'start_bytes must be greater than 0' self.assertRaisesRegex(ValueError, expected_msg, self.driver.download_object_range, obj=obj, destination_path=tmppath, start_bytes=-1) expected_msg = 'start_bytes must be smaller than end_bytes' self.assertRaisesRegex(ValueError, expected_msg, self.driver.download_object_range, obj=obj, destination_path=tmppath, start_bytes=5, end_bytes=4) expected_msg = 'start_bytes and end_bytes can\'t be the same' self.assertRaisesRegex(ValueError, expected_msg, self.driver.download_object_range, obj=obj, destination_path=tmppath, start_bytes=5, end_bytes=5)