def head_object(self, container, obj, headers=None, query_string=None): path = self.clean_path(container, obj) if not path.exists(): from swiftclient.exceptions import ClientException raise ClientException(f"No such path: {path!s}") try: max_date = max(path.stat().st_mtime, path.stat().st_ctime) current_timestamp = get_swift_object_date(datetime.datetime.now()) path_contents = path.read_bytes() except Exception: from swiftclient.exceptions import ClientException raise ClientException(f"Not a file: {path!s}") name = path.as_posix() mimetype, encoding = mimetypes.guess_type(name) if mimetype is None: mimetype = "application/octet-stream" if encoding is not None: mimetype = f"{mimetype}; encoding={encoding}" return { "date": current_timestamp, "server": "Apache/2.4.29 (Ubuntu)", "content-length": "{}".format(len(path_contents)), "accept-ranges": "bytes", "last-modified": f"{path.stat().st_mtime}", "etag": hashlib.md5(path_contents).hexdigest(), "x-timestamp": f"{max_date}", "content-type": mimetype, "x-trans-id": "txfcbf2e82791411eaa6bd-cf51efeb8527", "x-openstack-request-id": "txfcbf2e82791411eaa6bd-cf51efeb8527", }
def get_object(self, container_name, filename): if self.unable_to_connect: raise ClientException("Mocked unable to connect") try: return self.store[container_name][filename] except KeyError: raise ClientException('Mocked')
def put_object(self, container_name, filename, data): if self.unable_to_connect: raise ClientException("Mocked unable to connect") if self.unable_to_find_file: raise ClientException("Mocked resource could not be found") self.store[container_name].setdefault(filename, {}) self.store[container_name][filename] = ({}, bytes(data, 'utf-8'))
def test_invalid_auth(self): rv = self.app.post(self.url) self.assertEqual(401, rv.status_code) with patch('git_lfs_swift_server.server.client.get_auth') as m: m.side_effect = ClientException('', http_status=401) rv = self.app.post(self.url, headers=self.headers) self.assertEqual(401, rv.status_code) m.side_effect = ClientException('', http_status=402) rv = self.app.post(self.url, headers=self.headers) self.assertEqual(500, rv.status_code)
def head_container(url, token, container): path = _get_vault_path(container) if os.path.exists(path): response_headers = {} response_headers['content-type'] = 'mocking_ret' response_headers['x-container-bytes-used'] = 0 response_headers['x-container-object-count'] = 0 # If we really wanted to be pendantic about this field # then we'd set this to zero and find the epoch to 5 decimals # on each file below and take the latest (max) value between # them all response_headers['x-timestamp'] = 987654321.12345 total_size = 0 object_count = 0 for root, dirs, files in os.walk(path): total_size = total_size + sum( os.path.getsize(os.path.join(root, name)) for name in files) object_count = object_count + len(files) response_headers['x-container-bytes-used'] = total_size response_headers['x-container-object-count'] = object_count return response_headers else: raise ClientException('mocking')
def test_connection_client_exception(self, settings_mock, mock_connection): auth_token = "HPAUTH_9787665544434434" error_message = 'SWIFT CLIENT ERROR' mock_connection.side_effect = ClientException(error_message) result = connection(auth_token) self.assertRaises(ClientException, mock_connection, error_message)
def get_object(url, token, container, name, response_dict): path = _get_block_path(container, name) if not os.path.exists(path): raise ClientException('mocking') hdrs = {} buff = '' try: with open(path, 'rb') as infile: buff = infile.read() mdhash = hashlib.md5() mdhash.update(buff) etag = mdhash.hexdigest() hdrs['content-length'] = os.path.getsize(path) hdrs['last-modified'] = os.path.getmtime(path) hdrs['accept-ranges'] = 'bytes' hdrs['etag'] = etag except: pass response_dict['status'] = _mock_status_code() return hdrs, buff
def get_object(self, container, obj): path = self.clean_path(container, obj) if not path.exists(): from swiftclient.exceptions import ClientException raise ClientException(f"No such path: {path!s}") return {}, path.read_bytes()
def test_get_package_manifests_client_exception( self, mock_get_auth_token, mock_settings): error_message = 'ERROR' mock_get_auth_token.side_effect = ClientException(error_message) result = get_package_manifests() self.assertEqual(result['status'], 500)
def get_swift_file_attrs( self, path: pathlib.Path, container: str = "" ) -> TFILEHEADER_DICT: if not path.is_absolute(): path = self.get_path(container, key=path) if not path.exists(): from swiftclient.exceptions import ClientException raise ClientException(f"No such file: {path!s}") last_modified = get_swift_date( datetime.datetime.fromtimestamp(path.stat().st_mtime) ) data = path.read_bytes() name = str(self.get_relative_path(container, path)) mimetype, encoding = mimetypes.guess_type(str(path)) if mimetype is None: mimetype = "application/octet-stream" if encoding is not None: mimetype = f"{mimetype}; encoding={encoding}" extra_headers = self.get_path_metadata(container.strip("/"), name.lstrip("/")) content_type = extra_headers.get("content-type", mimetype) result_dict = { "bytes": len(data), "hash": hashlib.md5(data).hexdigest(), "name": type(path)(name.lstrip("/")), "content_type": content_type, "last_modified": last_modified, } return result_dict
def head_container(self, container): if container == valid_container: return file_to_dict( 'tests/data/openstack/swift.head_container.txt') elif container == invalid_container: raise ClientException(**file_to_dict( 'tests/data/openstack/swift.head_container.notfound.txt'))
def put_container(self, account_name=None, container_name=None, headers=None, response_dict=None): """ HTTP PUT Container handler.""" container_partition, containers = self.container_ring.get_nodes(account_name, container_name) statuses = [] for i in range(self.upload_replica_num): container_url = 'http://%s:%d/%s/%d/%s/%s' % (containers[0]['ip'], containers[0]['port'], containers[0]['device'], container_partition, account_name, container_name) parsed = urlparse(container_url) path = parsed.path http_url = parsed.scheme + '://' + parsed.netloc conn = HTTPConnection(http_url) if headers: headers = dict(headers) else: headers = {} headers['X-Timestamp'] = normalize_timestamp(time.time()) conn.request('PUT', path, '', headers) resp = conn.getresponse() body = resp.read() store_response(resp, response_dict) http_log(('%s%s' % (container_url.replace(parsed.path, ''), path), 'PUT',), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container PUT failed', body) statuses.append(resp.status) return statuses
def head_container(url, token, container): headers = {'X-Auth-Token': token} response = _request('HEAD', url + '/' + container, headers=headers) if response.status >= 200 and response.status < 300: return response.headers else: raise ClientException("Vault HEAD failed")
def test_swift_config_client_exception(self, mock_connection): error_message = 'SWIFT CLIENT ERROR' mock_connection.side_effect = ClientException(error_message) with self.assertRaises(ClientException) as cm: result = swift.SwiftConfig(self.auth_token, self.swift_url, self.container) self.assertEqual(str(cm.exception), error_message)
def test_check_container_missing_client_exception(self): error_message = 'SWIFT CLIENT ERROR' self.swift_cfg.connection.get_account.side_effect \ = ClientException(error_message) with self.assertRaises(ClientException) as cm: result = swift.check_container_missing(config=self.swift_cfg) self.assertEqual(str(cm.exception), error_message) self.assertFalse(result)
def delete_object(url, token, container, name, response_dict): path = _get_block_path(container, name) if os.path.exists(path): os.remove(path) response_dict['status'] = 201 else: raise ClientException('mocking')
def test_get_package_manifests_404( self, mock_connection, mock_get_auth_token, mock_settings): error_message = 'CONNECTION ERROR' mock_connection.side_effect = ClientException(http_status=404, msg=error_message) result = get_package_manifests() self.assertEqual(result['status'], 404)
def test_stat_container_failed(self): self.mock_object( self.fake_driver.client, 'head_container', mock.Mock(side_effect=ClientException('Container HEAD failed'))) self.assertRaises(ClientException, self.fake_driver.stat_container, 'invalid-container') self.fake_driver.client.head_container.\ assert_called_once_with('invalid-container')
def test_delete_container_failed(self): self.mock_object( self.fake_driver.client, 'delete_container', mock.Mock(side_effect=ClientException('Container DELETE failed'))) self.assertRaises(ClientException, self.fake_driver.delete_container, 'invalid-container') self.fake_driver.client.delete_container.\ assert_called_once_with('invalid-container')
def _validate_obj( self, path: pathlib.Path, etag: Optional[str], content_length: Optional[int] ) -> None: if etag: calculated_etag = hashlib.md5(path.read_bytes()).hexdigest() if calculated_etag != etag: from swiftclient.exceptions import ClientException raise ClientException( f"ETag value mismatch: {calculated_etag} != {etag}" ) if content_length and len(path.read_bytes()) != content_length: from swiftclient.exceptions import ClientException raise ClientException( f"Content does not match expected length: {len(path.read_bytes())} != " f"{content_length}" ) return None
async def test_swift_get(): with mock.patch("swiftclient.Connection") as connection_mock: connection_mock.return_value = Mock() connection_mock().head_container.return_value = True blockstore = SwiftBlockStoreComponent("http://url", "scille", "parsec", "john", "secret") # Ok connection_mock().get_object.return_value = True, "content" assert await blockstore.read("org42", 123) == "content" # Not found connection_mock().get_object.side_effect = ClientException( http_status=404, msg="") with pytest.raises(BlockNotFoundError): assert await blockstore.read("org42", 123) # Other exception connection_mock().get_object.side_effect = ClientException( http_status=500, msg="") with pytest.raises(BlockTimeoutError): assert await blockstore.read("org42", 123)
def test_verify_container_missing_client_exception(self): name = 'foo' connection = mock.Mock() error_message = 'SWIFT CLIENT ERROR' connection.get_account.side_effect = ClientException(error_message) result = verify_container_missing(connection, name) self.assertRaises(ClientException, connection.get_account, error_message) self.assertFalse(result)
async def test_swift_create(caplog): org_id = OrganizationID("org42") block_id = BlockID.from_hex("0694a21176354e8295e28a543e5887f9") def _assert_log(): log = caplog.assert_occured_once("[warning ] Block create error") assert f"organization_id={org_id}" in log assert f"block_id={block_id}" in log assert len(caplog.messages) == 1 caplog.clear() with mock.patch("swiftclient.Connection") as connection_mock: connection_mock.return_value = Mock() connection_mock().head_container.return_value = True blockstore = SwiftBlockStoreComponent("http://url", "scille", "parsec", "john", "secret") # Ok connection_mock().get_object.side_effect = ClientException( http_status=404, msg="") await blockstore.create(org_id, block_id, "content") connection_mock().put_object.assert_called_with( "parsec", "org42/0694a211-7635-4e82-95e2-8a543e5887f9", "content") connection_mock().put_object.reset_mock() assert not caplog.messages # Connection error at PUT connection_mock().get_object.side_effect = ClientException( msg="Connection error") connection_mock().put_object.side_effect = ClientException( msg="Connection error") with pytest.raises(BlockStoreError): await blockstore.create(org_id, block_id, "content") _assert_log() # Unknown exception at PUT connection_mock().put_object.side_effect = ClientException( http_status=500, msg="") with pytest.raises(BlockStoreError): await blockstore.create(org_id, block_id, "content") _assert_log()
def test_update_container_failed(self): self.mock_object( self.fake_driver.client, 'post_container', mock.Mock(side_effect=ClientException('Container POST failed'))) self.assertRaises(ClientException, self.fake_driver.update_container, 'invalid-container', {'newkey': 'newvalue'}) self.fake_driver.client.post_container.\ assert_called_once_with( 'invalid-container', {'x-container-meta-newkey': 'newvalue'} )
def delete_container(url, token, container, response_dict): try: # Basic response requirements response_dict['content-length'] = 0 response_dict['content-type'] = 'text/html; charset=UTF-8' response_dict['x-transaction-id'] = 'req-' + str(uuid.uuid4()) # Thu, 16 Jan 2014 18:04:04 GMT response_dict['date'] = datetime.datetime.utcnow().strftime( "%a, %d %b %Y %H:%M:%S %Z") path = _get_vault_path(container) blockpath = _get_vault_block_path(container) response_dict['x-vault-path'] = path response_dict['x-block-path'] = blockpath if os.path.exists(path): if os.path.exists(blockpath): if os.listdir(blockpath) != []: # container not empty response_dict['x-error-message'] = 'container not empty' response_dict['status'] = 409 else: # no blocks we're okay # container exists and is empty (no blocks) shutil.rmtree(path) response_dict['status'] = 204 elif os.listdir(path) == []: # container exists and is empty (no blocks) shutil.rmtree(path) response_dict['status'] = 204 else: # else there is some other issue response_dict['x-error-message'] = 'mocking: listing directory' response_dict['status'] = 500 assert False else: # Container does not exist response_dict['x-error-message'] = 'vault does not exist' response_dict['status'] = 404 except Exception as ex: response_dict['x-error-message'] = 'mocking error: {0:}'.format(ex) response_dict['status'] = 500 raise ClientException('mocking error: {0:}'.format(ex))
def test_download_object_failed(self): self.mock_object( self.fake_driver.client, 'get_object', mock.Mock(side_effect=ClientException('Object GET failed'))) self.assertRaises(ClientException, self.fake_driver.download_object, 'invalid-container', 'invalid-obj') self.fake_driver.client.get_object.\ assert_called_once_with( 'invalid-container', 'invalid-obj' )
def put_container(url, token, container, response_dict): path = _get_vault_path(container) if not os.path.exists(path): shutil.os.makedirs(path) else: raise ClientException('mocking') block_path = _get_vault_block_path(container) if not os.path.exists(block_path): shutil.os.makedirs(block_path) response_dict['status'] = 201
def test_ensure_addins_container_exists_client_exception( self, mock_check_container, mock_swift_connection): container_name = "dummy_container_name" error_message = 'SWIFT CLIENT ERROR' with mock.patch.object( mock_swift_connection, 'put_container', side_effect=ClientException(error_message)) as mocked_put: ensure_addins_container_exists(mock_swift_connection, container_name) self.assertRaises(ClientException, mocked_put, error_message)
def test_cache_delete(self, mock_conn, mock_client, mock_endpoint): mock_swift = mock.Mock() mock_conn.return_value = mock_swift cache_container = "__cache__" container = "TestContainer" key = "testkey" cache_key = "__cache_TestContainer_testkey" mock_swift.delete_object.side_effect = ClientException("Foo") self.action.cache_delete(container, key) mock_swift.delete_object.assert_called_once_with( cache_container, cache_key)
def get_object( self, container: str, obj: str, resp_chunk_size: Optional[int] = None, query_string: Optional[str] = None, response_dict: Optional[Dict[str, str]] = None, headers: Optional[Dict[str, str]] = None, attempts: Optional[int] = None, ) -> Tuple[Dict[str, str], Union[bytes, MockRetryBody, None]]: from swiftclient.client import get_object self._retry( None, get_object, container, obj, resp_chunk_size=resp_chunk_size, query_string=query_string, response_dict=response_dict, headers=headers, ) path = self.get_path(container, key=obj) if attempts is not None: self.attempts = attempts if not path.exists(): from swiftclient.exceptions import ClientException raise ClientException(f"No such path: {path!s}") headers = self._head_object( container, obj, headers=headers, query_string=query_string ) if "X-Symlink-Target" in headers: container, _, obj = headers["X-Symlink-Target"].lstrip("/").partition("/") container = container.strip("/") obj = obj.lstrip("/") path = self.get_path(container, key=obj) resp = MockRetryBody( self, container, obj, path, resp_chunk_size=resp_chunk_size, query_string=query_string, response_dict=response_dict, headers=headers, ) if not resp_chunk_size: content = resp.read() assert isinstance(content, bytes) return headers, content return headers, resp
def test_write_package_client_exception(self, mock_settings, mock_put_object, mock_extract_manifest, mock_connection, mock_get_auth_token): name = "thepackage/manifest.json" filedata = mock.Mock() error_message = 'SAVE TO SWIFT ERROR' mock_connection.side_effect = ClientException(error_message) result = write_package(name, filedata) self.assertRaises(ClientException, mock_connection, error_message)
def put_object(self, account_name=None, container_name=None, object_name=None, uploadfile_path=None, headers=None, response_dict=None): """HTTP PUT Object handler. First to fetch container info to update the header, or you will not find object info in container even you put a container by manual""" container_partition, containers = self.container_ring.get_nodes(account_name, container_name) container_info = {'X-Container-Host': str(containers[0]['ip']) + ':' + str(containers[0]['port']), 'X-Container-Device': containers[0]['device'], 'X-Container-Partition': container_partition} part, nodes = self.object_ring.get_nodes(account_name, container_name, object_name) # there may be incompatibility problem, or just use file read file_data = StringIO.StringIO() with open(uploadfile_path, 'r') as upload_file: file_data.write(upload_file.read()) # not concurrent http_connect statuses = [] for i in range(self.upload_replica_num): object_url = 'http://%s:%d/%s/%d/%s/%s/%s' % (nodes[i]['ip'], nodes[i]['port'], nodes[i]['device'], part, account_name, container_name, object_name) parsed = urlparse(object_url) path = parsed.path http_url = parsed.scheme + '://' + parsed.netloc conn = HTTPConnection(http_url) if headers: headers = dict(headers) else: headers = {} headers['x-timestamp'] = normalize_timestamp(time.time()) headers['Content-Type'] = 'application/octet-stream' headers.update(container_info) conn.request('PUT', path, file_data.getvalue(), headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (object_url.replace(parsed.path, ''), path), 'PUT',), {'headers': headers}, resp, body) store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Object PUT failed', body) # etag = resp.getheader('etag', '').strip('"') statuses.append(resp.status) return statuses