def head_object(self, container, obj, headers=None, query_string=None):
path = self.clean_path(container, obj)
if not path.exists():
from swiftclient.exceptions import ClientException
raise ClientException(f"No such path: {path!s}")
try:
max_date = max(path.stat().st_mtime, path.stat().st_ctime)
current_timestamp = get_swift_object_date(datetime.datetime.now())
path_contents = path.read_bytes()
except Exception:
from swiftclient.exceptions import ClientException
raise ClientException(f"Not a file: {path!s}")
name = path.as_posix()
mimetype, encoding = mimetypes.guess_type(name)
if mimetype is None:
mimetype = "application/octet-stream"
if encoding is not None:
mimetype = f"{mimetype}; encoding={encoding}"
return {
"date": current_timestamp,
"server": "Apache/2.4.29 (Ubuntu)",
"content-length": "{}".format(len(path_contents)),
"accept-ranges": "bytes",
"last-modified": f"{path.stat().st_mtime}",
"etag": hashlib.md5(path_contents).hexdigest(),
"x-timestamp": f"{max_date}",
"content-type": mimetype,
"x-trans-id": "txfcbf2e82791411eaa6bd-cf51efeb8527",
"x-openstack-request-id": "txfcbf2e82791411eaa6bd-cf51efeb8527",
}
def get_object(self, container_name, filename):
if self.unable_to_connect:
raise ClientException("Mocked unable to connect")
try:
return self.store[container_name][filename]
except KeyError:
raise ClientException('Mocked')
def put_object(self, container_name, filename, data):
if self.unable_to_connect:
raise ClientException("Mocked unable to connect")
if self.unable_to_find_file:
raise ClientException("Mocked resource could not be found")
self.store[container_name].setdefault(filename, {})
self.store[container_name][filename] = ({}, bytes(data, 'utf-8'))
def test_invalid_auth(self):
rv = self.app.post(self.url)
self.assertEqual(401, rv.status_code)
with patch('git_lfs_swift_server.server.client.get_auth') as m:
m.side_effect = ClientException('', http_status=401)
rv = self.app.post(self.url, headers=self.headers)
self.assertEqual(401, rv.status_code)
m.side_effect = ClientException('', http_status=402)
rv = self.app.post(self.url, headers=self.headers)
self.assertEqual(500, rv.status_code)
def head_container(url, token, container):
path = _get_vault_path(container)
if os.path.exists(path):
response_headers = {}
response_headers['content-type'] = 'mocking_ret'
response_headers['x-container-bytes-used'] = 0
response_headers['x-container-object-count'] = 0
# If we really wanted to be pendantic about this field
# then we'd set this to zero and find the epoch to 5 decimals
# on each file below and take the latest (max) value between
# them all
response_headers['x-timestamp'] = 987654321.12345
total_size = 0
object_count = 0
for root, dirs, files in os.walk(path):
total_size = total_size + sum(
os.path.getsize(os.path.join(root, name)) for name in files)
object_count = object_count + len(files)
response_headers['x-container-bytes-used'] = total_size
response_headers['x-container-object-count'] = object_count
return response_headers
else:
raise ClientException('mocking')
def test_connection_client_exception(self, settings_mock, mock_connection):
auth_token = "HPAUTH_9787665544434434"
error_message = 'SWIFT CLIENT ERROR'
mock_connection.side_effect = ClientException(error_message)
result = connection(auth_token)
self.assertRaises(ClientException, mock_connection, error_message)
def get_object(url, token, container, name, response_dict):
path = _get_block_path(container, name)
if not os.path.exists(path):
raise ClientException('mocking')
hdrs = {}
buff = ''
try:
with open(path, 'rb') as infile:
buff = infile.read()
mdhash = hashlib.md5()
mdhash.update(buff)
etag = mdhash.hexdigest()
hdrs['content-length'] = os.path.getsize(path)
hdrs['last-modified'] = os.path.getmtime(path)
hdrs['accept-ranges'] = 'bytes'
hdrs['etag'] = etag
except:
pass
response_dict['status'] = _mock_status_code()
return hdrs, buff
def get_object(self, container, obj):
path = self.clean_path(container, obj)
if not path.exists():
from swiftclient.exceptions import ClientException
raise ClientException(f"No such path: {path!s}")
return {}, path.read_bytes()
def test_get_package_manifests_client_exception(
self, mock_get_auth_token, mock_settings):
error_message = 'ERROR'
mock_get_auth_token.side_effect = ClientException(error_message)
result = get_package_manifests()
self.assertEqual(result['status'], 500)
def get_swift_file_attrs(
self, path: pathlib.Path, container: str = ""
) -> TFILEHEADER_DICT:
if not path.is_absolute():
path = self.get_path(container, key=path)
if not path.exists():
from swiftclient.exceptions import ClientException
raise ClientException(f"No such file: {path!s}")
last_modified = get_swift_date(
datetime.datetime.fromtimestamp(path.stat().st_mtime)
)
data = path.read_bytes()
name = str(self.get_relative_path(container, path))
mimetype, encoding = mimetypes.guess_type(str(path))
if mimetype is None:
mimetype = "application/octet-stream"
if encoding is not None:
mimetype = f"{mimetype}; encoding={encoding}"
extra_headers = self.get_path_metadata(container.strip("/"), name.lstrip("/"))
content_type = extra_headers.get("content-type", mimetype)
result_dict = {
"bytes": len(data),
"hash": hashlib.md5(data).hexdigest(),
"name": type(path)(name.lstrip("/")),
"content_type": content_type,
"last_modified": last_modified,
}
return result_dict
def head_container(self, container):
if container == valid_container:
return file_to_dict(
'tests/data/openstack/swift.head_container.txt')
elif container == invalid_container:
raise ClientException(**file_to_dict(
'tests/data/openstack/swift.head_container.notfound.txt'))
def put_container(self, account_name=None, container_name=None, headers=None, response_dict=None):
""" HTTP PUT Container handler."""
container_partition, containers = self.container_ring.get_nodes(account_name, container_name)
statuses = []
for i in range(self.upload_replica_num):
container_url = 'http://%s:%d/%s/%d/%s/%s' % (containers[0]['ip'], containers[0]['port'],
containers[0]['device'], container_partition,
account_name, container_name)
parsed = urlparse(container_url)
path = parsed.path
http_url = parsed.scheme + '://' + parsed.netloc
conn = HTTPConnection(http_url)
if headers:
headers = dict(headers)
else:
headers = {}
headers['X-Timestamp'] = normalize_timestamp(time.time())
conn.request('PUT', path, '', headers)
resp = conn.getresponse()
body = resp.read()
store_response(resp, response_dict)
http_log(('%s%s' % (container_url.replace(parsed.path, ''), path), 'PUT',),
{'headers': headers}, resp, body)
if resp.status < 200 or resp.status >= 300:
raise ClientException.from_response(resp, 'Container PUT failed', body)
statuses.append(resp.status)
return statuses
def head_container(url, token, container):
headers = {'X-Auth-Token': token}
response = _request('HEAD', url + '/' + container, headers=headers)
if response.status >= 200 and response.status < 300:
return response.headers
else:
raise ClientException("Vault HEAD failed")
def test_swift_config_client_exception(self, mock_connection):
error_message = 'SWIFT CLIENT ERROR'
mock_connection.side_effect = ClientException(error_message)
with self.assertRaises(ClientException) as cm:
result = swift.SwiftConfig(self.auth_token, self.swift_url,
self.container)
self.assertEqual(str(cm.exception), error_message)
def test_check_container_missing_client_exception(self):
error_message = 'SWIFT CLIENT ERROR'
self.swift_cfg.connection.get_account.side_effect \
= ClientException(error_message)
with self.assertRaises(ClientException) as cm:
result = swift.check_container_missing(config=self.swift_cfg)
self.assertEqual(str(cm.exception), error_message)
self.assertFalse(result)
def delete_object(url, token, container, name, response_dict):
path = _get_block_path(container, name)
if os.path.exists(path):
os.remove(path)
response_dict['status'] = 201
else:
raise ClientException('mocking')
def test_get_package_manifests_404(
self, mock_connection, mock_get_auth_token, mock_settings):
error_message = 'CONNECTION ERROR'
mock_connection.side_effect = ClientException(http_status=404,
msg=error_message)
result = get_package_manifests()
self.assertEqual(result['status'], 404)
def test_stat_container_failed(self):
self.mock_object(
self.fake_driver.client, 'head_container',
mock.Mock(side_effect=ClientException('Container HEAD failed')))
self.assertRaises(ClientException, self.fake_driver.stat_container,
'invalid-container')
self.fake_driver.client.head_container.\
assert_called_once_with('invalid-container')
def test_delete_container_failed(self):
self.mock_object(
self.fake_driver.client, 'delete_container',
mock.Mock(side_effect=ClientException('Container DELETE failed')))
self.assertRaises(ClientException, self.fake_driver.delete_container,
'invalid-container')
self.fake_driver.client.delete_container.\
assert_called_once_with('invalid-container')
def _validate_obj(
self, path: pathlib.Path, etag: Optional[str], content_length: Optional[int]
) -> None:
if etag:
calculated_etag = hashlib.md5(path.read_bytes()).hexdigest()
if calculated_etag != etag:
from swiftclient.exceptions import ClientException
raise ClientException(
f"ETag value mismatch: {calculated_etag} != {etag}"
)
if content_length and len(path.read_bytes()) != content_length:
from swiftclient.exceptions import ClientException
raise ClientException(
f"Content does not match expected length: {len(path.read_bytes())} != "
f"{content_length}"
)
return None
async def test_swift_get():
with mock.patch("swiftclient.Connection") as connection_mock:
connection_mock.return_value = Mock()
connection_mock().head_container.return_value = True
blockstore = SwiftBlockStoreComponent("http://url", "scille", "parsec",
"john", "secret")
# Ok
connection_mock().get_object.return_value = True, "content"
assert await blockstore.read("org42", 123) == "content"
# Not found
connection_mock().get_object.side_effect = ClientException(
http_status=404, msg="")
with pytest.raises(BlockNotFoundError):
assert await blockstore.read("org42", 123)
# Other exception
connection_mock().get_object.side_effect = ClientException(
http_status=500, msg="")
with pytest.raises(BlockTimeoutError):
assert await blockstore.read("org42", 123)
def test_verify_container_missing_client_exception(self):
name = 'foo'
connection = mock.Mock()
error_message = 'SWIFT CLIENT ERROR'
connection.get_account.side_effect = ClientException(error_message)
result = verify_container_missing(connection, name)
self.assertRaises(ClientException, connection.get_account,
error_message)
self.assertFalse(result)
async def test_swift_create(caplog):
org_id = OrganizationID("org42")
block_id = BlockID.from_hex("0694a21176354e8295e28a543e5887f9")
def _assert_log():
log = caplog.assert_occured_once("[warning ] Block create error")
assert f"organization_id={org_id}" in log
assert f"block_id={block_id}" in log
assert len(caplog.messages) == 1
caplog.clear()
with mock.patch("swiftclient.Connection") as connection_mock:
connection_mock.return_value = Mock()
connection_mock().head_container.return_value = True
blockstore = SwiftBlockStoreComponent("http://url", "scille", "parsec",
"john", "secret")
# Ok
connection_mock().get_object.side_effect = ClientException(
http_status=404, msg="")
await blockstore.create(org_id, block_id, "content")
connection_mock().put_object.assert_called_with(
"parsec", "org42/0694a211-7635-4e82-95e2-8a543e5887f9", "content")
connection_mock().put_object.reset_mock()
assert not caplog.messages
# Connection error at PUT
connection_mock().get_object.side_effect = ClientException(
msg="Connection error")
connection_mock().put_object.side_effect = ClientException(
msg="Connection error")
with pytest.raises(BlockStoreError):
await blockstore.create(org_id, block_id, "content")
_assert_log()
# Unknown exception at PUT
connection_mock().put_object.side_effect = ClientException(
http_status=500, msg="")
with pytest.raises(BlockStoreError):
await blockstore.create(org_id, block_id, "content")
_assert_log()
def test_update_container_failed(self):
self.mock_object(
self.fake_driver.client, 'post_container',
mock.Mock(side_effect=ClientException('Container POST failed')))
self.assertRaises(ClientException, self.fake_driver.update_container,
'invalid-container', {'newkey': 'newvalue'})
self.fake_driver.client.post_container.\
assert_called_once_with(
'invalid-container',
{'x-container-meta-newkey': 'newvalue'}
)
def delete_container(url, token, container, response_dict):
try:
# Basic response requirements
response_dict['content-length'] = 0
response_dict['content-type'] = 'text/html; charset=UTF-8'
response_dict['x-transaction-id'] = 'req-' + str(uuid.uuid4())
# Thu, 16 Jan 2014 18:04:04 GMT
response_dict['date'] = datetime.datetime.utcnow().strftime(
"%a, %d %b %Y %H:%M:%S %Z")
path = _get_vault_path(container)
blockpath = _get_vault_block_path(container)
response_dict['x-vault-path'] = path
response_dict['x-block-path'] = blockpath
if os.path.exists(path):
if os.path.exists(blockpath):
if os.listdir(blockpath) != []:
# container not empty
response_dict['x-error-message'] = 'container not empty'
response_dict['status'] = 409
else:
# no blocks we're okay
# container exists and is empty (no blocks)
shutil.rmtree(path)
response_dict['status'] = 204
elif os.listdir(path) == []:
# container exists and is empty (no blocks)
shutil.rmtree(path)
response_dict['status'] = 204
else:
# else there is some other issue
response_dict['x-error-message'] = 'mocking: listing directory'
response_dict['status'] = 500
assert False
else:
# Container does not exist
response_dict['x-error-message'] = 'vault does not exist'
response_dict['status'] = 404
except Exception as ex:
response_dict['x-error-message'] = 'mocking error: {0:}'.format(ex)
response_dict['status'] = 500
raise ClientException('mocking error: {0:}'.format(ex))
def test_download_object_failed(self):
self.mock_object(
self.fake_driver.client, 'get_object',
mock.Mock(side_effect=ClientException('Object GET failed')))
self.assertRaises(ClientException, self.fake_driver.download_object,
'invalid-container', 'invalid-obj')
self.fake_driver.client.get_object.\
assert_called_once_with(
'invalid-container',
'invalid-obj'
)
def put_container(url, token, container, response_dict):
path = _get_vault_path(container)
if not os.path.exists(path):
shutil.os.makedirs(path)
else:
raise ClientException('mocking')
block_path = _get_vault_block_path(container)
if not os.path.exists(block_path):
shutil.os.makedirs(block_path)
response_dict['status'] = 201
def test_ensure_addins_container_exists_client_exception(
self, mock_check_container, mock_swift_connection):
container_name = "dummy_container_name"
error_message = 'SWIFT CLIENT ERROR'
with mock.patch.object(
mock_swift_connection,
'put_container',
side_effect=ClientException(error_message)) as mocked_put:
ensure_addins_container_exists(mock_swift_connection,
container_name)
self.assertRaises(ClientException, mocked_put, error_message)
def test_cache_delete(self, mock_conn, mock_client, mock_endpoint):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
mock_swift.delete_object.side_effect = ClientException("Foo")
self.action.cache_delete(container, key)
mock_swift.delete_object.assert_called_once_with(
cache_container, cache_key)
def get_object(
self,
container: str,
obj: str,
resp_chunk_size: Optional[int] = None,
query_string: Optional[str] = None,
response_dict: Optional[Dict[str, str]] = None,
headers: Optional[Dict[str, str]] = None,
attempts: Optional[int] = None,
) -> Tuple[Dict[str, str], Union[bytes, MockRetryBody, None]]:
from swiftclient.client import get_object
self._retry(
None,
get_object,
container,
obj,
resp_chunk_size=resp_chunk_size,
query_string=query_string,
response_dict=response_dict,
headers=headers,
)
path = self.get_path(container, key=obj)
if attempts is not None:
self.attempts = attempts
if not path.exists():
from swiftclient.exceptions import ClientException
raise ClientException(f"No such path: {path!s}")
headers = self._head_object(
container, obj, headers=headers, query_string=query_string
)
if "X-Symlink-Target" in headers:
container, _, obj = headers["X-Symlink-Target"].lstrip("/").partition("/")
container = container.strip("/")
obj = obj.lstrip("/")
path = self.get_path(container, key=obj)
resp = MockRetryBody(
self,
container,
obj,
path,
resp_chunk_size=resp_chunk_size,
query_string=query_string,
response_dict=response_dict,
headers=headers,
)
if not resp_chunk_size:
content = resp.read()
assert isinstance(content, bytes)
return headers, content
return headers, resp
def test_write_package_client_exception(self, mock_settings,
mock_put_object,
mock_extract_manifest,
mock_connection,
mock_get_auth_token):
name = "thepackage/manifest.json"
filedata = mock.Mock()
error_message = 'SAVE TO SWIFT ERROR'
mock_connection.side_effect = ClientException(error_message)
result = write_package(name, filedata)
self.assertRaises(ClientException, mock_connection, error_message)
def put_object(self, account_name=None, container_name=None, object_name=None,
uploadfile_path=None, headers=None, response_dict=None):
"""HTTP PUT Object handler.
First to fetch container info to update the header,
or you will not find object info in container
even you put a container by manual"""
container_partition, containers = self.container_ring.get_nodes(account_name, container_name)
container_info = {'X-Container-Host': str(containers[0]['ip']) + ':' + str(containers[0]['port']),
'X-Container-Device': containers[0]['device'],
'X-Container-Partition': container_partition}
part, nodes = self.object_ring.get_nodes(account_name, container_name, object_name)
# there may be incompatibility problem, or just use file read
file_data = StringIO.StringIO()
with open(uploadfile_path, 'r') as upload_file:
file_data.write(upload_file.read())
# not concurrent http_connect
statuses = []
for i in range(self.upload_replica_num):
object_url = 'http://%s:%d/%s/%d/%s/%s/%s' % (nodes[i]['ip'], nodes[i]['port'],
nodes[i]['device'], part,
account_name, container_name, object_name)
parsed = urlparse(object_url)
path = parsed.path
http_url = parsed.scheme + '://' + parsed.netloc
conn = HTTPConnection(http_url)
if headers:
headers = dict(headers)
else:
headers = {}
headers['x-timestamp'] = normalize_timestamp(time.time())
headers['Content-Type'] = 'application/octet-stream'
headers.update(container_info)
conn.request('PUT', path, file_data.getvalue(), headers)
resp = conn.getresponse()
body = resp.read()
http_log(('%s%s' % (object_url.replace(parsed.path, ''), path), 'PUT',),
{'headers': headers}, resp, body)
store_response(resp, response_dict)
if resp.status < 200 or resp.status >= 300:
raise ClientException.from_response(resp, 'Object PUT failed', body)
# etag = resp.getheader('etag', '').strip('"')
statuses.append(resp.status)
return statuses
