def quorum_or_fail(self, successes, failures):
"""
Compare the number of uploads against the quorum.
:param successes: a list of chunk objects whose upload succeded
:type successes: `list` or `tuple`
:param failures: a list of chunk objects whose upload failed
:type failures: `list` or `tuple`
:raises `exc.SourceReadError`: if there is an error while reading
data from the client
:raises `exc.SourceReadTimeout`: if there is a timeout while reading
data from the client
:raises `exc.OioTimeout`: if there is a timeout among the errors
:raises `exc.ServiceBusy`: if quorum has not been reached
for any other reason
"""
if len(successes) < self.quorum:
errors = group_chunk_errors(
((chunk["url"], chunk.get("error", "success"))
for chunk in successes + failures))
new_exc = exc.ServiceBusy(
message=("RAWX write failure, quorum not reached (%d/%d): %s" %
(len(successes), self.quorum, errors)))
for err in [x.get('error') for x in failures]:
if isinstance(err, exc.SourceReadError):
raise exc.SourceReadError(new_exc)
elif isinstance(err, green.SourceReadTimeout):
# Never raise 'green' timeouts out of our API
raise exc.SourceReadTimeout(new_exc)
elif isinstance(err, (exc.OioTimeout, green.OioTimeout)):
raise exc.OioTimeout(new_exc)
raise new_exc
def test_link_rdir_fail_to_force_one(self):
"""
Verify that the failure of one 'force' operation does
not break the whole operation.
"""
self._test_link_rdir_fail_to_force(
[exc.ServiceBusy('Failed :('), None, None], exc.ServiceBusy)
def test_link_rdir_fail_to_force_several(self):
"""
Verify that the failure of two 'force' operations does
not break the whole operation.
"""
self._test_link_rdir_fail_to_force(
[exc.ServiceBusy('Failed :('),
exc.OioTimeout('Timeout :('), None], exc.OioException)
def _store_object(self, req, data_source, headers):
kwargs = {}
content_type = req.headers.get('content-type', 'octet/stream')
policy = None
container_info = self.container_info(self.account_name,
self.container_name, req)
if 'X-Oio-Storage-Policy' in req.headers:
policy = req.headers.get('X-Oio-Storage-Policy')
if not self.app.POLICIES.get_by_name(policy):
raise HTTPBadRequest(
"invalid policy '%s', must be in %s" %
(policy, self.app.POLICIES.by_name.keys()))
else:
try:
policy_index = int(
req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy']))
except TypeError:
policy_index = 0
if policy_index != 0:
policy = self.app.POLICIES.get_by_index(policy_index).name
else:
content_length = int(req.headers.get('content-length', 0))
policy = self._get_auto_policy_from_size(content_length)
ct_props = {'properties': {}, 'system': {}}
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
# only send headers if needed
if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
oio_headers[FORCEVERSIONING_HEADER] = \
headers.get(FORCEVERSIONING_HEADER)
if req.environ.get('oio.force-version'):
# In a case of MPU, it contains version of the UploadId
# to be able to include version-id of MPU in S3 reponse
kwargs['version'] = req.environ.get('oio.force-version')
# In case a shard is being created, save the name of the S3 bucket
# in a container property. This will be used when aggregating
# container statistics to make bucket statistics.
if BUCKET_NAME_HEADER in headers:
bname = headers[BUCKET_NAME_HEADER]
# FIXME(FVE): the segments container is not part of another bucket!
# We should not have to strip this here.
if bname and bname.endswith(MULTIUPLOAD_SUFFIX):
bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
ct_props['system'][BUCKET_NAME_PROP] = bname
try:
_chunks, _size, checksum, _meta = self._object_create(
self.account_name,
self.container_name,
obj_name=self.object_name,
file_or_path=data_source,
mime_type=content_type,
policy=policy,
headers=oio_headers,
etag=req.headers.get('etag', '').strip('"'),
properties=metadata,
container_properties=ct_props,
cache=oio_cache,
perfdata=perfdata,
**kwargs)
# TODO(FVE): when oio-sds supports it, do that in a callback
# passed to object_create (or whatever upload method supports it)
footer_md = self.load_object_metadata(self._get_footers(req))
if footer_md:
self.app.storage.object_set_properties(self.account_name,
self.container_name,
self.object_name,
version=_meta.get(
'version', None),
properties=footer_md,
headers=oio_headers,
cache=oio_cache,
perfdata=perfdata)
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except SourceReadTimeout as err:
self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
self.app.logger.increment('client_timeouts')
raise HTTPRequestTimeout(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except exceptions.NoSuchContainer:
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
last_modified = int(_meta.get('mtime', math.ceil(time.time())))
# FIXME(FVE): if \x10 character in object name, decode version
# number and set it in the response headers, instead of the oio
# version number.
version_id = _meta.get('version', 'null')
resp = HTTPCreated(request=req,
etag=checksum,
last_modified=last_modified,
headers={'x-object-sysmeta-version-id': version_id})
return resp
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info(
"Creating link from %s/%s/%s to %s/%s/%s",
# Existing
from_account,
container,
obj,
# New
self.account_name,
self.container_name,
self.object_name)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
# FIXME(FVE): use object_show, cache in req.environ
version = obj_version_from_env(req.environ)
props = storage.object_get_properties(from_account,
container,
obj,
headers=oio_headers,
version=version,
cache=oio_cache,
perfdata=perfdata)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
else:
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
link_meta = storage.object_link(from_account,
container,
obj,
self.account_name,
self.container_name,
self.object_name,
headers=oio_headers,
properties=metadata,
properties_directive='REPLACE',
target_version=version,
cache=oio_cache,
perfdata=perfdata)
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=link_meta['hash'])
return resp
def _store_object(self, req, data_source, headers):
content_type = req.headers.get('content-type', 'octet/stream')
storage = self.app.storage
policy = None
container_info = self.container_info(self.account_name,
self.container_name, req)
if 'X-Oio-Storage-Policy' in req.headers:
policy = req.headers.get('X-Oio-Storage-Policy')
if not self.app.POLICIES.get_by_name(policy):
raise HTTPBadRequest(
"invalid policy '%s', must be in %s" %
(policy, self.app.POLICIES.by_name.keys()))
else:
try:
policy_index = int(
req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy']))
except TypeError:
policy_index = 0
if policy_index != 0:
policy = self.app.POLICIES.get_by_index(policy_index).name
else:
content_length = int(req.headers.get('content-length', 0))
policy = self._get_auto_policy_from_size(content_length)
metadata = self.load_object_metadata(headers)
oio_headers = {'X-oio-req-id': self.trans_id}
try:
# FIXME(FVE): use 'properties' instead of 'metadata'
# as soon as we require oio>=4.2.0
_chunks, _size, checksum = storage.object_create(
self.account_name,
self.container_name,
obj_name=self.object_name,
file_or_path=data_source,
mime_type=content_type,
policy=policy,
headers=oio_headers,
etag=req.headers.get('etag', '').strip('"'),
metadata=metadata)
# TODO(FVE): when oio-sds supports it, do that in a callback
# passed to object_create (or whatever upload method supports it)
footer_md = self.load_object_metadata(self._get_footers(req))
if footer_md:
storage.object_set_properties(self.account_name,
self.container_name,
self.object_name,
properties=footer_md)
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except (SourceReadTimeout, GreenSourceReadTimeout) as err:
self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
self.app.logger.increment('client_timeouts')
raise HTTPRequestTimeout(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout):
raise
# TODO(FVE): exceptions.NotFound is never raised from oio
# Remove when dependency is oio>=4.2.0
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
# TODO(FVE): use the timestamp of the object.
# Unfortunately the oio-sds API does not return the actual ctime.
resp = HTTPCreated(request=req,
etag=checksum,
last_modified=int(time.time()))
return resp
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)", from_account,
self.container_name, self.object_name,
self.account_name, container, obj)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {'X-oio-req-id': self.trans_id}
# FIXME(FVE): use object_show, cache in req.environ
props = storage.object_get_properties(from_account, container, obj)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
if ranges is None:
raise HTTPInternalServerError(request=req,
body="LINK a MPU requires range")
self.app.logger.debug("LINK, original object is a SLO")
# retrieve manifest
_, data = storage.object_fetch(from_account, container, obj)
manifest = json.loads("".join(data))
offset = 0
# identify segment to copy
for entry in manifest:
if (ranges[0] == offset
and ranges[1] + 1 == offset + entry['bytes']):
_, container, obj = entry['name'].split('/', 2)
checksum = entry['hash']
self.app.logger.info("LINK SLO (%s,%s,%s) TO (%s,%s,%s)",
from_account, self.container_name,
self.object_name, self.account_name,
container, obj)
break
offset += entry['bytes']
else:
raise HTTPInternalServerError(request=req,
body="no segment matching range")
else:
checksum = props['hash']
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
storage.object_fastcopy(from_account,
container,
obj,
self.account_name,
self.container_name,
self.object_name,
headers=oio_headers,
properties=metadata,
properties_directive='REPLACE')
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=checksum)
return resp
def _store_object(self, req, data_source, headers):
content_type = req.headers.get('content-type', 'octet/stream')
storage = self.app.storage
policy = None
container_info = self.container_info(self.account_name,
self.container_name, req)
if 'X-Oio-Storage-Policy' in req.headers:
policy = req.headers.get('X-Oio-Storage-Policy')
if not self.app.POLICIES.get_by_name(policy):
raise HTTPBadRequest(
"invalid policy '%s', must be in %s" %
(policy, self.app.POLICIES.by_name.keys()))
else:
try:
policy_index = int(
req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy']))
except TypeError:
policy_index = 0
if policy_index != 0:
policy = self.app.POLICIES.get_by_index(policy_index).name
else:
content_length = int(req.headers.get('content-length', 0))
policy = self._get_auto_policy_from_size(content_length)
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
# only send headers if needed
if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
oio_headers[FORCEVERSIONING_HEADER] = \
headers.get(FORCEVERSIONING_HEADER)
try:
_chunks, _size, checksum, _meta = self._object_create(
self.account_name,
self.container_name,
obj_name=self.object_name,
file_or_path=data_source,
mime_type=content_type,
policy=policy,
headers=oio_headers,
etag=req.headers.get('etag', '').strip('"'),
properties=metadata)
# TODO(FVE): when oio-sds supports it, do that in a callback
# passed to object_create (or whatever upload method supports it)
footer_md = self.load_object_metadata(self._get_footers(req))
if footer_md:
storage.object_set_properties(self.account_name,
self.container_name,
self.object_name,
version=_meta.get(
'version', None),
properties=footer_md)
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except SourceReadTimeout as err:
self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
self.app.logger.increment('client_timeouts')
raise HTTPRequestTimeout(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except exceptions.NoSuchContainer:
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
last_modified = int(_meta.get('mtime', math.ceil(time.time())))
resp = HTTPCreated(request=req,
etag=checksum,
last_modified=last_modified,
headers={
'x-object-sysmeta-version-id':
_meta.get('version', None)
})
return resp
