def fill_ranges(self, start, end, length):
"""
Fill the request ranges.
"""
if length == 0:
return
if self.align and self.buf_size:
# discard bytes
# so we only yield complete EC segments
self.discard_bytes = discard_bytes(self.buf_size, start)
# change headers for efficient recovery
if 'Range' in self.request_headers:
try:
orig_ranges = ranges_from_http_header(
self.request_headers['Range'])
new_ranges = [(start, end)] + orig_ranges[1:]
except ValueError:
new_ranges = [(start, end)]
else:
new_ranges = [(start, end)]
self.request_headers['Range'] = http_header_from_ranges(
new_ranges)
def get_object_fetch_resp(self, req):
storage = self.app.storage
if req.headers.get('Range'):
ranges = ranges_from_http_header(req.headers.get('Range'))
else:
ranges = None
oio_headers = {REQID_HEADER: self.trans_id}
force_master = False
while True:
try:
metadata, stream = storage.object_fetch(
self.account_name,
self.container_name,
self.object_name,
ranges=ranges,
headers=oio_headers,
version=req.environ.get('oio.query', {}).get('version'),
force_master=force_master)
break
except (exceptions.NoSuchObject, exceptions.NoSuchContainer):
if force_master \
or not self.container_name.endswith('+segments'):
# Either the request failed with the master,
# or it is not an MPU
return HTTPNotFound(request=req)
# This part appears in the manifest, so it should be there.
# To be sure, we must go check the master
# in case of desynchronization.
force_master = True
resp = self.make_object_response(req, metadata, stream)
return resp
def _convert_range(self, req_start, req_end, length):
try:
ranges = ranges_from_http_header("bytes=%s-%s" % (
req_start if req_start is not None else '',
req_end if req_end is not None else ''))
except ValueError:
return (None, None)
result = fix_ranges(ranges, length)
if not result:
return (None, None)
else:
return (result[0][0], result[0][1])
def _convert_range(self, req_start, req_end, length):
try:
ranges = ranges_from_http_header("bytes=%s-%s" % (
req_start if req_start is not None else '',
req_end if req_end is not None else ''))
except ValueError:
return (None, None)
result = fix_ranges(ranges, length)
if not result:
return (None, None)
else:
return (result[0][0], result[0][1])
def get_object_fetch_resp(self, req):
storage = self.app.storage
if req.headers.get('Range'):
ranges = ranges_from_http_header(req.headers.get('Range'))
else:
ranges = None
oio_headers = {'X-oio-req-id': self.trans_id}
try:
metadata, stream = storage.object_fetch(
self.account_name, self.container_name, self.object_name,
ranges=ranges, headers=oio_headers,
version=req.environ.get('oio.query', {}).get('version'))
except (exceptions.NoSuchObject, exceptions.NoSuchContainer):
return HTTPNotFound(request=req)
resp = self.make_object_response(req, metadata, stream, ranges=ranges)
return resp
def get_object_fetch_resp(self, req):
storage = self.app.storage
if req.headers.get('Range'):
ranges = ranges_from_http_header(req.headers.get('Range'))
else:
ranges = None
try:
metadata, stream = storage.object_fetch(self.account_name,
self.container_name,
self.object_name,
ranges=ranges,
version=req.environ.get(
'oio_query',
{}).get('version'))
except (exceptions.NoSuchObject, exceptions.NoSuchContainer):
return HTTPNotFound(request=req)
resp = self.make_object_response(req, metadata, stream, ranges=ranges)
return resp
def recover(self, nb_bytes):
"""
Recover the request.
:param nb_bytes: number of bytes already consumed that we need to
discard if we perform a recovery from another source.
:raises `ValueError`: if range header is not valid
:raises `oio.common.exceptions.UnsatisfiableRange`:
:raises `oio.common.exceptions.EmptyByteRange`:
"""
if 'Range' in self.request_headers:
request_range = ranges_from_http_header(
self.request_headers['Range'])
start, end = request_range[0]
if start is None:
# suffix byte range
end -= nb_bytes
else:
start += nb_bytes
if end is not None:
if start == end + 1:
# no more bytes to serve in the requested byte range
raise exc.EmptyByteRange()
if start > end:
# invalid range
raise exc.UnsatisfiableRange()
if end and start:
# full byte range
request_range = [(start, end)] + request_range[1:]
else:
# suffix byte range
request_range = [(None, end)] + request_range[1:]
else:
# prefix byte range
request_range = [(start, None)] + request_range[1:]
self.request_headers['Range'] = http_header_from_ranges(
request_range)
else:
# just add an offset to the request
self.request_headers['Range'] = 'bytes=%d-' % nb_bytes
def recover(self, nb_bytes):
"""
Recover the request.
:params nb_bytes: number of bytes already consumed that we need to
discard if we perform a recovery from another source.
:raises ValueError: if range header is not valid
:raises UnsatisfiableRange
:raises EmptyByteRange
"""
if "Range" in self.request_headers:
request_range = ranges_from_http_header(self.request_headers["Range"])
start, end = request_range[0]
if start is None:
# suffix byte range
end -= nb_bytes
else:
start += nb_bytes
if end is not None:
if start == end + 1:
# no more bytes to serve in the requested byte range
raise exc.EmptyByteRange()
if start > end:
# invalid range
raise exc.UnsatisfiableRange()
if end and start:
# full byte range
request_range = [(start, end)] + request_range[1:]
else:
# suffix byte range
request_range = [(None, end)] + request_range[1:]
else:
# prefix byte range
request_range = [(start, None)] + request_range[1:]
self.request_headers["Range"] = http_header_from_ranges(request_range)
else:
# just add an offset to the request
self.request_headers["Range"] = "bytes=%d-" % nb_bytes
def fill_ranges(self, start, end, length):
"""
Fill the request ranges.
"""
if length == 0:
return
if self.buf_size:
# discard bytes
# so we only yield complete EC segments
self.discard_bytes = discard_bytes(self.buf_size, start)
# change headers for efficient recovery
if "Range" in self.request_headers:
try:
orig_ranges = ranges_from_http_header(self.request_headers["Range"])
new_ranges = [(start, end)] + orig_ranges[1:]
except ValueError:
new_ranges = [(start, end)]
else:
new_ranges = [(start, end)]
self.request_headers["Range"] = http_header_from_ranges(new_ranges)
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info(
"Creating link from %s/%s/%s to %s/%s/%s",
# Existing
from_account,
container,
obj,
# New
self.account_name,
self.container_name,
self.object_name)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
# FIXME(FVE): use object_show, cache in req.environ
version = obj_version_from_env(req.environ)
props = storage.object_get_properties(from_account,
container,
obj,
headers=oio_headers,
version=version,
cache=oio_cache,
perfdata=perfdata)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
else:
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
link_meta = storage.object_link(from_account,
container,
obj,
self.account_name,
self.container_name,
self.object_name,
headers=oio_headers,
properties=metadata,
properties_directive='REPLACE',
target_version=version,
cache=oio_cache,
perfdata=perfdata)
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=link_meta['hash'])
return resp
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)",
from_account, self.container_name,
self.object_name,
self.account_name, container, obj)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {'X-oio-req-id': self.trans_id}
# FIXME(FVE): use object_show, cache in req.environ
props = storage.object_get_properties(from_account, container, obj)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
if ranges is None:
raise HTTPInternalServerError(request=req,
body="LINK a MPU requires range")
self.app.logger.debug("LINK, original object is a SLO")
# retrieve manifest
_, data = storage.object_fetch(from_account, container, obj)
manifest = json.loads("".join(data))
offset = 0
# identify segment to copy
for entry in manifest:
if (ranges[0] == offset and
ranges[1] + 1 == offset + entry['bytes']):
_, container, obj = entry['name'].split('/', 2)
checksum = entry['hash']
self.app.logger.info(
"LINK SLO (%s,%s,%s) TO (%s,%s,%s)",
from_account, self.container_name,
self.object_name,
self.account_name, container, obj)
break
offset += entry['bytes']
else:
raise HTTPInternalServerError(
request=req, body="no segment matching range")
else:
checksum = props['hash']
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
storage.object_fastcopy(
from_account, container, obj,
self.account_name, self.container_name, self.object_name,
headers=oio_headers, properties=metadata,
properties_directive='REPLACE')
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'),
{'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'),
{'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=checksum)
return resp
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)",
from_account, self.container_name,
self.object_name,
self.account_name, container, obj)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {'X-oio-req-id': self.trans_id}
# FIXME(FVE): use object_show, cache in req.environ
version = req.environ.get('oio.query', {}).get('version')
props = storage.object_get_properties(from_account, container, obj,
headers=oio_headers,
version=version)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
else:
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
link_meta = storage.object_link(
from_account, container, obj,
self.account_name, self.container_name, self.object_name,
headers=oio_headers, properties=metadata,
properties_directive='REPLACE', target_version=version)
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'),
{'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'),
{'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=link_meta['hash'])
return resp
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)", from_account,
self.container_name, self.object_name,
self.account_name, container, obj)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {'X-oio-req-id': self.trans_id}
# FIXME(FVE): use object_show, cache in req.environ
props = storage.object_get_properties(from_account, container, obj)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
if ranges is None:
raise HTTPInternalServerError(request=req,
body="LINK a MPU requires range")
self.app.logger.debug("LINK, original object is a SLO")
# retrieve manifest
_, data = storage.object_fetch(from_account, container, obj)
manifest = json.loads("".join(data))
offset = 0
# identify segment to copy
for entry in manifest:
if (ranges[0] == offset
and ranges[1] + 1 == offset + entry['bytes']):
_, container, obj = entry['name'].split('/', 2)
checksum = entry['hash']
self.app.logger.info("LINK SLO (%s,%s,%s) TO (%s,%s,%s)",
from_account, self.container_name,
self.object_name, self.account_name,
container, obj)
break
offset += entry['bytes']
else:
raise HTTPInternalServerError(request=req,
body="no segment matching range")
else:
checksum = props['hash']
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
storage.object_fastcopy(from_account,
container,
obj,
self.account_name,
self.container_name,
self.object_name,
headers=oio_headers,
properties=metadata,
properties_directive='REPLACE')
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=checksum)
return resp